From 493100ba4d333b2008149d17871d7cc5b51f821d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 20 Aug 2020 20:34:31 +0200 Subject: [PATCH 001/709] consensus/ethash: less lookups of block data --- consensus/ethash/consensus.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index bbc554951d..50bd4efcfd 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -204,15 +204,23 @@ func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Blo number, parent := block.NumberU64()-1, block.ParentHash() for i := 0; i < 7; i++ { - ancestor := chain.GetBlock(parent, number) - if ancestor == nil { + ancestorHeader := chain.GetHeader(parent, number) + if ancestorHeader == nil { break } - ancestors[ancestor.Hash()] = ancestor.Header() - for _, uncle := range ancestor.Uncles() { - uncles.Add(uncle.Hash()) + ancestors[parent] = ancestorHeader + // If the ancestor doesn't have any uncles, we don't have to iterate them + if ancestorHeader.UncleHash != types.EmptyUncleHash { + // Need to add those uncles to the blacklist too + ancestor := chain.GetBlock(parent, number) + if ancestor == nil { + break + } + for _, uncle := range ancestor.Uncles() { + uncles.Add(uncle.Hash()) + } } - parent, number = ancestor.ParentHash(), number-1 + parent, number = ancestorHeader.ParentHash, number-1 } ancestors[block.Hash()] = block.Header() uncles.Add(block.Hash()) From fc0662bb2372f9a94dea60c3dcf122afd6bda55f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 11 Dec 2020 08:59:46 +0100 Subject: [PATCH 002/709] params: begin v1.9.26 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 3237b51f85..b6d6bd3f1d 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 25 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 26 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 1a715d7db57997307d309a498e8f819dd08725ad Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 11 Dec 2020 09:28:01 +0000 Subject: [PATCH 003/709] les: rework float conversion on arm64 and other architectures (#21994) The previous fix #21960 converted the float to an intermediate signed int, before attempting the uint conversion. Although this works, this doesn't guarantee that other architectures will work the same. --- les/utils/expiredvalue.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/les/utils/expiredvalue.go b/les/utils/expiredvalue.go index 1a2b3d995e..3fd52616fa 100644 --- a/les/utils/expiredvalue.go +++ b/les/utils/expiredvalue.go @@ -86,11 +86,15 @@ func (e *ExpiredValue) Add(amount int64, logOffset Fixed64) int64 { e.Exp = integer } if base >= 0 || uint64(-base) <= e.Base { - // This is a temporary fix to circumvent a golang - // uint conversion issue on arm64, which needs to - // be investigated further. More details at: + // The conversion from negative float64 to + // uint64 is undefined in golang, and doesn't + // work with ARMv8. More details at: // https://github.com/golang/go/issues/43047 - e.Base += uint64(int64(base)) + if base >= 0 { + e.Base += uint64(base) + } else { + e.Base -= uint64(-base) + } return amount } net := int64(-float64(e.Base) / factor) From 62dc59c2bd6c80b711e873300d7cb91afa91e830 Mon Sep 17 00:00:00 2001 From: lzhfromustc <43191155+lzhfromustc@users.noreply.github.com> Date: Fri, 11 Dec 2020 04:29:42 -0500 Subject: [PATCH 004/709] miner, test: fix potential goroutine leak (#21989) In miner/worker.go, there are two goroutine using channel w.newWorkCh: newWorkerLoop() sends to this channel, and mainLoop() receives from this channel. Only the receive operation is in a select. However, w.exitCh may be closed by another goroutine. This is fine for the receive since receive is in select, but if the send operation is blocking, then it will block forever. This commit puts the send in a select, so it won't block even if w.exitCh is closed. Similarly, there are two goroutines using channel errc: the parent that runs the test receives from it, and the child created at line 573 sends to it. If the parent goroutine exits too early by calling t.Fatalf() at line 614, then the child goroutine will be blocked at line 574 forever. This commit adds 1 buffer to errc. Now send will not block, and receive is not influenced because receive still needs to wait for the send. --- eth/downloader/downloader_test.go | 2 +- miner/worker.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 7645f04e4f..5e46042ae4 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -569,7 +569,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { <-proceed } // Start a synchronisation concurrently - errc := make(chan error) + errc := make(chan error, 1) go func() { errc <- tester.sync("peer", nil, mode) }() diff --git a/miner/worker.go b/miner/worker.go index d5813c18a8..5f07affdc4 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -347,7 +347,11 @@ func (w *worker) newWorkLoop(recommit time.Duration) { atomic.StoreInt32(interrupt, s) } interrupt = new(int32) - w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty, timestamp: timestamp} + select { + case w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty, timestamp: timestamp}: + case <-w.exitCh: + return + } timer.Reset(recommit) atomic.StoreInt32(&w.newTxs, 0) } From b47f4ca5cf3adf7c29e9ee00a6056196f295763c Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Fri, 11 Dec 2020 15:05:39 +0530 Subject: [PATCH 005/709] cmd/faucet: use Twitter API instead of scraping webpage (#21850) This PR adds support for using Twitter API to query the tweet and author details. There are two reasons behind this change: - Twitter will be deprecating the legacy website on 15th December. The current method is expected to stop working then. - More importantly, the current system uses Twitter handle for spam protection but the Twitter handle can be changed via automated calls. This allows bots to use the same tweet to withdraw funds infinite times as long as they keep changing their handle between every request. The Rinkeby as well as the Goerli faucet are being actively drained via this method. This PR changes the spam protection to be based on Twitter IDs instead of usernames. A user can not change their Twitter ID. --- cmd/faucet/faucet.go | 93 +++++++++++++++++++++++++++++++----- cmd/puppeth/module_faucet.go | 7 +++ cmd/puppeth/wizard_faucet.go | 23 +++++++++ 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index eaf0dc30c1..d7927ac491 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -83,6 +83,8 @@ var ( noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication") logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet") + + twitterBearerToken = flag.String("twitter.token", "", "Twitter bearer token to authenticate with the twitter API") ) var ( @@ -443,6 +445,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } // Retrieve the Ethereum address to fund, the requesting user and a profile picture var ( + id string username string avatar string address common.Address @@ -462,11 +465,13 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue case strings.HasPrefix(msg.URL, "https://twitter.com/"): - username, avatar, address, err = authTwitter(msg.URL) + id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken) case strings.HasPrefix(msg.URL, "https://www.facebook.com/"): username, avatar, address, err = authFacebook(msg.URL) + id = username case *noauthFlag: username, avatar, address, err = authNoAuth(msg.URL) + id = username default: //lint:ignore ST1005 This error is to be displayed in the browser err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues") @@ -486,7 +491,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { fund bool timeout time.Time ) - if timeout = f.timeouts[username]; time.Now().After(timeout) { + if timeout = f.timeouts[id]; time.Now().After(timeout) { // User wasn't funded recently, create the funding transaction amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether) amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil)) @@ -520,7 +525,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute grace := timeout / 288 // 24h timeout => 5m grace - f.timeouts[username] = time.Now().Add(timeout - grace) + f.timeouts[id] = time.Now().Add(timeout - grace) fund = true } f.lock.Unlock() @@ -684,23 +689,32 @@ func sendSuccess(conn *websocket.Conn, msg string) error { } // authTwitter tries to authenticate a faucet request using Twitter posts, returning -// the username, avatar URL and Ethereum address to fund on success. -func authTwitter(url string) (string, string, common.Address, error) { +// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. +func authTwitter(url string, token string) (string, string, string, common.Address, error) { // Ensure the user specified a meaningful URL, no fancy nonsense parts := strings.Split(url, "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") + } + + // Twitter's API isn't really friendly with direct links. + // It is restricted to 300 queries / 15 minute with an app api key. + // Anything more will require read only authorization from the users and that we want to avoid. + + // If twitter bearer token is provided, use the twitter api + if token != "" { + return authTwitterWithToken(parts[len(parts)-1], token) } - // Twitter's API isn't really friendly with direct links. Still, we don't - // want to do ask read permissions from users, so just load the public posts + + // Twiter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) res, err := http.Get(url) if err != nil { - return "", "", common.Address{}, err + return "", "", "", common.Address{}, err } defer res.Body.Close() @@ -708,24 +722,77 @@ func authTwitter(url string) (string, string, common.Address, error) { parts = strings.Split(res.Request.URL.String(), "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") } username := parts[len(parts)-3] body, err := ioutil.ReadAll(res.Body) if err != nil { - return "", "", common.Address{}, err + return "", "", "", common.Address{}, err } address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("No Ethereum address found to fund") + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") } var avatar string if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 { avatar = parts[1] } - return username + "@twitter", avatar, address, nil + return username + "@twitter", username, avatar, address, nil +} + +// authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning +// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. +func authTwitterWithToken(tweetID string, token string) (string, string, string, common.Address, error) { + // Strip any query parameters from the tweet id + sanitizedTweetID := strings.Split(tweetID, "?")[0] + + // Ensure numeric tweetID + if !regexp.MustCompile("^[0-9]+$").MatchString(sanitizedTweetID) { + return "", "", "", common.Address{}, errors.New("Invalid Tweet URL") + } + + // Query the tweet details from Twitter + url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", "", common.Address{}, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", "", common.Address{}, err + } + defer res.Body.Close() + + var result struct { + Data struct { + AuthorID string `json:"author_id"` + ID string `json:"id"` + Text string `json:"text"` + } `json:"data"` + Includes struct { + Users []struct { + ProfileImageURL string `json:"profile_image_url"` + Username string `json:"username"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"users"` + } `json:"includes"` + } + + err = json.NewDecoder(res.Body).Decode(&result) + if err != nil { + return "", "", "", common.Address{}, err + } + + address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text)) + if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") + } + return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil } // authFacebook tries to authenticate a faucet request using Facebook posts, diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 987bed14aa..2527e137f2 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -46,6 +46,7 @@ ENTRYPOINT [ \ "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}", \ "--account.json", "/account.json", "--account.pass", "/account.pass" \ {{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}} \ + {{if .TwitterToken}}, "--twitter.token", "{{.TwitterToken}}", ]` // faucetComposefile is the docker-compose.yml file required to deploy and maintain @@ -71,6 +72,7 @@ services: - FAUCET_TIERS={{.FaucetTiers}} - CAPTCHA_TOKEN={{.CaptchaToken}} - CAPTCHA_SECRET={{.CaptchaSecret}} + - TWITTER_TOKEN={{.TwitterToken}} - NO_AUTH={{.NoAuth}}{{if .VHost}} - VIRTUAL_HOST={{.VHost}} - VIRTUAL_PORT=8080{{end}} @@ -103,6 +105,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "FaucetMinutes": config.minutes, "FaucetTiers": config.tiers, "NoAuth": config.noauth, + "TwitterToken": config.twitterToken, }) files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes() @@ -120,6 +123,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "FaucetMinutes": config.minutes, "FaucetTiers": config.tiers, "NoAuth": config.noauth, + "TwitterToken": config.twitterToken, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -152,6 +156,7 @@ type faucetInfos struct { noauth bool captchaToken string captchaSecret string + twitterToken string } // Report converts the typed struct into a plain string->string map, containing @@ -165,6 +170,7 @@ func (info *faucetInfos) Report() map[string]string { "Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes), "Funding tiers": strconv.Itoa(info.tiers), "Captha protection": fmt.Sprintf("%v", info.captchaToken != ""), + "Using Twitter API": fmt.Sprintf("%v", info.twitterToken != ""), "Ethstats username": info.node.ethstats, } if info.noauth { @@ -243,5 +249,6 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) { captchaToken: infos.envvars["CAPTCHA_TOKEN"], captchaSecret: infos.envvars["CAPTCHA_SECRET"], noauth: infos.envvars["NO_AUTH"] == "true", + twitterToken: infos.envvars["TWITTER_TOKEN"], }, nil } diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 9f753ad68b..47e05cd9c1 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -102,6 +102,29 @@ func (w *wizard) deployFaucet() { infos.captchaSecret = w.readPassword() } } + + // Accessing the twitter api requires a bearer token, request it + if infos.twitterToken != "" { + fmt.Println() + fmt.Println("Reuse previous twitter API Bearer token (y/n)? (default = yes)") + if !w.readDefaultYesNo(true) { + infos.twitterToken = "" + } + } + if infos.twitterToken == "" { + // No previous twitter token (or old one discarded) + fmt.Println() + fmt.Println("Enable twitter API (y/n)? (default = no)") + if !w.readDefaultYesNo(false) { + log.Warn("The faucet will fallback to using direct calls") + } else { + // Twitter api explicitly requested, read the bearer token + fmt.Println() + fmt.Printf("What is the twitter API Bearer token?\n") + infos.twitterToken = w.readString() + } + } + // Figure out where the user wants to store the persistent data fmt.Println() if infos.node.datadir == "" { From 88c696240dc7dfd99588d9e2ef0b04f03a06d1a5 Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 11 Dec 2020 17:44:57 +0800 Subject: [PATCH 006/709] core/txpool: remove "local" notion from the txpool price heap (#21478) * core: separate the local notion from the pricedHeap * core: add benchmarks * core: improve tests * core: address comments * core: degrade the panic to error message * core: fix typo * core: address comments * core: address comment * core: use PEAK instead of POP * core: address comments --- core/tx_list.go | 152 +++++++++++++++------------------ core/tx_pool.go | 195 +++++++++++++++++++++++++++++++++---------- core/tx_pool_test.go | 69 ++++++++++++--- 3 files changed, 271 insertions(+), 145 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index cdd3df14c5..894640d570 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" ) // nonceHeap is a heap.Interface implementation over 64bit unsigned integers for @@ -439,24 +438,29 @@ func (h *priceHeap) Pop() interface{} { } // txPricedList is a price-sorted heap to allow operating on transactions pool -// contents in a price-incrementing way. +// contents in a price-incrementing way. It's built opon the all transactions +// in txpool but only interested in the remote part. It means only remote transactions +// will be considered for tracking, sorting, eviction, etc. type txPricedList struct { - all *txLookup // Pointer to the map of all transactions - items *priceHeap // Heap of prices of all the stored transactions - stales int // Number of stale price points to (re-heap trigger) + all *txLookup // Pointer to the map of all transactions + remotes *priceHeap // Heap of prices of all the stored **remote** transactions + stales int // Number of stale price points to (re-heap trigger) } // newTxPricedList creates a new price-sorted transaction heap. func newTxPricedList(all *txLookup) *txPricedList { return &txPricedList{ - all: all, - items: new(priceHeap), + all: all, + remotes: new(priceHeap), } } // Put inserts a new transaction into the heap. -func (l *txPricedList) Put(tx *types.Transaction) { - heap.Push(l.items, tx) +func (l *txPricedList) Put(tx *types.Transaction, local bool) { + if local { + return + } + heap.Push(l.remotes, tx) } // Removed notifies the prices transaction list that an old transaction dropped @@ -465,121 +469,95 @@ func (l *txPricedList) Put(tx *types.Transaction) { func (l *txPricedList) Removed(count int) { // Bump the stale counter, but exit if still too low (< 25%) l.stales += count - if l.stales <= len(*l.items)/4 { + if l.stales <= len(*l.remotes)/4 { return } // Seems we've reached a critical number of stale transactions, reheap - reheap := make(priceHeap, 0, l.all.Count()) - - l.stales, l.items = 0, &reheap - l.all.Range(func(hash common.Hash, tx *types.Transaction) bool { - *l.items = append(*l.items, tx) - return true - }) - heap.Init(l.items) + l.Reheap() } // Cap finds all the transactions below the given price threshold, drops them // from the priced list and returns them for further removal from the entire pool. -func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transactions { +// +// Note: only remote transactions will be considered for eviction. +func (l *txPricedList) Cap(threshold *big.Int) types.Transactions { drop := make(types.Transactions, 0, 128) // Remote underpriced transactions to drop - save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep - - for len(*l.items) > 0 { + for len(*l.remotes) > 0 { // Discard stale transactions if found during cleanup - tx := heap.Pop(l.items).(*types.Transaction) - if l.all.Get(tx.Hash()) == nil { + cheapest := (*l.remotes)[0] + if l.all.GetRemote(cheapest.Hash()) == nil { // Removed or migrated + heap.Pop(l.remotes) l.stales-- continue } // Stop the discards if we've reached the threshold - if tx.GasPriceIntCmp(threshold) >= 0 { - save = append(save, tx) + if cheapest.GasPriceIntCmp(threshold) >= 0 { break } - // Non stale transaction found, discard unless local - if local.containsTx(tx) { - save = append(save, tx) - } else { - drop = append(drop, tx) - } - } - for _, tx := range save { - heap.Push(l.items, tx) + heap.Pop(l.remotes) + drop = append(drop, cheapest) } return drop } // Underpriced checks whether a transaction is cheaper than (or as cheap as) the -// lowest priced transaction currently being tracked. -func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) bool { - // Local transactions cannot be underpriced - if local.containsTx(tx) { - return false - } +// lowest priced (remote) transaction currently being tracked. +func (l *txPricedList) Underpriced(tx *types.Transaction) bool { // Discard stale price points if found at the heap start - for len(*l.items) > 0 { - head := []*types.Transaction(*l.items)[0] - if l.all.Get(head.Hash()) == nil { + for len(*l.remotes) > 0 { + head := []*types.Transaction(*l.remotes)[0] + if l.all.GetRemote(head.Hash()) == nil { // Removed or migrated l.stales-- - heap.Pop(l.items) + heap.Pop(l.remotes) continue } break } // Check if the transaction is underpriced or not - if len(*l.items) == 0 { - log.Error("Pricing query for empty pool") // This cannot happen, print to catch programming errors - return false + if len(*l.remotes) == 0 { + return false // There is no remote transaction at all. } - cheapest := []*types.Transaction(*l.items)[0] + // If the remote transaction is even cheaper than the + // cheapest one tracked locally, reject it. + cheapest := []*types.Transaction(*l.remotes)[0] return cheapest.GasPriceCmp(tx) >= 0 } // Discard finds a number of most underpriced transactions, removes them from the // priced list and returns them for further removal from the entire pool. -func (l *txPricedList) Discard(slots int, local *accountSet) types.Transactions { - // If we have some local accountset, those will not be discarded - if !local.empty() { - // In case the list is filled to the brim with 'local' txs, we do this - // little check to avoid unpacking / repacking the heap later on, which - // is very expensive - discardable := 0 - for _, tx := range *l.items { - if !local.containsTx(tx) { - discardable++ - } - if discardable >= slots { - break - } - } - if slots > discardable { - slots = discardable - } - } - if slots == 0 { - return nil - } - drop := make(types.Transactions, 0, slots) // Remote underpriced transactions to drop - save := make(types.Transactions, 0, len(*l.items)-slots) // Local underpriced transactions to keep - - for len(*l.items) > 0 && slots > 0 { +// +// Note local transaction won't be considered for eviction. +func (l *txPricedList) Discard(slots int, force bool) (types.Transactions, bool) { + drop := make(types.Transactions, 0, slots) // Remote underpriced transactions to drop + for len(*l.remotes) > 0 && slots > 0 { // Discard stale transactions if found during cleanup - tx := heap.Pop(l.items).(*types.Transaction) - if l.all.Get(tx.Hash()) == nil { + tx := heap.Pop(l.remotes).(*types.Transaction) + if l.all.GetRemote(tx.Hash()) == nil { // Removed or migrated l.stales-- continue } - // Non stale transaction found, discard unless local - if local.containsTx(tx) { - save = append(save, tx) - } else { - drop = append(drop, tx) - slots -= numSlots(tx) + // Non stale transaction found, discard it + drop = append(drop, tx) + slots -= numSlots(tx) + } + // If we still can't make enough room for the new transaction + if slots > 0 && !force { + for _, tx := range drop { + heap.Push(l.remotes, tx) } + return nil, false } - for _, tx := range save { - heap.Push(l.items, tx) - } - return drop + return drop, true +} + +// Reheap forcibly rebuilds the heap based on the current remote transaction set. +func (l *txPricedList) Reheap() { + reheap := make(priceHeap, 0, l.all.RemoteCount()) + + l.stales, l.remotes = 0, &reheap + l.all.Range(func(hash common.Hash, tx *types.Transaction, local bool) bool { + *l.remotes = append(*l.remotes, tx) + return true + }, false, true) // Only iterate remotes + heap.Init(l.remotes) } diff --git a/core/tx_pool.go b/core/tx_pool.go index e3ffe103cf..4a17c31ca8 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -63,6 +63,10 @@ var ( // configured for the transaction pool. ErrUnderpriced = errors.New("transaction underpriced") + // ErrTxPoolOverflow is returned if the transaction pool is full and can't accpet + // another remote transaction. + ErrTxPoolOverflow = errors.New("txpool is full") + // ErrReplaceUnderpriced is returned if a transaction is attempted to be replaced // with a different one without the required price bump. ErrReplaceUnderpriced = errors.New("replacement transaction underpriced") @@ -105,6 +109,7 @@ var ( validTxMeter = metrics.NewRegisteredMeter("txpool/valid", nil) invalidTxMeter = metrics.NewRegisteredMeter("txpool/invalid", nil) underpricedTxMeter = metrics.NewRegisteredMeter("txpool/underpriced", nil) + overflowedTxMeter = metrics.NewRegisteredMeter("txpool/overflowed", nil) pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) @@ -421,7 +426,7 @@ func (pool *TxPool) SetGasPrice(price *big.Int) { defer pool.mu.Unlock() pool.gasPrice = price - for _, tx := range pool.priced.Cap(price, pool.locals) { + for _, tx := range pool.priced.Cap(price) { pool.removeTx(tx.Hash(), false) } log.Info("Transaction pool price threshold updated", "price", price) @@ -536,7 +541,6 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { return ErrInvalidSender } // Drop non-local transactions under our own minimal accepted gas price - local = local || pool.locals.contains(from) // account may be local even if the transaction arrived from the network if !local && tx.GasPriceIntCmp(pool.gasPrice) < 0 { return ErrUnderpriced } @@ -575,22 +579,36 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e knownTxMeter.Mark(1) return false, ErrAlreadyKnown } + // Make the local flag. If it's from local source or it's from the network but + // the sender is marked as local previously, treat it as the local transaction. + isLocal := local || pool.locals.containsTx(tx) + // If the transaction fails basic validation, discard it - if err := pool.validateTx(tx, local); err != nil { + if err := pool.validateTx(tx, isLocal); err != nil { log.Trace("Discarding invalid transaction", "hash", hash, "err", err) invalidTxMeter.Mark(1) return false, err } // If the transaction pool is full, discard underpriced transactions - if uint64(pool.all.Count()) >= pool.config.GlobalSlots+pool.config.GlobalQueue { + if uint64(pool.all.Count()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it - if !local && pool.priced.Underpriced(tx, pool.locals) { + if !isLocal && pool.priced.Underpriced(tx) { log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice()) underpricedTxMeter.Mark(1) return false, ErrUnderpriced } - // New transaction is better than our worse ones, make room for it - drop := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), pool.locals) + // New transaction is better than our worse ones, make room for it. + // If it's a local transaction, forcibly discard all available transactions. + // Otherwise if we can't make enough room for new one, abort the operation. + drop, success := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), isLocal) + + // Special case, we still can't make the room for the new remote one. + if !isLocal && !success { + log.Trace("Discarding overflown transaction", "hash", hash) + overflowedTxMeter.Mark(1) + return false, ErrTxPoolOverflow + } + // Kick out the underpriced remote transactions. for _, tx := range drop { log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice()) underpricedTxMeter.Mark(1) @@ -612,8 +630,8 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e pool.priced.Removed(1) pendingReplaceMeter.Mark(1) } - pool.all.Add(tx) - pool.priced.Put(tx) + pool.all.Add(tx, isLocal) + pool.priced.Put(tx, isLocal) pool.journalTx(from, tx) pool.queueTxEvent(tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) @@ -623,18 +641,17 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e return old != nil, nil } // New transaction isn't replacing a pending one, push into queue - replaced, err = pool.enqueueTx(hash, tx) + replaced, err = pool.enqueueTx(hash, tx, isLocal, true) if err != nil { return false, err } // Mark local addresses and journal local transactions - if local { - if !pool.locals.contains(from) { - log.Info("Setting new local account", "address", from) - pool.locals.add(from) - } + if local && !pool.locals.contains(from) { + log.Info("Setting new local account", "address", from) + pool.locals.add(from) + pool.priced.Removed(pool.all.RemoteToLocals(pool.locals)) // Migrate the remotes if it's marked as local first time. } - if local || pool.locals.contains(from) { + if isLocal { localGauge.Inc(1) } pool.journalTx(from, tx) @@ -646,7 +663,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e // enqueueTx inserts a new transaction into the non-executable transaction queue. // // Note, this method assumes the pool lock is held! -func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, error) { +func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction, local bool, addAll bool) (bool, error) { // Try to insert the transaction into the future queue from, _ := types.Sender(pool.signer, tx) // already validated if pool.queue[from] == nil { @@ -667,9 +684,14 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er // Nothing was replaced, bump the queued counter queuedGauge.Inc(1) } - if pool.all.Get(hash) == nil { - pool.all.Add(tx) - pool.priced.Put(tx) + // If the transaction isn't in lookup set but it's expected to be there, + // show the error log. + if pool.all.Get(hash) == nil && !addAll { + log.Error("Missing transaction in lookup set, please report the issue", "hash", hash) + } + if addAll { + pool.all.Add(tx, local) + pool.priced.Put(tx, local) } // If we never record the heartbeat, do it right now. if _, exist := pool.beats[from]; !exist { @@ -718,11 +740,6 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T // Nothing was replaced, bump the pending counter pendingGauge.Inc(1) } - // Failsafe to work around direct pending inserts (tests) - if pool.all.Get(hash) == nil { - pool.all.Add(tx) - pool.priced.Put(tx) - } // Set the potentially new pending nonce and notify any subsystems of the new tx pool.pendingNonces.set(addr, tx.Nonce()+1) @@ -904,7 +921,8 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { } // Postpone any invalidated transactions for _, tx := range invalids { - pool.enqueueTx(tx.Hash(), tx) + // Internal shuffle shouldn't touch the lookup set. + pool.enqueueTx(tx.Hash(), tx, false, false) } // Update the account nonce if needed pool.pendingNonces.setIfLower(addr, tx.Nonce()) @@ -1408,7 +1426,9 @@ func (pool *TxPool) demoteUnexecutables() { for _, tx := range invalids { hash := tx.Hash() log.Trace("Demoting pending transaction", "hash", hash) - pool.enqueueTx(hash, tx) + + // Internal shuffle shouldn't touch the lookup set. + pool.enqueueTx(hash, tx, false, false) } pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) if pool.locals.contains(addr) { @@ -1420,7 +1440,9 @@ func (pool *TxPool) demoteUnexecutables() { for _, tx := range gapped { hash := tx.Hash() log.Error("Demoting invalidated transaction", "hash", hash) - pool.enqueueTx(hash, tx) + + // Internal shuffle shouldn't touch the lookup set. + pool.enqueueTx(hash, tx, false, false) } pendingGauge.Dec(int64(len(gapped))) // This might happen in a reorg, so log it to the metering @@ -1519,8 +1541,8 @@ func (as *accountSet) merge(other *accountSet) { as.cache = nil } -// txLookup is used internally by TxPool to track transactions while allowing lookup without -// mutex contention. +// txLookup is used internally by TxPool to track transactions while allowing +// lookup without mutex contention. // // Note, although this type is properly protected against concurrent access, it // is **not** a type that should ever be mutated or even exposed outside of the @@ -1528,27 +1550,43 @@ func (as *accountSet) merge(other *accountSet) { // internal mechanisms. The sole purpose of the type is to permit out-of-bound // peeking into the pool in TxPool.Get without having to acquire the widely scoped // TxPool.mu mutex. +// +// This lookup set combines the notion of "local transactions", which is useful +// to build upper-level structure. type txLookup struct { - all map[common.Hash]*types.Transaction - slots int - lock sync.RWMutex + slots int + lock sync.RWMutex + locals map[common.Hash]*types.Transaction + remotes map[common.Hash]*types.Transaction } // newTxLookup returns a new txLookup structure. func newTxLookup() *txLookup { return &txLookup{ - all: make(map[common.Hash]*types.Transaction), + locals: make(map[common.Hash]*types.Transaction), + remotes: make(map[common.Hash]*types.Transaction), } } -// Range calls f on each key and value present in the map. -func (t *txLookup) Range(f func(hash common.Hash, tx *types.Transaction) bool) { +// Range calls f on each key and value present in the map. The callback passed +// should return the indicator whether the iteration needs to be continued. +// Callers need to specify which set (or both) to be iterated. +func (t *txLookup) Range(f func(hash common.Hash, tx *types.Transaction, local bool) bool, local bool, remote bool) { t.lock.RLock() defer t.lock.RUnlock() - for key, value := range t.all { - if !f(key, value) { - break + if local { + for key, value := range t.locals { + if !f(key, value, true) { + return + } + } + } + if remote { + for key, value := range t.remotes { + if !f(key, value, false) { + return + } } } } @@ -1558,15 +1596,50 @@ func (t *txLookup) Get(hash common.Hash) *types.Transaction { t.lock.RLock() defer t.lock.RUnlock() - return t.all[hash] + if tx := t.locals[hash]; tx != nil { + return tx + } + return t.remotes[hash] } -// Count returns the current number of items in the lookup. +// GetLocal returns a transaction if it exists in the lookup, or nil if not found. +func (t *txLookup) GetLocal(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.locals[hash] +} + +// GetRemote returns a transaction if it exists in the lookup, or nil if not found. +func (t *txLookup) GetRemote(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.remotes[hash] +} + +// Count returns the current number of transactions in the lookup. func (t *txLookup) Count() int { t.lock.RLock() defer t.lock.RUnlock() - return len(t.all) + return len(t.locals) + len(t.remotes) +} + +// LocalCount returns the current number of local transactions in the lookup. +func (t *txLookup) LocalCount() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.locals) +} + +// RemoteCount returns the current number of remote transactions in the lookup. +func (t *txLookup) RemoteCount() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.remotes) } // Slots returns the current number of slots used in the lookup. @@ -1578,14 +1651,18 @@ func (t *txLookup) Slots() int { } // Add adds a transaction to the lookup. -func (t *txLookup) Add(tx *types.Transaction) { +func (t *txLookup) Add(tx *types.Transaction, local bool) { t.lock.Lock() defer t.lock.Unlock() t.slots += numSlots(tx) slotsGauge.Update(int64(t.slots)) - t.all[tx.Hash()] = tx + if local { + t.locals[tx.Hash()] = tx + } else { + t.remotes[tx.Hash()] = tx + } } // Remove removes a transaction from the lookup. @@ -1593,10 +1670,36 @@ func (t *txLookup) Remove(hash common.Hash) { t.lock.Lock() defer t.lock.Unlock() - t.slots -= numSlots(t.all[hash]) + tx, ok := t.locals[hash] + if !ok { + tx, ok = t.remotes[hash] + } + if !ok { + log.Error("No transaction found to be deleted", "hash", hash) + return + } + t.slots -= numSlots(tx) slotsGauge.Update(int64(t.slots)) - delete(t.all, hash) + delete(t.locals, hash) + delete(t.remotes, hash) +} + +// RemoteToLocals migrates the transactions belongs to the given locals to locals +// set. The assumption is held the locals set is thread-safe to be used. +func (t *txLookup) RemoteToLocals(locals *accountSet) int { + t.lock.Lock() + defer t.lock.Unlock() + + var migrated int + for hash, tx := range t.remotes { + if locals.containsTx(tx) { + t.locals[hash] = tx + delete(t.remotes, hash) + migrated += 1 + } + } + return migrated } // numSlots calculates the number of slots needed for a single transaction. diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 246b3977d3..47d3830b06 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -107,10 +107,11 @@ func validateTxPoolInternals(pool *TxPool) error { if total := pool.all.Count(); total != pending+queued { return fmt.Errorf("total transaction count %d != %d pending + %d queued", total, pending, queued) } - if priced := pool.priced.items.Len() - pool.priced.stales; priced != pending+queued { - return fmt.Errorf("total priced transaction count %d != %d pending + %d queued", priced, pending, queued) + pool.priced.Reheap() + priced, remote := pool.priced.remotes.Len(), pool.all.RemoteCount() + if priced != remote { + return fmt.Errorf("total priced transaction count %d != %d", priced, remote) } - // Ensure the next nonce to assign is the correct one for addr, txs := range pool.pending { // Find the last transaction @@ -280,7 +281,7 @@ func TestTransactionQueue(t *testing.T) { pool.currentState.AddBalance(from, big.NewInt(1000)) <-pool.requestReset(nil, nil) - pool.enqueueTx(tx.Hash(), tx) + pool.enqueueTx(tx.Hash(), tx, false, true) <-pool.requestPromoteExecutables(newAccountSet(pool.signer, from)) if len(pool.pending) != 1 { t.Error("expected valid txs to be 1 is", len(pool.pending)) @@ -289,7 +290,7 @@ func TestTransactionQueue(t *testing.T) { tx = transaction(1, 100, key) from, _ = deriveSender(tx) pool.currentState.SetNonce(from, 2) - pool.enqueueTx(tx.Hash(), tx) + pool.enqueueTx(tx.Hash(), tx, false, true) <-pool.requestPromoteExecutables(newAccountSet(pool.signer, from)) if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { @@ -313,9 +314,9 @@ func TestTransactionQueue2(t *testing.T) { pool.currentState.AddBalance(from, big.NewInt(1000)) pool.reset(nil, nil) - pool.enqueueTx(tx1.Hash(), tx1) - pool.enqueueTx(tx2.Hash(), tx2) - pool.enqueueTx(tx3.Hash(), tx3) + pool.enqueueTx(tx1.Hash(), tx1, false, true) + pool.enqueueTx(tx2.Hash(), tx2, false, true) + pool.enqueueTx(tx3.Hash(), tx3, false, true) pool.promoteExecutables([]common.Address{from}) if len(pool.pending) != 1 { @@ -488,12 +489,21 @@ func TestTransactionDropping(t *testing.T) { tx11 = transaction(11, 200, key) tx12 = transaction(12, 300, key) ) + pool.all.Add(tx0, false) + pool.priced.Put(tx0, false) pool.promoteTx(account, tx0.Hash(), tx0) + + pool.all.Add(tx1, false) + pool.priced.Put(tx1, false) pool.promoteTx(account, tx1.Hash(), tx1) + + pool.all.Add(tx2, false) + pool.priced.Put(tx2, false) pool.promoteTx(account, tx2.Hash(), tx2) - pool.enqueueTx(tx10.Hash(), tx10) - pool.enqueueTx(tx11.Hash(), tx11) - pool.enqueueTx(tx12.Hash(), tx12) + + pool.enqueueTx(tx10.Hash(), tx10, false, true) + pool.enqueueTx(tx11.Hash(), tx11, false, true) + pool.enqueueTx(tx12.Hash(), tx12, false, true) // Check that pre and post validations leave the pool as is if pool.pending[account].Len() != 3 { @@ -1964,7 +1974,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { for i := 0; i < size; i++ { tx := transaction(uint64(1+i), 100000, key) - pool.enqueueTx(tx.Hash(), tx) + pool.enqueueTx(tx.Hash(), tx, false, true) } // Benchmark the speed of pool validation b.ResetTimer() @@ -2007,3 +2017,38 @@ func benchmarkPoolBatchInsert(b *testing.B, size int, local bool) { } } } + +func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { + // Allocate keys for testing + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + + remoteKey, _ := crypto.GenerateKey() + remoteAddr := crypto.PubkeyToAddress(remoteKey.PublicKey) + + locals := make([]*types.Transaction, 4096+1024) // Occupy all slots + for i := 0; i < len(locals); i++ { + locals[i] = transaction(uint64(i), 100000, key) + } + remotes := make([]*types.Transaction, 1000) + for i := 0; i < len(remotes); i++ { + remotes[i] = pricedTransaction(uint64(i), 100000, big.NewInt(2), remoteKey) // Higher gasprice + } + // Benchmark importing the transactions into the queue + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + pool, _ := setupTxPool() + pool.currentState.AddBalance(account, big.NewInt(100000000)) + for _, local := range locals { + pool.AddLocal(local) + } + b.StartTimer() + // Assign a high enough balance for testing + pool.currentState.AddBalance(remoteAddr, big.NewInt(100000000)) + for i := 0; i < len(remotes); i++ { + pool.AddRemotes([]*types.Transaction{remotes[i]}) + } + pool.Stop() + } +} From efe6dd29042b36d543420a422fc21d123f1e67e3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 11 Dec 2020 11:06:44 +0100 Subject: [PATCH 007/709] consensus/ethash: implement faster difficulty calculators (#21976) This PR adds re-written difficulty calculators, which are based on uint256. It also adds a fuzzer + oss-fuzz integration for the new fuzzer. It does differential fuzzing between the new and old calculators. Note: this PR does not actually enable the new calculators. --- consensus/ethash/consensus.go | 5 + consensus/ethash/consensus_test.go | 102 +++++++++++ consensus/ethash/difficulty.go | 193 ++++++++++++++++++++ oss-fuzz.sh | 1 + tests/fuzzers/difficulty/debug/main.go | 23 +++ tests/fuzzers/difficulty/difficulty-fuzz.go | 145 +++++++++++++++ 6 files changed, 469 insertions(+) create mode 100644 consensus/ethash/difficulty.go create mode 100644 tests/fuzzers/difficulty/debug/main.go create mode 100644 tests/fuzzers/difficulty/difficulty-fuzz.go diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index bdc02098af..8e401af7ca 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -485,6 +485,11 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int { return diff } +// Exported for fuzzing +var FrontierDifficultyCalulator = calcDifficultyFrontier +var HomesteadDifficultyCalulator = calcDifficultyHomestead +var DynamicDifficultyCalculator = makeDifficultyCalculator + // VerifySeal implements consensus.Engine, checking whether the given block satisfies // the PoW difficulty requirements. func (ethash *Ethash) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error { diff --git a/consensus/ethash/consensus_test.go b/consensus/ethash/consensus_test.go index 675737d9e1..6f6dc79fd8 100644 --- a/consensus/ethash/consensus_test.go +++ b/consensus/ethash/consensus_test.go @@ -17,12 +17,15 @@ package ethash import ( + "encoding/binary" "encoding/json" "math/big" + "math/rand" "os" "path/filepath" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" @@ -84,3 +87,102 @@ func TestCalcDifficulty(t *testing.T) { } } } + +func randSlice(min, max uint32) []byte { + var b = make([]byte, 4) + rand.Read(b) + a := binary.LittleEndian.Uint32(b) + size := min + a%(max-min) + out := make([]byte, size) + rand.Read(out) + return out +} + +func TestDifficultyCalculators(t *testing.T) { + rand.Seed(2) + for i := 0; i < 5000; i++ { + // 1 to 300 seconds diff + var timeDelta = uint64(1 + rand.Uint32()%3000) + diffBig := big.NewInt(0).SetBytes(randSlice(2, 10)) + if diffBig.Cmp(params.MinimumDifficulty) < 0 { + diffBig.Set(params.MinimumDifficulty) + } + //rand.Read(difficulty) + header := &types.Header{ + Difficulty: diffBig, + Number: new(big.Int).SetUint64(rand.Uint64() % 50_000_000), + Time: rand.Uint64() - timeDelta, + } + if rand.Uint32()&1 == 0 { + header.UncleHash = types.EmptyUncleHash + } + bombDelay := new(big.Int).SetUint64(rand.Uint64() % 50_000_000) + for i, pair := range []struct { + bigFn func(time uint64, parent *types.Header) *big.Int + u256Fn func(time uint64, parent *types.Header) *big.Int + }{ + {FrontierDifficultyCalulator, CalcDifficultyFrontierU256}, + {HomesteadDifficultyCalulator, CalcDifficultyHomesteadU256}, + {DynamicDifficultyCalculator(bombDelay), MakeDifficultyCalculatorU256(bombDelay)}, + } { + time := header.Time + timeDelta + want := pair.bigFn(time, header) + have := pair.u256Fn(time, header) + if want.BitLen() > 256 { + continue + } + if want.Cmp(have) != 0 { + t.Fatalf("pair %d: want %x have %x\nparent.Number: %x\np.Time: %x\nc.Time: %x\nBombdelay: %v\n", i, want, have, + header.Number, header.Time, time, bombDelay) + } + } + } +} + +func BenchmarkDifficultyCalculator(b *testing.B) { + x1 := makeDifficultyCalculator(big.NewInt(1000000)) + x2 := MakeDifficultyCalculatorU256(big.NewInt(1000000)) + h := &types.Header{ + ParentHash: common.Hash{}, + UncleHash: types.EmptyUncleHash, + Difficulty: big.NewInt(0xffffff), + Number: big.NewInt(500000), + Time: 1000000, + } + b.Run("big-frontier", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + calcDifficultyFrontier(1000014, h) + } + }) + b.Run("u256-frontier", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + CalcDifficultyFrontierU256(1000014, h) + } + }) + b.Run("big-homestead", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + calcDifficultyHomestead(1000014, h) + } + }) + b.Run("u256-homestead", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + CalcDifficultyHomesteadU256(1000014, h) + } + }) + b.Run("big-generic", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + x1(1000014, h) + } + }) + b.Run("u256-generic", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + x2(1000014, h) + } + }) +} diff --git a/consensus/ethash/difficulty.go b/consensus/ethash/difficulty.go new file mode 100644 index 0000000000..59c4ac7419 --- /dev/null +++ b/consensus/ethash/difficulty.go @@ -0,0 +1,193 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethash + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" +) + +const ( + // frontierDurationLimit is for Frontier: + // The decision boundary on the blocktime duration used to determine + // whether difficulty should go up or down. + frontierDurationLimit = 13 + // minimumDifficulty The minimum that the difficulty may ever be. + minimumDifficulty = 131072 + // expDiffPeriod is the exponential difficulty period + expDiffPeriodUint = 100000 + // difficultyBoundDivisorBitShift is the bound divisor of the difficulty (2048), + // This constant is the right-shifts to use for the division. + difficultyBoundDivisor = 11 +) + +// CalcDifficultyFrontierU256 is the difficulty adjustment algorithm. It returns the +// difficulty that a new block should have when created at time given the parent +// block's time and difficulty. The calculation uses the Frontier rules. +func CalcDifficultyFrontierU256(time uint64, parent *types.Header) *big.Int { + /* + Algorithm + block_diff = pdiff + pdiff / 2048 * (1 if time - ptime < 13 else -1) + int(2^((num // 100000) - 2)) + + Where: + - pdiff = parent.difficulty + - ptime = parent.time + - time = block.timestamp + - num = block.number + */ + + pDiff := uint256.NewInt() + pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff + adjust := pDiff.Clone() + adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048 + + if time-parent.Time < frontierDurationLimit { + pDiff.Add(pDiff, adjust) + } else { + pDiff.Sub(pDiff, adjust) + } + if pDiff.LtUint64(minimumDifficulty) { + pDiff.SetUint64(minimumDifficulty) + } + // 'pdiff' now contains: + // pdiff + pdiff / 2048 * (1 if time - ptime < 13 else -1) + + if periodCount := (parent.Number.Uint64() + 1) / expDiffPeriodUint; periodCount > 1 { + // diff = diff + 2^(periodCount - 2) + expDiff := adjust.SetOne() + expDiff.Lsh(expDiff, uint(periodCount-2)) // expdiff: 2 ^ (periodCount -2) + pDiff.Add(pDiff, expDiff) + } + return pDiff.ToBig() +} + +// CalcDifficultyHomesteadU256 is the difficulty adjustment algorithm. It returns +// the difficulty that a new block should have when created at time given the +// parent block's time and difficulty. The calculation uses the Homestead rules. +func CalcDifficultyHomesteadU256(time uint64, parent *types.Header) *big.Int { + /* + https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.md + Algorithm: + block_diff = pdiff + pdiff / 2048 * max(1 - (time - ptime) / 10, -99) + 2 ^ int((num / 100000) - 2)) + + Our modification, to use unsigned ints: + block_diff = pdiff - pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + 2 ^ int((num / 100000) - 2)) + + Where: + - pdiff = parent.difficulty + - ptime = parent.time + - time = block.timestamp + - num = block.number + */ + + pDiff := uint256.NewInt() + pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff + adjust := pDiff.Clone() + adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048 + + x := (time - parent.Time) / 10 // (time - ptime) / 10) + var neg = true + if x == 0 { + x = 1 + neg = false + } else if x >= 100 { + x = 99 + } else { + x = x - 1 + } + z := new(uint256.Int).SetUint64(x) + adjust.Mul(adjust, z) // adjust: (pdiff / 2048) * max((time - ptime) / 10 - 1, 99) + if neg { + pDiff.Sub(pDiff, adjust) // pdiff - pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + } else { + pDiff.Add(pDiff, adjust) // pdiff + pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + } + if pDiff.LtUint64(minimumDifficulty) { + pDiff.SetUint64(minimumDifficulty) + } + // for the exponential factor, a.k.a "the bomb" + // diff = diff + 2^(periodCount - 2) + if periodCount := (1 + parent.Number.Uint64()) / expDiffPeriodUint; periodCount > 1 { + expFactor := adjust.Lsh(adjust.SetOne(), uint(periodCount-2)) + pDiff.Add(pDiff, expFactor) + } + return pDiff.ToBig() +} + +// MakeDifficultyCalculatorU256 creates a difficultyCalculator with the given bomb-delay. +// the difficulty is calculated with Byzantium rules, which differs from Homestead in +// how uncles affect the calculation +func MakeDifficultyCalculatorU256(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int { + // Note, the calculations below looks at the parent number, which is 1 below + // the block number. Thus we remove one from the delay given + bombDelayFromParent := bombDelay.Uint64() - 1 + return func(time uint64, parent *types.Header) *big.Int { + /* + https://github.com/ethereum/EIPs/issues/100 + pDiff = parent.difficulty + BLOCK_DIFF_FACTOR = 9 + a = pDiff + (pDiff // BLOCK_DIFF_FACTOR) * adj_factor + b = min(parent.difficulty, MIN_DIFF) + child_diff = max(a,b ) + */ + x := (time - parent.Time) / 9 // (block_timestamp - parent_timestamp) // 9 + c := uint64(1) // if parent.unclehash == emptyUncleHashHash + if parent.UncleHash != types.EmptyUncleHash { + c = 2 + } + xNeg := x >= c + if xNeg { + // x is now _negative_ adjustment factor + x = x - c // - ( (t-p)/p -( 2 or 1) ) + } else { + x = c - x // (2 or 1) - (t-p)/9 + } + if x > 99 { + x = 99 // max(x, 99) + } + // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) + y := new(uint256.Int) + y.SetFromBig(parent.Difficulty) // y: p_diff + pDiff := y.Clone() // pdiff: p_diff + z := new(uint256.Int).SetUint64(x) //z : +-adj_factor (either pos or negative) + y.Rsh(y, difficultyBoundDivisor) // y: p__diff / 2048 + z.Mul(y, z) // z: (p_diff / 2048 ) * (+- adj_factor) + + if xNeg { + y.Sub(pDiff, z) // y: parent_diff + parent_diff/2048 * adjustment_factor + } else { + y.Add(pDiff, z) // y: parent_diff + parent_diff/2048 * adjustment_factor + } + // minimum difficulty can ever be (before exponential factor) + if y.LtUint64(minimumDifficulty) { + y.SetUint64(minimumDifficulty) + } + // calculate a fake block number for the ice-age delay + // Specification: https://eips.ethereum.org/EIPS/eip-1234 + var pNum = parent.Number.Uint64() + if pNum >= bombDelayFromParent { + if fakeBlockNumber := pNum - bombDelayFromParent; fakeBlockNumber >= 2*expDiffPeriodUint { + z.SetOne() + z.Lsh(z, uint(fakeBlockNumber/expDiffPeriodUint-2)) + y.Add(z, y) + } + } + return y.ToBig() + } +} diff --git a/oss-fuzz.sh b/oss-fuzz.sh index e0a293a6d6..e060ea88e1 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -57,6 +57,7 @@ compile_fuzzer tests/fuzzers/txfetcher Fuzz fuzzTxfetcher compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie +compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul diff --git a/tests/fuzzers/difficulty/debug/main.go b/tests/fuzzers/difficulty/debug/main.go new file mode 100644 index 0000000000..23516b3a0d --- /dev/null +++ b/tests/fuzzers/difficulty/debug/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/tests/fuzzers/difficulty" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: debug ") + os.Exit(1) + } + crasher := os.Args[1] + data, err := ioutil.ReadFile(crasher) + if err != nil { + fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) + os.Exit(1) + } + difficulty.Fuzz(data) +} diff --git a/tests/fuzzers/difficulty/difficulty-fuzz.go b/tests/fuzzers/difficulty/difficulty-fuzz.go new file mode 100644 index 0000000000..e4c5dcf57c --- /dev/null +++ b/tests/fuzzers/difficulty/difficulty-fuzz.go @@ -0,0 +1,145 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package difficulty + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/types" +) + +type fuzzer struct { + input io.Reader + exhausted bool + debugging bool +} + +func (f *fuzzer) read(size int) []byte { + out := make([]byte, size) + if _, err := f.input.Read(out); err != nil { + f.exhausted = true + } + return out +} + +func (f *fuzzer) readSlice(min, max int) []byte { + var a uint16 + binary.Read(f.input, binary.LittleEndian, &a) + size := min + int(a)%(max-min) + out := make([]byte, size) + if _, err := f.input.Read(out); err != nil { + f.exhausted = true + } + return out +} + +func (f *fuzzer) readUint64(min, max uint64) uint64 { + if min == max { + return min + } + var a uint64 + if err := binary.Read(f.input, binary.LittleEndian, &a); err != nil { + f.exhausted = true + } + a = min + a%(max-min) + return a +} +func (f *fuzzer) readBool() bool { + return f.read(1)[0]&0x1 == 0 +} + +// The function must return +// 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// -1 if the input must not be added to corpus even if gives new coverage; and +// 0 otherwise +// other values are reserved for future use. +func Fuzz(data []byte) int { + f := fuzzer{ + input: bytes.NewReader(data), + exhausted: false, + } + return f.fuzz() +} + +var minDifficulty = big.NewInt(0x2000) + +type calculator func(time uint64, parent *types.Header) *big.Int + +func (f *fuzzer) fuzz() int { + // A parent header + header := &types.Header{} + if f.readBool() { + header.UncleHash = types.EmptyUncleHash + } + // Difficulty can range between 0x2000 (2 bytes) and up to 32 bytes + { + diff := new(big.Int).SetBytes(f.readSlice(2, 32)) + if diff.Cmp(minDifficulty) < 0 { + diff.Set(minDifficulty) + } + header.Difficulty = diff + } + // Number can range between 0 and up to 32 bytes (but not so that the child exceeds it) + { + // However, if we use astronomic numbers, then the bomb exp karatsuba calculation + // in the legacy methods) + // times out, so we limit it to fit within reasonable bounds + number := new(big.Int).SetBytes(f.readSlice(0, 4)) // 4 bytes: 32 bits: block num max 4 billion + header.Number = number + } + // Both parent and child time must fit within uint64 + var time uint64 + { + childTime := f.readUint64(1, 0xFFFFFFFFFFFFFFFF) + //fmt.Printf("childTime: %x\n",childTime) + delta := f.readUint64(1, childTime) + //fmt.Printf("delta: %v\n", delta) + pTime := childTime - delta + header.Time = pTime + time = childTime + } + // Bomb delay will never exceed uint64 + bombDelay := new(big.Int).SetUint64(f.readUint64(1, 0xFFFFFFFFFFFFFFFe)) + + if f.exhausted { + return 0 + } + + for i, pair := range []struct { + bigFn calculator + u256Fn calculator + }{ + {ethash.FrontierDifficultyCalulator, ethash.CalcDifficultyFrontierU256}, + {ethash.HomesteadDifficultyCalulator, ethash.CalcDifficultyHomesteadU256}, + {ethash.DynamicDifficultyCalculator(bombDelay), ethash.MakeDifficultyCalculatorU256(bombDelay)}, + } { + want := pair.bigFn(time, header) + have := pair.u256Fn(time, header) + if want.Cmp(have) != 0 { + panic(fmt.Sprintf("pair %d: want %x have %x\nparent.Number: %x\np.Time: %x\nc.Time: %x\nBombdelay: %v\n", i, want, have, + header.Number, header.Time, time, bombDelay)) + } + } + return 1 +} From c49aae987040b3c0b846c5acb006fdba1eae282b Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 11 Dec 2020 16:49:44 +0200 Subject: [PATCH 008/709] consensus: refactor FinalizeAndAssemble to use Finalize (#21993) --- consensus/clique/clique.go | 5 ++--- consensus/ethash/consensus.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index c05f84cc2e..6c667804d8 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -561,9 +561,8 @@ func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Heade // FinalizeAndAssemble implements consensus.Engine, ensuring no uncles are set, // nor block rewards given, and returns the final block. func (c *Clique) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) { - // No block rewards in PoA, so the state remains as is and uncles are dropped - header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) - header.UncleHash = types.CalcUncleHash(nil) + // Finalize block + c.Finalize(chain, header, state, txs, uncles) // Assemble and return the final block for sealing return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)), nil diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 8e401af7ca..ae0905ee3a 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -584,9 +584,8 @@ func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, accumulating the block and // uncle rewards, setting the final state and assembling the block. func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) { - // Accumulate any block and uncle rewards and commit the final state root - accumulateRewards(chain.Config(), state, header, uncles) - header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) + // Finalize block + ethash.Finalize(chain, header, state, txs, uncles) // Header seems complete, assemble into a block and return return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie)), nil From 4d48980e74e7925d40fb89683eac0b43f3540d77 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 11 Dec 2020 15:56:00 +0100 Subject: [PATCH 009/709] core, eth, les: implement unclean-shutdown marker (#21893) This PR implements unclean shutdown marker. Every time geth boots, it adds a timestamp to a list of timestamps in the database. This list is capped at 10. At a clean shutdown, the timestamp is removed again. Thus, when geth exits unclean, the marker remains, and at boot up we show the most recent unclean shutdowns to the user, which makes it easier to diagnose root-causes to certain problems. Co-authored-by: Nagy Salem --- core/rawdb/accessors_metadata.go | 63 +++++++++++++++++++++++++++++++- core/rawdb/database.go | 2 +- core/rawdb/schema.go | 6 ++- eth/backend.go | 15 ++++++++ les/client.go | 14 +++++++ 5 files changed, 95 insertions(+), 5 deletions(-) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 14a302a127..079e335fa6 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -18,6 +18,7 @@ package rawdb import ( "encoding/json" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" @@ -30,7 +31,7 @@ import ( func ReadDatabaseVersion(db ethdb.KeyValueReader) *uint64 { var version uint64 - enc, _ := db.Get(databaseVerisionKey) + enc, _ := db.Get(databaseVersionKey) if len(enc) == 0 { return nil } @@ -47,7 +48,7 @@ func WriteDatabaseVersion(db ethdb.KeyValueWriter, version uint64) { if err != nil { log.Crit("Failed to encode database version", "err", err) } - if err = db.Put(databaseVerisionKey, enc); err != nil { + if err = db.Put(databaseVersionKey, enc); err != nil { log.Crit("Failed to store the database version", "err", err) } } @@ -79,3 +80,61 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha log.Crit("Failed to store chain config", "err", err) } } + +// crashList is a list of unclean-shutdown-markers, for rlp-encoding to the +// database +type crashList struct { + Discarded uint64 // how many ucs have we deleted + Recent []uint64 // unix timestamps of 10 latest unclean shutdowns +} + +const crashesToKeep = 10 + +// PushUncleanShutdownMarker appends a new unclean shutdown marker and returns +// the previous data +// - a list of timestamps +// - a count of how many old unclean-shutdowns have been discarded +func PushUncleanShutdownMarker(db ethdb.KeyValueStore) ([]uint64, uint64, error) { + var uncleanShutdowns crashList + // Read old data + if data, err := db.Get(uncleanShutdownKey); err != nil { + log.Warn("Error reading unclean shutdown markers", "error", err) + } else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { + return nil, 0, err + } + var discarded = uncleanShutdowns.Discarded + var previous = make([]uint64, len(uncleanShutdowns.Recent)) + copy(previous, uncleanShutdowns.Recent) + // Add a new (but cap it) + uncleanShutdowns.Recent = append(uncleanShutdowns.Recent, uint64(time.Now().Unix())) + if count := len(uncleanShutdowns.Recent); count > crashesToKeep+1 { + numDel := count - (crashesToKeep + 1) + uncleanShutdowns.Recent = uncleanShutdowns.Recent[numDel:] + uncleanShutdowns.Discarded += uint64(numDel) + } + // And save it again + data, _ := rlp.EncodeToBytes(uncleanShutdowns) + if err := db.Put(uncleanShutdownKey, data); err != nil { + log.Warn("Failed to write unclean-shutdown marker", "err", err) + return nil, 0, err + } + return previous, discarded, nil +} + +// PopUncleanShutdownMarker removes the last unclean shutdown marker +func PopUncleanShutdownMarker(db ethdb.KeyValueStore) { + var uncleanShutdowns crashList + // Read old data + if data, err := db.Get(uncleanShutdownKey); err != nil { + log.Warn("Error reading unclean shutdown markers", "error", err) + } else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { + log.Error("Error decoding unclean shutdown markers", "error", err) // Should mos def _not_ happen + } + if l := len(uncleanShutdowns.Recent); l > 0 { + uncleanShutdowns.Recent = uncleanShutdowns.Recent[:l-1] + } + data, _ := rlp.EncodeToBytes(uncleanShutdowns) + if err := db.Put(uncleanShutdownKey, data); err != nil { + log.Warn("Failed to clear unclean-shutdown marker", "err", err) + } +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index b1ac3e9587..b01a31ebcd 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -355,7 +355,7 @@ func InspectDatabase(db ethdb.Database) error { bloomTrieNodes.Add(size) default: var accounted bool - for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { + for _, meta := range [][]byte{databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey, uncleanShutdownKey} { if bytes.Equal(key, meta) { metadata.Add(size) accounted = true diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index dbc5025d5d..cff27b4bb0 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -27,8 +27,8 @@ import ( // The fields below define the low level database schema prefixing. var ( - // databaseVerisionKey tracks the current database version. - databaseVerisionKey = []byte("DatabaseVersion") + // databaseVersionKey tracks the current database version. + databaseVersionKey = []byte("DatabaseVersion") // headHeaderKey tracks the latest known header's hash. headHeaderKey = []byte("LastHeader") @@ -81,6 +81,8 @@ var ( preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage configPrefix = []byte("ethereum-config-") // config prefix for the db + uncleanShutdownKey = []byte("unclean-shutdown") // config prefix for the db + // Chain index prefixes (use `i` + single byte to avoid mixing data types). BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress diff --git a/eth/backend.go b/eth/backend.go index 03b0b319b7..bb4275b92c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -24,6 +24,7 @@ import ( "runtime" "sync" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -220,6 +221,19 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { stack.RegisterAPIs(eth.APIs()) stack.RegisterProtocols(eth.Protocols()) stack.RegisterLifecycle(eth) + // Check for unclean shutdown + if uncleanShutdowns, discards, err := rawdb.PushUncleanShutdownMarker(chainDb); err != nil { + log.Error("Could not update unclean-shutdown-marker list", "error", err) + } else { + if discards > 0 { + log.Warn("Old unclean shutdowns found", "count", discards) + } + for _, tstamp := range uncleanShutdowns { + t := time.Unix(int64(tstamp), 0) + log.Warn("Unclean shutdown detected", "booted", t, + "age", common.PrettyAge(t)) + } + } return eth, nil } @@ -543,6 +557,7 @@ func (s *Ethereum) Stop() error { s.miner.Stop() s.blockchain.Stop() s.engine.Close() + rawdb.PopUncleanShutdownMarker(s.chainDb) s.chainDb.Close() s.eventMux.Stop() return nil diff --git a/les/client.go b/les/client.go index 37250d076f..47997a098b 100644 --- a/les/client.go +++ b/les/client.go @@ -178,6 +178,19 @@ func New(stack *node.Node, config *eth.Config) (*LightEthereum, error) { stack.RegisterProtocols(leth.Protocols()) stack.RegisterLifecycle(leth) + // Check for unclean shutdown + if uncleanShutdowns, discards, err := rawdb.PushUncleanShutdownMarker(chainDb); err != nil { + log.Error("Could not update unclean-shutdown-marker list", "error", err) + } else { + if discards > 0 { + log.Warn("Old unclean shutdowns found", "count", discards) + } + for _, tstamp := range uncleanShutdowns { + t := time.Unix(int64(tstamp), 0) + log.Warn("Unclean shutdown detected", "booted", t, + "age", common.PrettyAge(t)) + } + } return leth, nil } @@ -313,6 +326,7 @@ func (s *LightEthereum) Stop() error { s.engine.Close() s.pruner.close() s.eventMux.Stop() + rawdb.PopUncleanShutdownMarker(s.chainDb) s.chainDb.Close() s.wg.Wait() log.Info("Light ethereum stopped") From 38c1d592b7121f26dc661c1bc2bf0e32eba9d888 Mon Sep 17 00:00:00 2001 From: Connor Stein Date: Sat, 12 Dec 2020 04:16:34 -0500 Subject: [PATCH 010/709] abi/bind: fix error-handling in generated wrappers for functions returning structs (#22005) Fixes the template used when generating code, which in some scenarios would lead to panic instead of returning an error. --- accounts/abi/bind/bind_test.go | 39 ++++++++++++++++++++++++++++++++++ accounts/abi/bind/template.go | 3 +++ 2 files changed, 42 insertions(+) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 1a8a17e45e..4a504516bb 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -569,6 +569,45 @@ var bindTests = []struct { nil, nil, }, + { + `NonExistentStruct`, + ` + contract NonExistentStruct { + function Struct() public view returns(uint256 a, uint256 b) { + return (10, 10); + } + } + `, + []string{`6080604052348015600f57600080fd5b5060888061001e6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063d5f6622514602d575b600080fd5b6033604c565b6040805192835260208301919091528051918290030190f35b600a809156fea264697066735822beefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeef64736f6c6343decafe0033`}, + []string{`[{"inputs":[],"name":"Struct","outputs":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"stateMutability":"pure","type":"function"}]`}, + ` + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + `, + ` + // Create a simulator and wrap a non-deployed contract + + sim := backends.NewSimulatedBackend(core.GenesisAlloc{}, uint64(10000000000)) + defer sim.Close() + + nonexistent, err := NewNonExistentStruct(common.Address{}, sim) + if err != nil { + t.Fatalf("Failed to access non-existent contract: %v", err) + } + // Ensure that contract calls fail with the appropriate error + if res, err := nonexistent.Struct(nil); err == nil { + t.Fatalf("Call succeeded on non-existent contract: %v", res) + } else if (err != bind.ErrNoCode) { + t.Fatalf("Error mismatch: have %v, want %v", err, bind.ErrNoCode) + } + `, + nil, + nil, + nil, + nil, + }, // Tests that gas estimation works for contracts with weird gas mechanics too. { `FunkyGasPattern`, diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index 8dac11f79f..351eabd258 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -304,6 +304,9 @@ var ( err := _{{$contract.Type}}.contract.Call(opts, &out, "{{.Original.Name}}" {{range .Normalized.Inputs}}, {{.Name}}{{end}}) {{if .Structured}} outstruct := new(struct{ {{range .Normalized.Outputs}} {{.Name}} {{bindtype .Type $structs}}; {{end}} }) + if err != nil { + return *outstruct, err + } {{range $i, $t := .Normalized.Outputs}} outstruct.{{.Name}} = out[{{$i}}].({{bindtype .Type $structs}}){{end}} From 00d10e610f9fef56b5ee9c27f7fe7c842eba2e9b Mon Sep 17 00:00:00 2001 From: Shiming Date: Sun, 13 Dec 2020 00:36:32 +0800 Subject: [PATCH 011/709] cmd/abigen: clarify abigen alias flag usage (#21875) * doc: clarify abigen alias flag usage update the `abigen --alias` flag help info, give an example to make it more clear related issue: https://github.com/ethereum/go-ethereum/issues/21846 * Update cmd/abigen/main.go Co-authored-by: ligi Co-authored-by: Martin Holst Swende Co-authored-by: ligi --- cmd/abigen/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index a74b0396d4..7b3b35e4e5 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -96,7 +96,7 @@ var ( } aliasFlag = cli.StringFlag{ Name: "alias", - Usage: "Comma separated aliases for function and event renaming, e.g. foo=bar", + Usage: "Comma separated aliases for function and event renaming, e.g. original1=alias1, original2=alias2", } ) From 017831dd5b33a68076aed7c9ff05e62b0dcb5f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 14 Dec 2020 11:27:15 +0200 Subject: [PATCH 012/709] core, eth: split eth package, implement snap protocol (#21482) This commit splits the eth package, separating the handling of eth and snap protocols. It also includes the capability to run snap sync (https://github.com/ethereum/devp2p/blob/master/caps/snap.md) , but does not enable it by default. Co-authored-by: Marius van der Wijden Co-authored-by: Martin Holst Swende --- cmd/geth/misccmd.go | 2 - cmd/utils/flags.go | 29 +- core/blockchain.go | 8 +- core/blockchain_snapshot_test.go | 2 +- core/forkid/forkid.go | 9 + core/rawdb/accessors_snapshot.go | 21 + core/rawdb/schema.go | 3 + core/state/snapshot/generate.go | 2 +- core/state/statedb.go | 24 +- eth/api_backend.go | 6 +- eth/api_test.go | 6 + eth/backend.go | 60 +- eth/config.go | 3 +- eth/discovery.go | 9 +- eth/downloader/downloader.go | 108 +- eth/downloader/downloader_test.go | 114 +- eth/downloader/modes.go | 9 +- eth/downloader/peer.go | 14 +- eth/downloader/queue.go | 49 +- eth/downloader/statesync.go | 31 +- eth/gen_config.go | 10 +- eth/handler.go | 867 ++---- eth/handler_eth.go | 218 ++ eth/handler_eth_test.go | 740 +++++ eth/handler_snap.go | 48 + eth/handler_test.go | 736 +---- eth/helper_test.go | 231 -- eth/peer.go | 804 +----- eth/peerset.go | 301 ++ eth/protocol.go | 221 -- eth/protocol_test.go | 459 --- eth/protocols/eth/broadcast.go | 195 ++ eth/protocols/eth/discovery.go | 65 + eth/protocols/eth/handler.go | 512 ++++ eth/protocols/eth/handler_test.go | 519 ++++ eth/protocols/eth/handshake.go | 107 + eth/protocols/eth/handshake_test.go | 91 + eth/protocols/eth/peer.go | 429 +++ eth/protocols/eth/peer_test.go | 61 + eth/protocols/eth/protocol.go | 279 ++ eth/protocols/eth/protocol_test.go | 68 + eth/protocols/snap/discovery.go | 32 + eth/protocols/snap/handler.go | 490 ++++ eth/protocols/snap/peer.go | 111 + eth/protocols/snap/protocol.go | 218 ++ eth/protocols/snap/sync.go | 2481 +++++++++++++++++ eth/sync.go | 129 +- eth/sync_test.go | 50 +- ethstats/ethstats.go | 14 +- graphql/graphql.go | 4 - graphql/schema.go | 2 - internal/ethapi/api.go | 17 +- internal/ethapi/backend.go | 1 - les/client.go | 2 +- les/enr_entry.go | 4 +- les/handler_test.go | 2 +- les/peer.go | 13 +- les/server_handler.go | 2 +- tests/block_test_util.go | 2 +- ...1c14030f26872e57bf1481084f151d71eed8161c-1 | Bin 0 -> 16005 bytes ...27e54254422543060a13ea8a4bc913d768e4adb6-2 | Bin 0 -> 15965 bytes ...6bfc2cbe2d7a43361e240118439785445a0fdfb7-5 | Bin 0 -> 15976 bytes ...a67e63bc0c0004bd009944a6061297cb7d4ac238-1 | Bin 0 -> 14980 bytes ...ae892bbae0a843950bc8316496e595b1a194c009-4 | Bin 0 -> 15977 bytes ...ee05d0d813f6261b3dba16506f9ea03d9c5e993d-2 | Bin 0 -> 16000 bytes ...f50a6d57a46d30184aa294af5b252ab9701af7c9-2 | Bin 0 -> 1748 bytes tests/fuzzers/rangeproof/corpus/random.dat | Bin 0 -> 16000 bytes tests/fuzzers/rangeproof/debug/main.go | 41 + tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 218 ++ trie/notary.go | 57 + trie/proof.go | 104 +- trie/proof_test.go | 104 +- trie/sync_bloom.go | 6 +- trie/trie.go | 39 +- 74 files changed, 8224 insertions(+), 3389 deletions(-) create mode 100644 eth/handler_eth.go create mode 100644 eth/handler_eth_test.go create mode 100644 eth/handler_snap.go delete mode 100644 eth/helper_test.go create mode 100644 eth/peerset.go delete mode 100644 eth/protocol.go delete mode 100644 eth/protocol_test.go create mode 100644 eth/protocols/eth/broadcast.go create mode 100644 eth/protocols/eth/discovery.go create mode 100644 eth/protocols/eth/handler.go create mode 100644 eth/protocols/eth/handler_test.go create mode 100644 eth/protocols/eth/handshake.go create mode 100644 eth/protocols/eth/handshake_test.go create mode 100644 eth/protocols/eth/peer.go create mode 100644 eth/protocols/eth/peer_test.go create mode 100644 eth/protocols/eth/protocol.go create mode 100644 eth/protocols/eth/protocol_test.go create mode 100644 eth/protocols/snap/discovery.go create mode 100644 eth/protocols/snap/handler.go create mode 100644 eth/protocols/snap/peer.go create mode 100644 eth/protocols/snap/protocol.go create mode 100644 eth/protocols/snap/sync.go create mode 100644 tests/fuzzers/rangeproof/corpus/1c14030f26872e57bf1481084f151d71eed8161c-1 create mode 100644 tests/fuzzers/rangeproof/corpus/27e54254422543060a13ea8a4bc913d768e4adb6-2 create mode 100644 tests/fuzzers/rangeproof/corpus/6bfc2cbe2d7a43361e240118439785445a0fdfb7-5 create mode 100644 tests/fuzzers/rangeproof/corpus/a67e63bc0c0004bd009944a6061297cb7d4ac238-1 create mode 100644 tests/fuzzers/rangeproof/corpus/ae892bbae0a843950bc8316496e595b1a194c009-4 create mode 100644 tests/fuzzers/rangeproof/corpus/ee05d0d813f6261b3dba16506f9ea03d9c5e993d-2 create mode 100644 tests/fuzzers/rangeproof/corpus/f50a6d57a46d30184aa294af5b252ab9701af7c9-2 create mode 100644 tests/fuzzers/rangeproof/corpus/random.dat create mode 100644 tests/fuzzers/rangeproof/debug/main.go create mode 100644 tests/fuzzers/rangeproof/rangeproof-fuzzer.go create mode 100644 trie/notary.go diff --git a/cmd/geth/misccmd.go b/cmd/geth/misccmd.go index 967df2ada0..b347d31d97 100644 --- a/cmd/geth/misccmd.go +++ b/cmd/geth/misccmd.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/params" "gopkg.in/urfave/cli.v1" ) @@ -143,7 +142,6 @@ func version(ctx *cli.Context) error { fmt.Println("Git Commit Date:", gitDate) } fmt.Println("Architecture:", runtime.GOARCH) - fmt.Println("Protocol Versions:", eth.ProtocolVersions) fmt.Println("Go Version:", runtime.Version()) fmt.Println("Operating System:", runtime.GOOS) fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH")) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 051bdd6308..0b1695d0a5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -187,7 +187,7 @@ var ( defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", - Usage: `Blockchain sync mode ("fast", "full", or "light")`, + Usage: `Blockchain sync mode ("fast", "full", "snap" or "light")`, Value: &defaultSyncMode, } GCModeFlag = cli.StringFlag{ @@ -1555,8 +1555,14 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 } if !ctx.GlobalIsSet(SnapshotFlag.Name) { - cfg.TrieCleanCache += cfg.SnapshotCache - cfg.SnapshotCache = 0 // Disabled + // If snap-sync is requested, this flag is also required + if cfg.SyncMode == downloader.SnapSync { + log.Info("Snap sync requested, enabling --snapshot") + ctx.Set(SnapshotFlag.Name, "true") + } else { + cfg.TrieCleanCache += cfg.SnapshotCache + cfg.SnapshotCache = 0 // Disabled + } } if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) @@ -1585,16 +1591,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name) } if ctx.GlobalIsSet(NoDiscoverFlag.Name) { - cfg.DiscoveryURLs = []string{} + cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{} } else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) { urls := ctx.GlobalString(DNSDiscoveryFlag.Name) if urls == "" { - cfg.DiscoveryURLs = []string{} + cfg.EthDiscoveryURLs = []string{} } else { - cfg.DiscoveryURLs = SplitAndTrim(urls) + cfg.EthDiscoveryURLs = SplitAndTrim(urls) } } - // Override any default configs for hard coded networks. switch { case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name): @@ -1676,16 +1681,20 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if // no URLs are set. func SetDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) { - if cfg.DiscoveryURLs != nil { + if cfg.EthDiscoveryURLs != nil { return // already set through flags/config } - protocol := "all" if cfg.SyncMode == downloader.LightSync { protocol = "les" } if url := params.KnownDNSNetwork(genesis, protocol); url != "" { - cfg.DiscoveryURLs = []string{url} + cfg.EthDiscoveryURLs = []string{url} + } + if cfg.SyncMode == downloader.SnapSync { + if url := params.KnownDNSNetwork(genesis, "snap"); url != "" { + cfg.SnapDiscoveryURLs = []string{url} + } } } diff --git a/core/blockchain.go b/core/blockchain.go index bc1db49f37..d9505dcf69 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -659,12 +659,8 @@ func (bc *BlockChain) CurrentBlock() *types.Block { return bc.currentBlock.Load().(*types.Block) } -// Snapshot returns the blockchain snapshot tree. This method is mainly used for -// testing, to make it possible to verify the snapshot after execution. -// -// Warning: There are no guarantees about the safety of using the returned 'snap' if the -// blockchain is simultaneously importing blocks, so take care. -func (bc *BlockChain) Snapshot() *snapshot.Tree { +// Snapshots returns the blockchain snapshot tree. +func (bc *BlockChain) Snapshots() *snapshot.Tree { return bc.snaps } diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index e8d3b2470a..f35dae1678 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -751,7 +751,7 @@ func testSnapshot(t *testing.T, tt *snapshotTest) { t.Fatalf("Failed to recreate chain: %v", err) } chain.InsertChain(newBlocks) - chain.Snapshot().Cap(newBlocks[len(newBlocks)-1].Root(), 0) + chain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) // Simulate the blockchain crash // Don't call chain.Stop here, so that no snapshot diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go index c432858617..1bf3406828 100644 --- a/core/forkid/forkid.go +++ b/core/forkid/forkid.go @@ -84,6 +84,15 @@ func NewID(config *params.ChainConfig, genesis common.Hash, head uint64) ID { return ID{Hash: checksumToBytes(hash), Next: next} } +// NewIDWithChain calculates the Ethereum fork ID from an existing chain instance. +func NewIDWithChain(chain Blockchain) ID { + return NewID( + chain.Config(), + chain.Genesis().Hash(), + chain.CurrentHeader().Number.Uint64(), + ) +} + // NewFilter creates a filter that returns if a fork ID should be rejected or not // based on the local chain's status. func NewFilter(chain Blockchain) Filter { diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go index 5bd48ad5fa..0a91d9353b 100644 --- a/core/rawdb/accessors_snapshot.go +++ b/core/rawdb/accessors_snapshot.go @@ -175,3 +175,24 @@ func DeleteSnapshotRecoveryNumber(db ethdb.KeyValueWriter) { log.Crit("Failed to remove snapshot recovery number", "err", err) } } + +// ReadSanpshotSyncStatus retrieves the serialized sync status saved at shutdown. +func ReadSanpshotSyncStatus(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(snapshotSyncStatusKey) + return data +} + +// WriteSnapshotSyncStatus stores the serialized sync status to save at shutdown. +func WriteSnapshotSyncStatus(db ethdb.KeyValueWriter, status []byte) { + if err := db.Put(snapshotSyncStatusKey, status); err != nil { + log.Crit("Failed to store snapshot sync status", "err", err) + } +} + +// DeleteSnapshotSyncStatus deletes the serialized sync status saved at the last +// shutdown +func DeleteSnapshotSyncStatus(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotSyncStatusKey); err != nil { + log.Crit("Failed to remove snapshot sync status", "err", err) + } +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index cff27b4bb0..2aabfd3baa 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -57,6 +57,9 @@ var ( // snapshotRecoveryKey tracks the snapshot recovery marker across restarts. snapshotRecoveryKey = []byte("SnapshotRecovery") + // snapshotSyncStatusKey tracks the snapshot sync status across restarts. + snapshotSyncStatusKey = []byte("SnapshotSyncStatus") + // txIndexTailKey tracks the oldest block whose transactions have been indexed. txIndexTailKey = []byte("TransactionIndexTail") diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 92c7640c40..4a2fa78d3a 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -241,7 +241,7 @@ func (dl *diskLayer) generate(stats *generatorStats) { if acc.Root != emptyRoot { storeTrie, err := trie.NewSecure(acc.Root, dl.triedb) if err != nil { - log.Error("Generator failed to access storage trie", "accroot", dl.root, "acchash", common.BytesToHash(accIt.Key), "stroot", acc.Root, "err", err) + log.Error("Generator failed to access storage trie", "root", dl.root, "account", accountHash, "stroot", acc.Root, "err", err) abort := <-dl.genAbort abort <- stats return diff --git a/core/state/statedb.go b/core/state/statedb.go index ed9a82379f..a9d1de2e06 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -314,14 +314,19 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { return common.Hash{} } -// GetProof returns the MerkleProof for a given Account -func (s *StateDB) GetProof(a common.Address) ([][]byte, error) { +// GetProof returns the Merkle proof for a given account. +func (s *StateDB) GetProof(addr common.Address) ([][]byte, error) { + return s.GetProofByHash(crypto.Keccak256Hash(addr.Bytes())) +} + +// GetProofByHash returns the Merkle proof for a given account. +func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, error) { var proof proofList - err := s.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) + err := s.trie.Prove(addrHash[:], 0, &proof) return proof, err } -// GetStorageProof returns the StorageProof for given key +// GetStorageProof returns the Merkle proof for given storage slot. func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { var proof proofList trie := s.StorageTrie(a) @@ -332,6 +337,17 @@ func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, return proof, err } +// GetStorageProofByHash returns the Merkle proof for given storage slot. +func (s *StateDB) GetStorageProofByHash(a common.Address, key common.Hash) ([][]byte, error) { + var proof proofList + trie := s.StorageTrie(a) + if trie == nil { + return proof, errors.New("storage trie for requested address does not exist") + } + err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + return proof, err +} + // GetCommittedState retrieves a value from the given account's committed storage trie. func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) diff --git a/eth/api_backend.go b/eth/api_backend.go index e7f676f178..2f7020475f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -56,7 +56,7 @@ func (b *EthAPIBackend) CurrentBlock() *types.Block { } func (b *EthAPIBackend) SetHead(number uint64) { - b.eth.protocolManager.downloader.Cancel() + b.eth.handler.downloader.Cancel() b.eth.blockchain.SetHead(number) } @@ -272,10 +272,6 @@ func (b *EthAPIBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } -func (b *EthAPIBackend) ProtocolVersion() int { - return b.eth.EthVersion() -} - func (b *EthAPIBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } diff --git a/eth/api_test.go b/eth/api_test.go index 2c9a2e54e8..b44eed40bc 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -57,6 +57,8 @@ func (h resultHash) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h resultHash) Less(i, j int) bool { return bytes.Compare(h[i].Bytes(), h[j].Bytes()) < 0 } func TestAccountRange(t *testing.T) { + t.Parallel() + var ( statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), nil) state, _ = state.New(common.Hash{}, statedb, nil) @@ -126,6 +128,8 @@ func TestAccountRange(t *testing.T) { } func TestEmptyAccountRange(t *testing.T) { + t.Parallel() + var ( statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) state, _ = state.New(common.Hash{}, statedb, nil) @@ -142,6 +146,8 @@ func TestEmptyAccountRange(t *testing.T) { } func TestStorageRangeAt(t *testing.T) { + t.Parallel() + // Create a state where account 0x010000... has a few storage entries. var ( state, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) diff --git a/eth/backend.go b/eth/backend.go index bb4275b92c..987dee6d55 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -40,6 +40,8 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -48,7 +50,6 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" @@ -59,10 +60,11 @@ type Ethereum struct { config *Config // Handlers - txPool *core.TxPool - blockchain *core.BlockChain - protocolManager *ProtocolManager - dialCandidates enode.Iterator + txPool *core.TxPool + blockchain *core.BlockChain + handler *handler + ethDialCandidates enode.Iterator + snapDialCandidates enode.Iterator // DB interfaces chainDb ethdb.Database // Block chain database @@ -145,7 +147,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { if bcVersion != nil { dbVer = fmt.Sprintf("%d", *bcVersion) } - log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId, "dbversion", dbVer) + log.Info("Initialising Ethereum protocol", "network", config.NetworkId, "dbversion", dbVer) if !config.SkipBcVersionCheck { if bcVersion != nil && *bcVersion > core.BlockChainVersion { @@ -196,7 +198,17 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { if checkpoint == nil { checkpoint = params.TrustedCheckpoints[genesisHash] } - if eth.protocolManager, err = NewProtocolManager(chainConfig, checkpoint, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb, cacheLimit, config.Whitelist); err != nil { + if eth.handler, err = newHandler(&handlerConfig{ + Database: chainDb, + Chain: eth.blockchain, + TxPool: eth.txPool, + Network: config.NetworkId, + Sync: config.SyncMode, + BloomCache: uint64(cacheLimit), + EventMux: eth.eventMux, + Checkpoint: checkpoint, + Whitelist: config.Whitelist, + }); err != nil { return nil, err } eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) @@ -209,13 +221,16 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) - eth.dialCandidates, err = eth.setupDiscovery() + eth.ethDialCandidates, err = setupDiscovery(eth.config.EthDiscoveryURLs) + if err != nil { + return nil, err + } + eth.snapDialCandidates, err = setupDiscovery(eth.config.SnapDiscoveryURLs) if err != nil { return nil, err } - // Start the RPC service - eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, eth.NetVersion()) + eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer) // Register the backend on the node stack.RegisterAPIs(eth.APIs()) @@ -310,7 +325,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: downloader.NewPublicDownloaderAPI(s.protocolManager.downloader, s.eventMux), + Service: downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux), Public: true, }, { Namespace: "miner", @@ -473,7 +488,7 @@ func (s *Ethereum) StartMining(threads int) error { } // If mining is started, we can disable the transaction rejection mechanism // introduced to speed sync times. - atomic.StoreUint32(&s.protocolManager.acceptTxs, 1) + atomic.StoreUint32(&s.handler.acceptTxs, 1) go s.miner.Start(eb) } @@ -504,21 +519,17 @@ func (s *Ethereum) EventMux() *event.TypeMux { return s.eventMux } func (s *Ethereum) Engine() consensus.Engine { return s.engine } func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) IsListening() bool { return true } // Always listening -func (s *Ethereum) EthVersion() int { return int(ProtocolVersions[0]) } -func (s *Ethereum) NetVersion() uint64 { return s.networkID } -func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManager.downloader } -func (s *Ethereum) Synced() bool { return atomic.LoadUint32(&s.protocolManager.acceptTxs) == 1 } +func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader } +func (s *Ethereum) Synced() bool { return atomic.LoadUint32(&s.handler.acceptTxs) == 1 } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } // Protocols returns all the currently configured // network protocols to start. func (s *Ethereum) Protocols() []p2p.Protocol { - protos := make([]p2p.Protocol, len(ProtocolVersions)) - for i, vsn := range ProtocolVersions { - protos[i] = s.protocolManager.makeProtocol(vsn) - protos[i].Attributes = []enr.Entry{s.currentEthEntry()} - protos[i].DialCandidates = s.dialCandidates + protos := eth.MakeProtocols((*ethHandler)(s.handler), s.networkID, s.ethDialCandidates) + if s.config.SnapshotCache > 0 { + protos = append(protos, snap.MakeProtocols((*snapHandler)(s.handler), s.snapDialCandidates)...) } return protos } @@ -526,7 +537,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { // Start implements node.Lifecycle, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *Ethereum) Start() error { - s.startEthEntryUpdate(s.p2pServer.LocalNode()) + eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode()) // Start the bloom bits servicing goroutines s.startBloomHandlers(params.BloomBitsBlocks) @@ -540,7 +551,7 @@ func (s *Ethereum) Start() error { maxPeers -= s.config.LightPeers } // Start the networking layer and the light server if requested - s.protocolManager.Start(maxPeers) + s.handler.Start(maxPeers) return nil } @@ -548,7 +559,7 @@ func (s *Ethereum) Start() error { // Ethereum protocol. func (s *Ethereum) Stop() error { // Stop all the peer-related stuff first. - s.protocolManager.Stop() + s.handler.Stop() // Then stop everything else. s.bloomIndexer.Close() @@ -560,5 +571,6 @@ func (s *Ethereum) Stop() error { rawdb.PopUncleanShutdownMarker(s.chainDb) s.chainDb.Close() s.eventMux.Stop() + return nil } diff --git a/eth/config.go b/eth/config.go index 0d90376d94..77d03e9569 100644 --- a/eth/config.go +++ b/eth/config.go @@ -115,7 +115,8 @@ type Config struct { // This can be set to list of enrtree:// URLs which will be queried for // for nodes to connect to. - DiscoveryURLs []string + EthDiscoveryURLs []string + SnapDiscoveryURLs []string NoPruning bool // Whether to disable pruning and flush everything to disk NoPrefetch bool // Whether to disable prefetching and only load state on demand diff --git a/eth/discovery.go b/eth/discovery.go index e7a281d356..855ce3b0e1 100644 --- a/eth/discovery.go +++ b/eth/discovery.go @@ -63,11 +63,12 @@ func (eth *Ethereum) currentEthEntry() *ethEntry { eth.blockchain.CurrentHeader().Number.Uint64())} } -// setupDiscovery creates the node discovery source for the eth protocol. -func (eth *Ethereum) setupDiscovery() (enode.Iterator, error) { - if len(eth.config.DiscoveryURLs) == 0 { +// setupDiscovery creates the node discovery source for the `eth` and `snap` +// protocols. +func setupDiscovery(urls []string) (enode.Iterator, error) { + if len(urls) == 0 { return nil, nil } client := dnsdisc.NewClient(dnsdisc.Config{}) - return client.NewIterator(eth.config.DiscoveryURLs...) + return client.NewIterator(urls...) } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 686c1ace14..3123598437 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -38,7 +39,6 @@ import ( ) var ( - MaxHashFetch = 512 // Amount of hashes to be fetched per retrieval request MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request MaxSkeletonSize = 128 // Number of header fetches to need for a skeleton assembly @@ -89,7 +89,7 @@ var ( errCancelContentProcessing = errors.New("content processing canceled (requested)") errCanceled = errors.New("syncing canceled (requested)") errNoSyncActive = errors.New("no sync active") - errTooOld = errors.New("peer doesn't speak recent enough protocol version (need version >= 63)") + errTooOld = errors.New("peer doesn't speak recent enough protocol version (need version >= 64)") ) type Downloader struct { @@ -131,20 +131,22 @@ type Downloader struct { ancientLimit uint64 // The maximum block number which can be regarded as ancient data. // Channels - headerCh chan dataPack // [eth/62] Channel receiving inbound block headers - bodyCh chan dataPack // [eth/62] Channel receiving inbound block bodies - receiptCh chan dataPack // [eth/63] Channel receiving inbound receipts - bodyWakeCh chan bool // [eth/62] Channel to signal the block body fetcher of new tasks - receiptWakeCh chan bool // [eth/63] Channel to signal the receipt fetcher of new tasks - headerProcCh chan []*types.Header // [eth/62] Channel to feed the header processor new tasks + headerCh chan dataPack // Channel receiving inbound block headers + bodyCh chan dataPack // Channel receiving inbound block bodies + receiptCh chan dataPack // Channel receiving inbound receipts + bodyWakeCh chan bool // Channel to signal the block body fetcher of new tasks + receiptWakeCh chan bool // Channel to signal the receipt fetcher of new tasks + headerProcCh chan []*types.Header // Channel to feed the header processor new tasks // State sync pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root pivotLock sync.RWMutex // Lock protecting pivot header reads from updates + snapSync bool // Whether to run state sync over the snap protocol + SnapSyncer *snap.Syncer // TODO(karalabe): make private! hack for now stateSyncStart chan *stateSync trackStateReq chan *stateReq - stateCh chan dataPack // [eth/63] Channel receiving inbound node state data + stateCh chan dataPack // Channel receiving inbound node state data // Cancellation and termination cancelPeer string // Identifier of the peer currently being used as the master (cancel on drop) @@ -237,6 +239,7 @@ func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, headerProcCh: make(chan []*types.Header, 1), quitCh: make(chan struct{}), stateCh: make(chan dataPack), + SnapSyncer: snap.NewSyncer(stateDb, stateBloom), stateSyncStart: make(chan *stateSync), syncStatsState: stateSyncStats{ processed: rawdb.ReadFastTrieProgress(stateDb), @@ -286,19 +289,16 @@ func (d *Downloader) Synchronising() bool { return atomic.LoadInt32(&d.synchronising) > 0 } -// SyncBloomContains tests if the syncbloom filter contains the given hash: -// - false: the bloom definitely does not contain hash -// - true: the bloom maybe contains hash -// -// While the bloom is being initialized (or is closed), all queries will return true. -func (d *Downloader) SyncBloomContains(hash []byte) bool { - return d.stateBloom == nil || d.stateBloom.Contains(hash) -} - // RegisterPeer injects a new download peer into the set of block source to be // used for fetching hashes and blocks from. -func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error { - logger := log.New("peer", id) +func (d *Downloader) RegisterPeer(id string, version uint, peer Peer) error { + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:16]) + } logger.Trace("Registering sync peer") if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil { logger.Error("Failed to register sync peer", "err", err) @@ -310,7 +310,7 @@ func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error { } // RegisterLightPeer injects a light client peer, wrapping it so it appears as a regular peer. -func (d *Downloader) RegisterLightPeer(id string, version int, peer LightPeer) error { +func (d *Downloader) RegisterLightPeer(id string, version uint, peer LightPeer) error { return d.RegisterPeer(id, version, &lightPeerWrapper{peer}) } @@ -319,7 +319,13 @@ func (d *Downloader) RegisterLightPeer(id string, version int, peer LightPeer) e // the queue. func (d *Downloader) UnregisterPeer(id string) error { // Unregister the peer from the active peer set and revoke any fetch tasks - logger := log.New("peer", id) + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:16]) + } logger.Trace("Unregistering sync peer") if err := d.peers.Unregister(id); err != nil { logger.Error("Failed to unregister sync peer", "err", err) @@ -381,6 +387,16 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode if mode == FullSync && d.stateBloom != nil { d.stateBloom.Close() } + // If snap sync was requested, create the snap scheduler and switch to fast + // sync mode. Long term we could drop fast sync or merge the two together, + // but until snap becomes prevalent, we should support both. TODO(karalabe). + if mode == SnapSync { + if !d.snapSync { + log.Warn("Enabling snapshot sync prototype") + d.snapSync = true + } + mode = FastSync + } // Reset the queue, peer set and wake channels to clean any internal leftover state d.queue.Reset(blockCacheMaxItems, blockCacheInitialItems) d.peers.Reset() @@ -443,8 +459,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < 63 { - return errTooOld + if p.version < 64 { + return fmt.Errorf("%w, peer version: %d", errTooOld, p.version) } mode := d.getMode() @@ -1910,27 +1926,53 @@ func (d *Downloader) commitPivotBlock(result *fetchResult) error { // DeliverHeaders injects a new batch of block headers received from a remote // node into the download schedule. -func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) (err error) { - return d.deliver(id, d.headerCh, &headerPack{id, headers}, headerInMeter, headerDropMeter) +func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) error { + return d.deliver(d.headerCh, &headerPack{id, headers}, headerInMeter, headerDropMeter) } // DeliverBodies injects a new batch of block bodies received from a remote node. -func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transaction, uncles [][]*types.Header) (err error) { - return d.deliver(id, d.bodyCh, &bodyPack{id, transactions, uncles}, bodyInMeter, bodyDropMeter) +func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transaction, uncles [][]*types.Header) error { + return d.deliver(d.bodyCh, &bodyPack{id, transactions, uncles}, bodyInMeter, bodyDropMeter) } // DeliverReceipts injects a new batch of receipts received from a remote node. -func (d *Downloader) DeliverReceipts(id string, receipts [][]*types.Receipt) (err error) { - return d.deliver(id, d.receiptCh, &receiptPack{id, receipts}, receiptInMeter, receiptDropMeter) +func (d *Downloader) DeliverReceipts(id string, receipts [][]*types.Receipt) error { + return d.deliver(d.receiptCh, &receiptPack{id, receipts}, receiptInMeter, receiptDropMeter) } // DeliverNodeData injects a new batch of node state data received from a remote node. -func (d *Downloader) DeliverNodeData(id string, data [][]byte) (err error) { - return d.deliver(id, d.stateCh, &statePack{id, data}, stateInMeter, stateDropMeter) +func (d *Downloader) DeliverNodeData(id string, data [][]byte) error { + return d.deliver(d.stateCh, &statePack{id, data}, stateInMeter, stateDropMeter) +} + +// DeliverSnapPacket is invoked from a peer's message handler when it transmits a +// data packet for the local node to consume. +func (d *Downloader) DeliverSnapPacket(peer *snap.Peer, packet snap.Packet) error { + switch packet := packet.(type) { + case *snap.AccountRangePacket: + hashes, accounts, err := packet.Unpack() + if err != nil { + return err + } + return d.SnapSyncer.OnAccounts(peer, packet.ID, hashes, accounts, packet.Proof) + + case *snap.StorageRangesPacket: + hashset, slotset := packet.Unpack() + return d.SnapSyncer.OnStorage(peer, packet.ID, hashset, slotset, packet.Proof) + + case *snap.ByteCodesPacket: + return d.SnapSyncer.OnByteCodes(peer, packet.ID, packet.Codes) + + case *snap.TrieNodesPacket: + return d.SnapSyncer.OnTrieNodes(peer, packet.ID, packet.Nodes) + + default: + return fmt.Errorf("unexpected snap packet type: %T", packet) + } } // deliver injects a new batch of data received from a remote node. -func (d *Downloader) deliver(id string, destCh chan dataPack, packet dataPack, inMeter, dropMeter metrics.Meter) (err error) { +func (d *Downloader) deliver(destCh chan dataPack, packet dataPack, inMeter, dropMeter metrics.Meter) (err error) { // Update the delivery metrics for both good and failed deliveries inMeter.Mark(int64(packet.Items())) defer func() { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 5e46042ae4..6578275d0c 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -390,7 +390,7 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version int, chain *testChain) error { +func (dl *downloadTester) newPeer(id string, version uint, chain *testChain) error { dl.lock.Lock() defer dl.lock.Unlock() @@ -518,8 +518,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng // Tests that simple synchronization against a canonical chain works correctly. // In this test common ancestor lookup should be short circuited and not require // binary searching. -func TestCanonicalSynchronisation63Full(t *testing.T) { testCanonicalSynchronisation(t, 63, FullSync) } -func TestCanonicalSynchronisation63Fast(t *testing.T) { testCanonicalSynchronisation(t, 63, FastSync) } func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonicalSynchronisation(t, 64, FullSync) } func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonicalSynchronisation(t, 64, FastSync) } func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonicalSynchronisation(t, 65, FullSync) } @@ -528,7 +526,7 @@ func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonicalSynchronisation(t, 65, LightSync) } -func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { +func testCanonicalSynchronisation(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -547,14 +545,12 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling63Full(t *testing.T) { testThrottling(t, 63, FullSync) } -func TestThrottling63Fast(t *testing.T) { testThrottling(t, 63, FastSync) } func TestThrottling64Full(t *testing.T) { testThrottling(t, 64, FullSync) } func TestThrottling64Fast(t *testing.T) { testThrottling(t, 64, FastSync) } func TestThrottling65Full(t *testing.T) { testThrottling(t, 65, FullSync) } func TestThrottling65Fast(t *testing.T) { testThrottling(t, 65, FastSync) } -func testThrottling(t *testing.T, protocol int, mode SyncMode) { +func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -632,15 +628,13 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync63Full(t *testing.T) { testForkedSync(t, 63, FullSync) } -func TestForkedSync63Fast(t *testing.T) { testForkedSync(t, 63, FastSync) } func TestForkedSync64Full(t *testing.T) { testForkedSync(t, 64, FullSync) } func TestForkedSync64Fast(t *testing.T) { testForkedSync(t, 64, FastSync) } func TestForkedSync65Full(t *testing.T) { testForkedSync(t, 65, FullSync) } func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, 65, FastSync) } func TestForkedSync65Light(t *testing.T) { testForkedSync(t, 65, LightSync) } -func testForkedSync(t *testing.T, protocol int, mode SyncMode) { +func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -665,15 +659,13 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync63Full(t *testing.T) { testHeavyForkedSync(t, 63, FullSync) } -func TestHeavyForkedSync63Fast(t *testing.T) { testHeavyForkedSync(t, 63, FastSync) } func TestHeavyForkedSync64Full(t *testing.T) { testHeavyForkedSync(t, 64, FullSync) } func TestHeavyForkedSync64Fast(t *testing.T) { testHeavyForkedSync(t, 64, FastSync) } func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, 65, FullSync) } func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, 65, FastSync) } func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, 65, LightSync) } -func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { +func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -700,15 +692,13 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync63Full(t *testing.T) { testBoundedForkedSync(t, 63, FullSync) } -func TestBoundedForkedSync63Fast(t *testing.T) { testBoundedForkedSync(t, 63, FastSync) } func TestBoundedForkedSync64Full(t *testing.T) { testBoundedForkedSync(t, 64, FullSync) } func TestBoundedForkedSync64Fast(t *testing.T) { testBoundedForkedSync(t, 64, FastSync) } func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, 65, FullSync) } func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, 65, FastSync) } func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, 65, LightSync) } -func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { +func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -734,15 +724,13 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync63Full(t *testing.T) { testBoundedHeavyForkedSync(t, 63, FullSync) } -func TestBoundedHeavyForkedSync63Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 63, FastSync) } func TestBoundedHeavyForkedSync64Full(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FullSync) } func TestBoundedHeavyForkedSync64Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FastSync) } func TestBoundedHeavyForkedSync65Full(t *testing.T) { testBoundedHeavyForkedSync(t, 65, FullSync) } func TestBoundedHeavyForkedSync65Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 65, FastSync) } func TestBoundedHeavyForkedSync65Light(t *testing.T) { testBoundedHeavyForkedSync(t, 65, LightSync) } -func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { +func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -786,15 +774,13 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel63Full(t *testing.T) { testCancel(t, 63, FullSync) } -func TestCancel63Fast(t *testing.T) { testCancel(t, 63, FastSync) } func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) } func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) } func TestCancel65Full(t *testing.T) { testCancel(t, 65, FullSync) } func TestCancel65Fast(t *testing.T) { testCancel(t, 65, FastSync) } func TestCancel65Light(t *testing.T) { testCancel(t, 65, LightSync) } -func testCancel(t *testing.T, protocol int, mode SyncMode) { +func testCancel(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -819,15 +805,13 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation63Full(t *testing.T) { testMultiSynchronisation(t, 63, FullSync) } -func TestMultiSynchronisation63Fast(t *testing.T) { testMultiSynchronisation(t, 63, FastSync) } func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, 64, FullSync) } func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) } func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, 65, FullSync) } func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, 65, FastSync) } func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, 65, LightSync) } -func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { +func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -849,15 +833,13 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation63Full(t *testing.T) { testMultiProtoSync(t, 63, FullSync) } -func TestMultiProtoSynchronisation63Fast(t *testing.T) { testMultiProtoSync(t, 63, FastSync) } func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, 64, FullSync) } func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) } func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, 65, FullSync) } func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, 65, FastSync) } func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, 65, LightSync) } -func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { +func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -888,15 +870,13 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit63Full(t *testing.T) { testEmptyShortCircuit(t, 63, FullSync) } -func TestEmptyShortCircuit63Fast(t *testing.T) { testEmptyShortCircuit(t, 63, FastSync) } func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, FullSync) } func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) } func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, 65, FullSync) } func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, 65, FastSync) } func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, 65, LightSync) } -func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { +func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -942,15 +922,13 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack63Full(t *testing.T) { testMissingHeaderAttack(t, 63, FullSync) } -func TestMissingHeaderAttack63Fast(t *testing.T) { testMissingHeaderAttack(t, 63, FastSync) } func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 64, FullSync) } func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) } func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, 65, FullSync) } func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, 65, FastSync) } func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, 65, LightSync) } -func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { +func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -974,15 +952,13 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack63Full(t *testing.T) { testShiftedHeaderAttack(t, 63, FullSync) } -func TestShiftedHeaderAttack63Fast(t *testing.T) { testShiftedHeaderAttack(t, 63, FastSync) } func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 64, FullSync) } func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) } func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, 65, FullSync) } func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, 65, FastSync) } func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, 65, LightSync) } -func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { +func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -1011,11 +987,10 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. -func TestInvalidHeaderRollback63Fast(t *testing.T) { testInvalidHeaderRollback(t, 63, FastSync) } func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) } func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, 65, FastSync) } -func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { +func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -1103,15 +1078,13 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack63Full(t *testing.T) { testHighTDStarvationAttack(t, 63, FullSync) } -func TestHighTDStarvationAttack63Fast(t *testing.T) { testHighTDStarvationAttack(t, 63, FastSync) } func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttack(t, 64, FullSync) } func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } func TestHighTDStarvationAttack65Full(t *testing.T) { testHighTDStarvationAttack(t, 65, FullSync) } func TestHighTDStarvationAttack65Fast(t *testing.T) { testHighTDStarvationAttack(t, 65, FastSync) } func TestHighTDStarvationAttack65Light(t *testing.T) { testHighTDStarvationAttack(t, 65, LightSync) } -func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { +func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -1125,11 +1098,10 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping63(t *testing.T) { testBlockHeaderAttackerDropping(t, 63) } func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, 65) } -func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { +func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { t.Parallel() // Define the disconnection requirement for individual hash fetch errors @@ -1179,15 +1151,13 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) } -func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) } func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) } func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) } func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, 65, FullSync) } func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, 65, FastSync) } func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, 65, LightSync) } -func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { +func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -1263,21 +1233,19 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress63Full(t *testing.T) { testForkedSyncProgress(t, 63, FullSync) } -func TestForkedSyncProgress63Fast(t *testing.T) { testForkedSyncProgress(t, 63, FastSync) } func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, FullSync) } func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) } func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, 65, FullSync) } func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, 65, FastSync) } func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, 65, LightSync) } -func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { +func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() defer tester.terminate() - chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHashFetch) - chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHashFetch) + chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHeaderFetch) + chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHeaderFetch) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1339,15 +1307,13 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress63Full(t *testing.T) { testFailedSyncProgress(t, 63, FullSync) } -func TestFailedSyncProgress63Fast(t *testing.T) { testFailedSyncProgress(t, 63, FastSync) } func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, FullSync) } func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) } func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, 65, FullSync) } func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, 65, FastSync) } func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, 65, LightSync) } -func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { +func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -1412,15 +1378,13 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress63Full(t *testing.T) { testFakedSyncProgress(t, 63, FullSync) } -func TestFakedSyncProgress63Fast(t *testing.T) { testFakedSyncProgress(t, 63, FastSync) } func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, FullSync) } func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) } func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, 65, FullSync) } func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, 65, FastSync) } func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, 65, LightSync) } -func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { +func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -1489,31 +1453,15 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang(t *testing.T) { - t.Parallel() +func TestDeliverHeadersHang64Full(t *testing.T) { testDeliverHeadersHang(t, 64, FullSync) } +func TestDeliverHeadersHang64Fast(t *testing.T) { testDeliverHeadersHang(t, 64, FastSync) } +func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, 65, FullSync) } +func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, 65, FastSync) } +func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, 65, LightSync) } - testCases := []struct { - protocol int - syncMode SyncMode - }{ - {63, FullSync}, - {63, FastSync}, - {64, FullSync}, - {64, FastSync}, - {64, LightSync}, - {65, FullSync}, - {65, FastSync}, - {65, LightSync}, - } - for _, tc := range testCases { - t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) { - t.Parallel() - testDeliverHeadersHang(t, tc.protocol, tc.syncMode) - }) - } -} +func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() -func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { master := newTester() defer master.terminate() chain := testChainBase.shorten(15) @@ -1664,15 +1612,13 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement63Full(t *testing.T) { testCheckpointEnforcement(t, 63, FullSync) } -func TestCheckpointEnforcement63Fast(t *testing.T) { testCheckpointEnforcement(t, 63, FastSync) } func TestCheckpointEnforcement64Full(t *testing.T) { testCheckpointEnforcement(t, 64, FullSync) } func TestCheckpointEnforcement64Fast(t *testing.T) { testCheckpointEnforcement(t, 64, FastSync) } func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, 65, FullSync) } func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, 65, FastSync) } func TestCheckpointEnforcement65Light(t *testing.T) { testCheckpointEnforcement(t, 65, LightSync) } -func testCheckpointEnforcement(t *testing.T, protocol int, mode SyncMode) { +func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() // Create a new tester with a particular hard coded checkpoint block diff --git a/eth/downloader/modes.go b/eth/downloader/modes.go index d866ceabce..8ea7876a1f 100644 --- a/eth/downloader/modes.go +++ b/eth/downloader/modes.go @@ -24,7 +24,8 @@ type SyncMode uint32 const ( FullSync SyncMode = iota // Synchronise the entire blockchain history from full blocks - FastSync // Quickly download the headers, full sync only at the chain head + FastSync // Quickly download the headers, full sync only at the chain + SnapSync // Download the chain and the state via compact snashots LightSync // Download only the headers and terminate afterwards ) @@ -39,6 +40,8 @@ func (mode SyncMode) String() string { return "full" case FastSync: return "fast" + case SnapSync: + return "snap" case LightSync: return "light" default: @@ -52,6 +55,8 @@ func (mode SyncMode) MarshalText() ([]byte, error) { return []byte("full"), nil case FastSync: return []byte("fast"), nil + case SnapSync: + return []byte("snap"), nil case LightSync: return []byte("light"), nil default: @@ -65,6 +70,8 @@ func (mode *SyncMode) UnmarshalText(text []byte) error { *mode = FullSync case "fast": *mode = FastSync + case "snap": + *mode = SnapSync case "light": *mode = LightSync default: diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index c6671436f9..ba90bf31cb 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -69,7 +69,7 @@ type peerConnection struct { peer Peer - version int // Eth protocol version number to switch strategies + version uint // Eth protocol version number to switch strategies log log.Logger // Contextual logger to add extra infos to peer logs lock sync.RWMutex } @@ -112,7 +112,7 @@ func (w *lightPeerWrapper) RequestNodeData([]common.Hash) error { } // newPeerConnection creates a new downloader peer. -func newPeerConnection(id string, version int, peer Peer, logger log.Logger) *peerConnection { +func newPeerConnection(id string, version uint, peer Peer, logger log.Logger) *peerConnection { return &peerConnection{ id: id, lacking: make(map[common.Hash]struct{}), @@ -457,7 +457,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.headerThroughput } - return ps.idlePeers(63, 65, idle, throughput) + return ps.idlePeers(64, 65, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -471,7 +471,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.blockThroughput } - return ps.idlePeers(63, 65, idle, throughput) + return ps.idlePeers(64, 65, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -485,7 +485,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.receiptThroughput } - return ps.idlePeers(63, 65, idle, throughput) + return ps.idlePeers(64, 65, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -499,13 +499,13 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.stateThroughput } - return ps.idlePeers(63, 65, idle, throughput) + return ps.idlePeers(64, 65, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the // protocol version constraints, using the provided function to check idleness. // The resulting set of peers are sorted by their measure throughput. -func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peerConnection) bool, throughput func(*peerConnection) float64) ([]*peerConnection, int) { +func (ps *peerSet) idlePeers(minProtocol, maxProtocol uint, idleCheck func(*peerConnection) bool, throughput func(*peerConnection) float64) ([]*peerConnection, int) { ps.lock.RLock() defer ps.lock.RUnlock() diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index d2ec8ba694..2150842f8e 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -113,24 +113,24 @@ type queue struct { mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching // Headers are "special", they download in batches, supported by a skeleton chain - headerHead common.Hash // [eth/62] Hash of the last queued header to verify order - headerTaskPool map[uint64]*types.Header // [eth/62] Pending header retrieval tasks, mapping starting indexes to skeleton headers - headerTaskQueue *prque.Prque // [eth/62] Priority queue of the skeleton indexes to fetch the filling headers for - headerPeerMiss map[string]map[uint64]struct{} // [eth/62] Set of per-peer header batches known to be unavailable - headerPendPool map[string]*fetchRequest // [eth/62] Currently pending header retrieval operations - headerResults []*types.Header // [eth/62] Result cache accumulating the completed headers - headerProced int // [eth/62] Number of headers already processed from the results - headerOffset uint64 // [eth/62] Number of the first header in the result cache - headerContCh chan bool // [eth/62] Channel to notify when header download finishes + headerHead common.Hash // Hash of the last queued header to verify order + headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers + headerTaskQueue *prque.Prque // Priority queue of the skeleton indexes to fetch the filling headers for + headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable + headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations + headerResults []*types.Header // Result cache accumulating the completed headers + headerProced int // Number of headers already processed from the results + headerOffset uint64 // Number of the first header in the result cache + headerContCh chan bool // Channel to notify when header download finishes // All data retrievals below are based on an already assembles header chain - blockTaskPool map[common.Hash]*types.Header // [eth/62] Pending block (body) retrieval tasks, mapping hashes to headers - blockTaskQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the blocks (bodies) for - blockPendPool map[string]*fetchRequest // [eth/62] Currently pending block (body) retrieval operations + blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers + blockTaskQueue *prque.Prque // Priority queue of the headers to fetch the blocks (bodies) for + blockPendPool map[string]*fetchRequest // Currently pending block (body) retrieval operations - receiptTaskPool map[common.Hash]*types.Header // [eth/63] Pending receipt retrieval tasks, mapping hashes to headers - receiptTaskQueue *prque.Prque // [eth/63] Priority queue of the headers to fetch the receipts for - receiptPendPool map[string]*fetchRequest // [eth/63] Currently pending receipt retrieval operations + receiptTaskPool map[common.Hash]*types.Header // Pending receipt retrieval tasks, mapping hashes to headers + receiptTaskQueue *prque.Prque // Priority queue of the headers to fetch the receipts for + receiptPendPool map[string]*fetchRequest // Currently pending receipt retrieval operations resultCache *resultStore // Downloaded but not yet delivered fetch results resultSize common.StorageSize // Approximate size of a block (exponential moving average) @@ -690,6 +690,13 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh q.lock.Lock() defer q.lock.Unlock() + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:16]) + } // Short circuit if the data was never requested request := q.headerPendPool[id] if request == nil { @@ -704,10 +711,10 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh accepted := len(headers) == MaxHeaderFetch if accepted { if headers[0].Number.Uint64() != request.From { - log.Trace("First header broke chain ordering", "peer", id, "number", headers[0].Number, "hash", headers[0].Hash(), request.From) + logger.Trace("First header broke chain ordering", "number", headers[0].Number, "hash", headers[0].Hash(), "expected", request.From) accepted = false } else if headers[len(headers)-1].Hash() != target { - log.Trace("Last header broke skeleton structure ", "peer", id, "number", headers[len(headers)-1].Number, "hash", headers[len(headers)-1].Hash(), "expected", target) + logger.Trace("Last header broke skeleton structure ", "number", headers[len(headers)-1].Number, "hash", headers[len(headers)-1].Hash(), "expected", target) accepted = false } } @@ -716,12 +723,12 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh for i, header := range headers[1:] { hash := header.Hash() if want := request.From + 1 + uint64(i); header.Number.Uint64() != want { - log.Warn("Header broke chain ordering", "peer", id, "number", header.Number, "hash", hash, "expected", want) + logger.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", want) accepted = false break } if parentHash != header.ParentHash { - log.Warn("Header broke chain ancestry", "peer", id, "number", header.Number, "hash", hash) + logger.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) accepted = false break } @@ -731,7 +738,7 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh } // If the batch of headers wasn't accepted, mark as unavailable if !accepted { - log.Trace("Skeleton filling not accepted", "peer", id, "from", request.From) + logger.Trace("Skeleton filling not accepted", "from", request.From) miss := q.headerPeerMiss[id] if miss == nil { @@ -758,7 +765,7 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh select { case headerProcCh <- process: - log.Trace("Pre-scheduled new headers", "peer", id, "count", len(process), "from", process[0].Number) + logger.Trace("Pre-scheduled new headers", "count", len(process), "from", process[0].Number) q.headerProced += len(process) default: } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 6745aa54ac..69bd13c2f7 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -101,8 +101,16 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { finished []*stateReq // Completed or failed requests timeout = make(chan *stateReq) // Timed out active requests ) - // Run the state sync. log.Trace("State sync starting", "root", s.root) + + defer func() { + // Cancel active request timers on exit. Also set peers to idle so they're + // available for the next sync. + for _, req := range active { + req.timer.Stop() + req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) + } + }() go s.run() defer s.Cancel() @@ -252,8 +260,9 @@ func (d *Downloader) spindownStateSync(active map[string]*stateReq, finished []* type stateSync struct { d *Downloader // Downloader instance to access and manage current peerset - sched *trie.Sync // State trie sync scheduler defining the tasks - keccak hash.Hash // Keccak256 hasher to verify deliveries with + root common.Hash // State root currently being synced + sched *trie.Sync // State trie sync scheduler defining the tasks + keccak hash.Hash // Keccak256 hasher to verify deliveries with trieTasks map[common.Hash]*trieTask // Set of trie node tasks currently queued for retrieval codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval @@ -268,8 +277,6 @@ type stateSync struct { cancelOnce sync.Once // Ensures cancel only ever gets called once done chan struct{} // Channel to signal termination completion err error // Any error hit during sync (set before completion) - - root common.Hash } // trieTask represents a single trie node download task, containing a set of @@ -290,6 +297,7 @@ type codeTask struct { func newStateSync(d *Downloader, root common.Hash) *stateSync { return &stateSync{ d: d, + root: root, sched: state.NewStateSync(root, d.stateDB, d.stateBloom), keccak: sha3.NewLegacyKeccak256(), trieTasks: make(map[common.Hash]*trieTask), @@ -298,7 +306,6 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { cancel: make(chan struct{}), done: make(chan struct{}), started: make(chan struct{}), - root: root, } } @@ -306,7 +313,12 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { // it finishes, and finally notifying any goroutines waiting for the loop to // finish. func (s *stateSync) run() { - s.err = s.loop() + close(s.started) + if s.d.snapSync { + s.err = s.d.SnapSyncer.Sync(s.root, s.cancel) + } else { + s.err = s.loop() + } close(s.done) } @@ -318,7 +330,9 @@ func (s *stateSync) Wait() error { // Cancel cancels the sync and waits until it has shut down. func (s *stateSync) Cancel() error { - s.cancelOnce.Do(func() { close(s.cancel) }) + s.cancelOnce.Do(func() { + close(s.cancel) + }) return s.Wait() } @@ -329,7 +343,6 @@ func (s *stateSync) Cancel() error { // pushed here async. The reason is to decouple processing from data receipt // and timeouts. func (s *stateSync) loop() (err error) { - close(s.started) // Listen for new peer events to assign tasks to them newPeer := make(chan *peerConnection, 1024) peerSub := s.d.peers.SubscribeNewPeers(newPeer) diff --git a/eth/gen_config.go b/eth/gen_config.go index b0674c7d77..dd04635eee 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -20,7 +20,7 @@ func (c Config) MarshalTOML() (interface{}, error) { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode downloader.SyncMode - DiscoveryURLs []string + EthDiscoveryURLs []string NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` @@ -61,7 +61,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Genesis = c.Genesis enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode - enc.DiscoveryURLs = c.DiscoveryURLs + enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit @@ -106,7 +106,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *downloader.SyncMode - DiscoveryURLs []string + EthDiscoveryURLs []string NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` @@ -156,8 +156,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SyncMode != nil { c.SyncMode = *dec.SyncMode } - if dec.DiscoveryURLs != nil { - c.DiscoveryURLs = dec.DiscoveryURLs + if dec.EthDiscoveryURLs != nil { + c.EthDiscoveryURLs = dec.EthDiscoveryURLs } if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning diff --git a/eth/handler.go b/eth/handler.go index 5b89986539..76a429f6d3 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -17,9 +17,7 @@ package eth import ( - "encoding/json" "errors" - "fmt" "math" "math/big" "sync" @@ -27,26 +25,22 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) const ( - softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. - estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header - // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 @@ -56,26 +50,61 @@ var ( syncChallengeTimeout = 15 * time.Second // Time allowance for a node to reply to the sync progress challenge ) -func errResp(code errCode, format string, v ...interface{}) error { - return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) +// txPool defines the methods needed from a transaction pool implementation to +// support all the operations needed by the Ethereum chain protocols. +type txPool interface { + // Has returns an indicator whether txpool has a transaction + // cached with the given hash. + Has(hash common.Hash) bool + + // Get retrieves the transaction from local txpool with given + // tx hash. + Get(hash common.Hash) *types.Transaction + + // AddRemotes should add the given transactions to the pool. + AddRemotes([]*types.Transaction) []error + + // Pending should return pending transactions. + // The slice should be modifiable by the caller. + Pending() (map[common.Address]types.Transactions, error) + + // SubscribeNewTxsEvent should return an event subscription of + // NewTxsEvent and send events to the given channel. + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription } -type ProtocolManager struct { +// handlerConfig is the collection of initialization parameters to create a full +// node network handler. +type handlerConfig struct { + Database ethdb.Database // Database for direct sync insertions + Chain *core.BlockChain // Blockchain to serve data from + TxPool txPool // Transaction pool to propagate from + Network uint64 // Network identifier to adfvertise + Sync downloader.SyncMode // Whether to fast or full sync + BloomCache uint64 // Megabytes to alloc for fast sync bloom + EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` + Checkpoint *params.TrustedCheckpoint // Hard coded checkpoint for sync challenges + Whitelist map[uint64]common.Hash // Hard coded whitelist for sync challenged +} + +type handler struct { networkID uint64 forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node fastSync uint32 // Flag whether fast sync is enabled (gets disabled if we already have blocks) + snapSync uint32 // Flag whether fast sync should operate on top of the snap protocol acceptTxs uint32 // Flag whether we're considered synchronised (enables transaction processing) checkpointNumber uint64 // Block number for the sync progress validator to cross reference checkpointHash common.Hash // Block hash for the sync progress validator to cross reference - txpool txPool - blockchain *core.BlockChain - chaindb ethdb.Database - maxPeers int + database ethdb.Database + txpool txPool + chain *core.BlockChain + maxPeers int downloader *downloader.Downloader + stateBloom *trie.SyncBloom blockFetcher *fetcher.BlockFetcher txFetcher *fetcher.TxFetcher peers *peerSet @@ -94,29 +123,27 @@ type ProtocolManager struct { chainSync *chainSyncer wg sync.WaitGroup peerWG sync.WaitGroup - - // Test fields or hooks - broadcastTxAnnouncesOnly bool // Testing field, disable transaction propagation } -// NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable -// with the Ethereum network. -func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCheckpoint, mode downloader.SyncMode, networkID uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database, cacheLimit int, whitelist map[uint64]common.Hash) (*ProtocolManager, error) { +// newHandler returns a handler for all Ethereum chain management protocol. +func newHandler(config *handlerConfig) (*handler, error) { // Create the protocol manager with the base fields - manager := &ProtocolManager{ - networkID: networkID, - forkFilter: forkid.NewFilter(blockchain), - eventMux: mux, - txpool: txpool, - blockchain: blockchain, - chaindb: chaindb, + if config.EventMux == nil { + config.EventMux = new(event.TypeMux) // Nicety initialization for tests + } + h := &handler{ + networkID: config.Network, + forkFilter: forkid.NewFilter(config.Chain), + eventMux: config.EventMux, + database: config.Database, + txpool: config.TxPool, + chain: config.Chain, peers: newPeerSet(), - whitelist: whitelist, + whitelist: config.Whitelist, txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } - - if mode == downloader.FullSync { + if config.Sync == downloader.FullSync { // The database seems empty as the current block is the genesis. Yet the fast // block is ahead, so fast sync was enabled for this node at a certain point. // The scenarios where this can happen is @@ -125,42 +152,42 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh // * the last fast sync is not finished while user specifies a full sync this // time. But we don't have any recent state for full sync. // In these cases however it's safe to reenable fast sync. - fullBlock, fastBlock := blockchain.CurrentBlock(), blockchain.CurrentFastBlock() + fullBlock, fastBlock := h.chain.CurrentBlock(), h.chain.CurrentFastBlock() if fullBlock.NumberU64() == 0 && fastBlock.NumberU64() > 0 { - manager.fastSync = uint32(1) + h.fastSync = uint32(1) log.Warn("Switch sync mode from full sync to fast sync") } } else { - if blockchain.CurrentBlock().NumberU64() > 0 { + if h.chain.CurrentBlock().NumberU64() > 0 { // Print warning log if database is not empty to run fast sync. log.Warn("Switch sync mode from fast sync to full sync") } else { // If fast sync was requested and our database is empty, grant it - manager.fastSync = uint32(1) + h.fastSync = uint32(1) + if config.Sync == downloader.SnapSync { + h.snapSync = uint32(1) + } } } - // If we have trusted checkpoints, enforce them on the chain - if checkpoint != nil { - manager.checkpointNumber = (checkpoint.SectionIndex+1)*params.CHTFrequency - 1 - manager.checkpointHash = checkpoint.SectionHead + if config.Checkpoint != nil { + h.checkpointNumber = (config.Checkpoint.SectionIndex+1)*params.CHTFrequency - 1 + h.checkpointHash = config.Checkpoint.SectionHead } - // Construct the downloader (long sync) and its backing state bloom if fast // sync is requested. The downloader is responsible for deallocating the state // bloom when it's done. - var stateBloom *trie.SyncBloom - if atomic.LoadUint32(&manager.fastSync) == 1 { - stateBloom = trie.NewSyncBloom(uint64(cacheLimit), chaindb) + if atomic.LoadUint32(&h.fastSync) == 1 { + h.stateBloom = trie.NewSyncBloom(config.BloomCache, config.Database) } - manager.downloader = downloader.New(manager.checkpointNumber, chaindb, stateBloom, manager.eventMux, blockchain, nil, manager.removePeer) + h.downloader = downloader.New(h.checkpointNumber, config.Database, h.stateBloom, h.eventMux, h.chain, nil, h.removePeer) // Construct the fetcher (short sync) validator := func(header *types.Header) error { - return engine.VerifyHeader(blockchain, header, true) + return h.chain.Engine().VerifyHeader(h.chain, header, true) } heighter := func() uint64 { - return blockchain.CurrentBlock().NumberU64() + return h.chain.CurrentBlock().NumberU64() } inserter := func(blocks types.Blocks) (int, error) { // If sync hasn't reached the checkpoint yet, deny importing weird blocks. @@ -169,7 +196,7 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh // the propagated block if the head is too old. Unfortunately there is a corner // case when starting new networks, where the genesis might be ancient (0 unix) // which would prevent full nodes from accepting it. - if manager.blockchain.CurrentBlock().NumberU64() < manager.checkpointNumber { + if h.chain.CurrentBlock().NumberU64() < h.checkpointNumber { log.Warn("Unsynced yet, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } @@ -178,180 +205,88 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh // accept each others' blocks until a restart. Unfortunately we haven't figured // out a way yet where nodes can decide unilaterally whether the network is new // or not. This should be fixed if we figure out a solution. - if atomic.LoadUint32(&manager.fastSync) == 1 { + if atomic.LoadUint32(&h.fastSync) == 1 { log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } - n, err := manager.blockchain.InsertChain(blocks) + n, err := h.chain.InsertChain(blocks) if err == nil { - atomic.StoreUint32(&manager.acceptTxs, 1) // Mark initial sync done on any fetcher import + atomic.StoreUint32(&h.acceptTxs, 1) // Mark initial sync done on any fetcher import } return n, err } - manager.blockFetcher = fetcher.NewBlockFetcher(false, nil, blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, nil, inserter, manager.removePeer) + h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) fetchTx := func(peer string, hashes []common.Hash) error { - p := manager.peers.Peer(peer) + p := h.peers.ethPeer(peer) if p == nil { return errors.New("unknown peer") } return p.RequestTxs(hashes) } - manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, fetchTx) - - manager.chainSync = newChainSyncer(manager) - - return manager, nil -} - -func (pm *ProtocolManager) makeProtocol(version uint) p2p.Protocol { - length, ok := protocolLengths[version] - if !ok { - panic("makeProtocol for unknown version") - } - - return p2p.Protocol{ - Name: protocolName, - Version: version, - Length: length, - Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - return pm.runPeer(pm.newPeer(int(version), p, rw, pm.txpool.Get)) - }, - NodeInfo: func() interface{} { - return pm.NodeInfo() - }, - PeerInfo: func(id enode.ID) interface{} { - if p := pm.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil { - return p.Info() - } - return nil - }, - } + h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, h.txpool.AddRemotes, fetchTx) + h.chainSync = newChainSyncer(h) + return h, nil } -func (pm *ProtocolManager) removePeer(id string) { - // Short circuit if the peer was already removed - peer := pm.peers.Peer(id) - if peer == nil { - return - } - log.Debug("Removing Ethereum peer", "peer", id) - - // Unregister the peer from the downloader and Ethereum peer set - pm.downloader.UnregisterPeer(id) - pm.txFetcher.Drop(id) - - if err := pm.peers.Unregister(id); err != nil { - log.Error("Peer removal failed", "peer", id, "err", err) - } - // Hard disconnect at the networking layer - if peer != nil { - peer.Peer.Disconnect(p2p.DiscUselessPeer) - } -} - -func (pm *ProtocolManager) Start(maxPeers int) { - pm.maxPeers = maxPeers - - // broadcast transactions - pm.wg.Add(1) - pm.txsCh = make(chan core.NewTxsEvent, txChanSize) - pm.txsSub = pm.txpool.SubscribeNewTxsEvent(pm.txsCh) - go pm.txBroadcastLoop() - - // broadcast mined blocks - pm.wg.Add(1) - pm.minedBlockSub = pm.eventMux.Subscribe(core.NewMinedBlockEvent{}) - go pm.minedBroadcastLoop() - - // start sync handlers - pm.wg.Add(2) - go pm.chainSync.loop() - go pm.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. -} - -func (pm *ProtocolManager) Stop() { - pm.txsSub.Unsubscribe() // quits txBroadcastLoop - pm.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop - - // Quit chainSync and txsync64. - // After this is done, no new peers will be accepted. - close(pm.quitSync) - pm.wg.Wait() - - // Disconnect existing sessions. - // This also closes the gate for any new registrations on the peer set. - // sessions which are already established but not added to pm.peers yet - // will exit when they try to register. - pm.peers.Close() - pm.peerWG.Wait() - - log.Info("Ethereum protocol stopped") -} - -func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { - return newPeer(pv, p, rw, getPooledTx) -} - -func (pm *ProtocolManager) runPeer(p *peer) error { - if !pm.chainSync.handlePeerEvent(p) { +// runEthPeer +func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { + if !h.chainSync.handlePeerEvent(peer) { return p2p.DiscQuitting } - pm.peerWG.Add(1) - defer pm.peerWG.Done() - return pm.handle(p) -} - -// handle is the callback invoked to manage the life cycle of an eth peer. When -// this function terminates, the peer is disconnected. -func (pm *ProtocolManager) handle(p *peer) error { - // Ignore maxPeers if this is a trusted peer - if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { - return p2p.DiscTooManyPeers - } - p.Log().Debug("Ethereum peer connected", "name", p.Name()) + h.peerWG.Add(1) + defer h.peerWG.Done() // Execute the Ethereum handshake var ( - genesis = pm.blockchain.Genesis() - head = pm.blockchain.CurrentHeader() + genesis = h.chain.Genesis() + head = h.chain.CurrentHeader() hash = head.Hash() number = head.Number.Uint64() - td = pm.blockchain.GetTd(hash, number) + td = h.chain.GetTd(hash, number) ) - forkID := forkid.NewID(pm.blockchain.Config(), pm.blockchain.Genesis().Hash(), pm.blockchain.CurrentHeader().Number.Uint64()) - if err := p.Handshake(pm.networkID, td, hash, genesis.Hash(), forkID, pm.forkFilter); err != nil { - p.Log().Debug("Ethereum handshake failed", "err", err) + forkID := forkid.NewID(h.chain.Config(), h.chain.Genesis().Hash(), h.chain.CurrentHeader().Number.Uint64()) + if err := peer.Handshake(h.networkID, td, hash, genesis.Hash(), forkID, h.forkFilter); err != nil { + peer.Log().Debug("Ethereum handshake failed", "err", err) return err } + // Ignore maxPeers if this is a trusted peer + if h.peers.Len() >= h.maxPeers && !peer.Peer.Info().Network.Trusted { + return p2p.DiscTooManyPeers + } + peer.Log().Debug("Ethereum peer connected", "name", peer.Name()) // Register the peer locally - if err := pm.peers.Register(p, pm.removePeer); err != nil { - p.Log().Error("Ethereum peer registration failed", "err", err) + if err := h.peers.registerEthPeer(peer); err != nil { + peer.Log().Error("Ethereum peer registration failed", "err", err) return err } - defer pm.removePeer(p.id) + defer h.removePeer(peer.ID()) + p := h.peers.ethPeer(peer.ID()) + if p == nil { + return errors.New("peer dropped during handling") + } // Register the peer in the downloader. If the downloader considers it banned, we disconnect - if err := pm.downloader.RegisterPeer(p.id, p.version, p); err != nil { + if err := h.downloader.RegisterPeer(peer.ID(), peer.Version(), peer); err != nil { return err } - pm.chainSync.handlePeerEvent(p) + h.chainSync.handlePeerEvent(peer) // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. - pm.syncTransactions(p) + h.syncTransactions(peer) // If we have a trusted CHT, reject all peers below that (avoid fast sync eclipse) - if pm.checkpointHash != (common.Hash{}) { + if h.checkpointHash != (common.Hash{}) { // Request the peer's checkpoint header for chain height/weight validation - if err := p.RequestHeadersByNumber(pm.checkpointNumber, 1, 0, false); err != nil { + if err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false); err != nil { return err } // Start a timer to disconnect if the peer doesn't reply in time p.syncDrop = time.AfterFunc(syncChallengeTimeout, func() { - p.Log().Warn("Checkpoint challenge timed out, dropping", "addr", p.RemoteAddr(), "type", p.Name()) - pm.removePeer(p.id) + peer.Log().Warn("Checkpoint challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) + h.removePeer(peer.ID()) }) // Make sure it's cleaned up if the peer dies off defer func() { @@ -362,474 +297,115 @@ func (pm *ProtocolManager) handle(p *peer) error { }() } // If we have any explicit whitelist block hashes, request them - for number := range pm.whitelist { - if err := p.RequestHeadersByNumber(number, 1, 0, false); err != nil { + for number := range h.whitelist { + if err := peer.RequestHeadersByNumber(number, 1, 0, false); err != nil { return err } } // Handle incoming messages until the connection is torn down - for { - if err := pm.handleMsg(p); err != nil { - p.Log().Debug("Ethereum message handling failed", "err", err) - return err - } - } + return handler(peer) } -// handleMsg is invoked whenever an inbound message is received from a remote -// peer. The remote connection is torn down upon returning any error. -func (pm *ProtocolManager) handleMsg(p *peer) error { - // Read the next message from the remote peer, and ensure it's fully consumed - msg, err := p.rw.ReadMsg() - if err != nil { +// runSnapPeer +func (h *handler) runSnapPeer(peer *snap.Peer, handler snap.Handler) error { + h.peerWG.Add(1) + defer h.peerWG.Done() + + // Register the peer locally + if err := h.peers.registerSnapPeer(peer); err != nil { + peer.Log().Error("Snapshot peer registration failed", "err", err) return err } - if msg.Size > protocolMaxMsgSize { - return errResp(ErrMsgTooLarge, "%v > %v", msg.Size, protocolMaxMsgSize) - } - defer msg.Discard() - - // Handle the message depending on its contents - switch { - case msg.Code == StatusMsg: - // Status messages should never arrive after the handshake - return errResp(ErrExtraStatusMsg, "uncontrolled status message") - - // Block header query, collect the requested headers and reply - case msg.Code == GetBlockHeadersMsg: - // Decode the complex header query - var query getBlockHeadersData - if err := msg.Decode(&query); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - hashMode := query.Origin.Hash != (common.Hash{}) - first := true - maxNonCanonical := uint64(100) - - // Gather headers until the fetch or network limits is reached - var ( - bytes common.StorageSize - headers []*types.Header - unknown bool - ) - for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit && len(headers) < downloader.MaxHeaderFetch { - // Retrieve the next header satisfying the query - var origin *types.Header - if hashMode { - if first { - first = false - origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) - if origin != nil { - query.Origin.Number = origin.Number.Uint64() - } - } else { - origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) - } - } else { - origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) - } - if origin == nil { - break - } - headers = append(headers, origin) - bytes += estHeaderRlpSize - - // Advance to the next header of the query - switch { - case hashMode && query.Reverse: - // Hash based traversal towards the genesis block - ancestor := query.Skip + 1 - if ancestor == 0 { - unknown = true - } else { - query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) - unknown = (query.Origin.Hash == common.Hash{}) - } - case hashMode && !query.Reverse: - // Hash based traversal towards the leaf block - var ( - current = origin.Number.Uint64() - next = current + query.Skip + 1 - ) - if next <= current { - infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") - p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) - unknown = true - } else { - if header := pm.blockchain.GetHeaderByNumber(next); header != nil { - nextHash := header.Hash() - expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) - if expOldHash == query.Origin.Hash { - query.Origin.Hash, query.Origin.Number = nextHash, next - } else { - unknown = true - } - } else { - unknown = true - } - } - case query.Reverse: - // Number based traversal towards the genesis block - if query.Origin.Number >= query.Skip+1 { - query.Origin.Number -= query.Skip + 1 - } else { - unknown = true - } - - case !query.Reverse: - // Number based traversal towards the leaf block - query.Origin.Number += query.Skip + 1 - } - } - return p.SendBlockHeaders(headers) + defer h.removePeer(peer.ID()) - case msg.Code == BlockHeadersMsg: - // A batch of headers arrived to one of our previous requests - var headers []*types.Header - if err := msg.Decode(&headers); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // If no headers were received, but we're expencting a checkpoint header, consider it that - if len(headers) == 0 && p.syncDrop != nil { - // Stop the timer either way, decide later to drop or not - p.syncDrop.Stop() - p.syncDrop = nil - - // If we're doing a fast sync, we must enforce the checkpoint block to avoid - // eclipse attacks. Unsynced nodes are welcome to connect after we're done - // joining the network - if atomic.LoadUint32(&pm.fastSync) == 1 { - p.Log().Warn("Dropping unsynced node during fast sync", "addr", p.RemoteAddr(), "type", p.Name()) - return errors.New("unsynced node cannot serve fast sync") - } - } - // Filter out any explicitly requested headers, deliver the rest to the downloader - filter := len(headers) == 1 - if filter { - // If it's a potential sync progress check, validate the content and advertised chain weight - if p.syncDrop != nil && headers[0].Number.Uint64() == pm.checkpointNumber { - // Disable the sync drop timer - p.syncDrop.Stop() - p.syncDrop = nil - - // Validate the header and either drop the peer or continue - if headers[0].Hash() != pm.checkpointHash { - return errors.New("checkpoint hash mismatch") - } - return nil - } - // Otherwise if it's a whitelisted block, validate against the set - if want, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { - if hash := headers[0].Hash(); want != hash { - p.Log().Info("Whitelist mismatch, dropping peer", "number", headers[0].Number.Uint64(), "hash", hash, "want", want) - return errors.New("whitelist block mismatch") - } - p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", want) - } - // Irrelevant of the fork checks, send the header to the fetcher just in case - headers = pm.blockFetcher.FilterHeaders(p.id, headers, time.Now()) - } - if len(headers) > 0 || !filter { - err := pm.downloader.DeliverHeaders(p.id, headers) - if err != nil { - log.Debug("Failed to deliver headers", "err", err) - } - } - - case msg.Code == GetBlockBodiesMsg: - // Decode the retrieval message - msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) - if _, err := msgStream.List(); err != nil { - return err - } - // Gather blocks until the fetch or network limits is reached - var ( - hash common.Hash - bytes int - bodies []rlp.RawValue - ) - for bytes < softResponseLimit && len(bodies) < downloader.MaxBlockFetch { - // Retrieve the hash of the next block - if err := msgStream.Decode(&hash); err == rlp.EOL { - break - } else if err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Retrieve the requested block body, stopping if enough was found - if data := pm.blockchain.GetBodyRLP(hash); len(data) != 0 { - bodies = append(bodies, data) - bytes += len(data) - } - } - return p.SendBlockBodiesRLP(bodies) - - case msg.Code == BlockBodiesMsg: - // A batch of block bodies arrived to one of our previous requests - var request blockBodiesData - if err := msg.Decode(&request); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Deliver them all to the downloader for queuing - transactions := make([][]*types.Transaction, len(request)) - uncles := make([][]*types.Header, len(request)) - - for i, body := range request { - transactions[i] = body.Transactions - uncles[i] = body.Uncles - } - // Filter out any explicitly requested bodies, deliver the rest to the downloader - filter := len(transactions) > 0 || len(uncles) > 0 - if filter { - transactions, uncles = pm.blockFetcher.FilterBodies(p.id, transactions, uncles, time.Now()) - } - if len(transactions) > 0 || len(uncles) > 0 || !filter { - err := pm.downloader.DeliverBodies(p.id, transactions, uncles) - if err != nil { - log.Debug("Failed to deliver bodies", "err", err) - } - } + if err := h.downloader.SnapSyncer.Register(peer); err != nil { + return err + } + // Handle incoming messages until the connection is torn down + return handler(peer) +} - case p.version >= eth63 && msg.Code == GetNodeDataMsg: - // Decode the retrieval message - msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) - if _, err := msgStream.List(); err != nil { - return err - } - // Gather state data until the fetch or network limits is reached - var ( - hash common.Hash - bytes int - data [][]byte - ) - for bytes < softResponseLimit && len(data) < downloader.MaxStateFetch { - // Retrieve the hash of the next state entry - if err := msgStream.Decode(&hash); err == rlp.EOL { - break - } else if err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Retrieve the requested state entry, stopping if enough was found - // todo now the code and trienode is mixed in the protocol level, - // separate these two types. - if !pm.downloader.SyncBloomContains(hash[:]) { - // Only lookup the trie node if there's chance that we actually have it - continue - } - entry, err := pm.blockchain.TrieNode(hash) - if len(entry) == 0 || err != nil { - // Read the contract code with prefix only to save unnecessary lookups. - entry, err = pm.blockchain.ContractCodeWithPrefix(hash) - } - if err == nil && len(entry) > 0 { - data = append(data, entry) - bytes += len(entry) - } - } - return p.SendNodeData(data) +func (h *handler) removePeer(id string) { + // Remove the eth peer if it exists + eth := h.peers.ethPeer(id) + if eth != nil { + log.Debug("Removing Ethereum peer", "peer", id) + h.downloader.UnregisterPeer(id) + h.txFetcher.Drop(id) - case p.version >= eth63 && msg.Code == NodeDataMsg: - // A batch of node state data arrived to one of our previous requests - var data [][]byte - if err := msg.Decode(&data); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) + if err := h.peers.unregisterEthPeer(id); err != nil { + log.Error("Peer removal failed", "peer", id, "err", err) } - // Deliver all to the downloader - if err := pm.downloader.DeliverNodeData(p.id, data); err != nil { - log.Debug("Failed to deliver node state data", "err", err) + } + // Remove the snap peer if it exists + snap := h.peers.snapPeer(id) + if snap != nil { + log.Debug("Removing Snapshot peer", "peer", id) + h.downloader.SnapSyncer.Unregister(id) + if err := h.peers.unregisterSnapPeer(id); err != nil { + log.Error("Peer removal failed", "peer", id, "err", err) } + } + // Hard disconnect at the networking layer + if eth != nil { + eth.Peer.Disconnect(p2p.DiscUselessPeer) + } + if snap != nil { + snap.Peer.Disconnect(p2p.DiscUselessPeer) + } +} - case p.version >= eth63 && msg.Code == GetReceiptsMsg: - // Decode the retrieval message - msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) - if _, err := msgStream.List(); err != nil { - return err - } - // Gather state data until the fetch or network limits is reached - var ( - hash common.Hash - bytes int - receipts []rlp.RawValue - ) - for bytes < softResponseLimit && len(receipts) < downloader.MaxReceiptFetch { - // Retrieve the hash of the next block - if err := msgStream.Decode(&hash); err == rlp.EOL { - break - } else if err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Retrieve the requested block's receipts, skipping if unknown to us - results := pm.blockchain.GetReceiptsByHash(hash) - if results == nil { - if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - continue - } - } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(results); err != nil { - log.Error("Failed to encode receipt", "err", err) - } else { - receipts = append(receipts, encoded) - bytes += len(encoded) - } - } - return p.SendReceiptsRLP(receipts) +func (h *handler) Start(maxPeers int) { + h.maxPeers = maxPeers - case p.version >= eth63 && msg.Code == ReceiptsMsg: - // A batch of receipts arrived to one of our previous requests - var receipts [][]*types.Receipt - if err := msg.Decode(&receipts); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Deliver all to the downloader - if err := pm.downloader.DeliverReceipts(p.id, receipts); err != nil { - log.Debug("Failed to deliver receipts", "err", err) - } + // broadcast transactions + h.wg.Add(1) + h.txsCh = make(chan core.NewTxsEvent, txChanSize) + h.txsSub = h.txpool.SubscribeNewTxsEvent(h.txsCh) + go h.txBroadcastLoop() - case msg.Code == NewBlockHashesMsg: - var announces newBlockHashesData - if err := msg.Decode(&announces); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - // Mark the hashes as present at the remote node - for _, block := range announces { - p.MarkBlock(block.Hash) - } - // Schedule all the unknown hashes for retrieval - unknown := make(newBlockHashesData, 0, len(announces)) - for _, block := range announces { - if !pm.blockchain.HasBlock(block.Hash, block.Number) { - unknown = append(unknown, block) - } - } - for _, block := range unknown { - pm.blockFetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) - } + // broadcast mined blocks + h.wg.Add(1) + h.minedBlockSub = h.eventMux.Subscribe(core.NewMinedBlockEvent{}) + go h.minedBroadcastLoop() - case msg.Code == NewBlockMsg: - // Retrieve and decode the propagated block - var request newBlockData - if err := msg.Decode(&request); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - if hash := types.CalcUncleHash(request.Block.Uncles()); hash != request.Block.UncleHash() { - log.Warn("Propagated block has invalid uncles", "have", hash, "exp", request.Block.UncleHash()) - break // TODO(karalabe): return error eventually, but wait a few releases - } - if hash := types.DeriveSha(request.Block.Transactions(), trie.NewStackTrie(nil)); hash != request.Block.TxHash() { - log.Warn("Propagated block has invalid body", "have", hash, "exp", request.Block.TxHash()) - break // TODO(karalabe): return error eventually, but wait a few releases - } - if err := request.sanityCheck(); err != nil { - return err - } - request.Block.ReceivedAt = msg.ReceivedAt - request.Block.ReceivedFrom = p - - // Mark the peer as owning the block and schedule it for import - p.MarkBlock(request.Block.Hash()) - pm.blockFetcher.Enqueue(p.id, request.Block) - - // Assuming the block is importable by the peer, but possibly not yet done so, - // calculate the head hash and TD that the peer truly must have. - var ( - trueHead = request.Block.ParentHash() - trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty()) - ) - // Update the peer's total difficulty if better than the previous - if _, td := p.Head(); trueTD.Cmp(td) > 0 { - p.SetHead(trueHead, trueTD) - pm.chainSync.handlePeerEvent(p) - } + // start sync handlers + h.wg.Add(2) + go h.chainSync.loop() + go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. +} - case msg.Code == NewPooledTransactionHashesMsg && p.version >= eth65: - // New transaction announcement arrived, make sure we have - // a valid and fresh chain to handle them - if atomic.LoadUint32(&pm.acceptTxs) == 0 { - break - } - var hashes []common.Hash - if err := msg.Decode(&hashes); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Schedule all the unknown hashes for retrieval - for _, hash := range hashes { - p.MarkTransaction(hash) - } - pm.txFetcher.Notify(p.id, hashes) +func (h *handler) Stop() { + h.txsSub.Unsubscribe() // quits txBroadcastLoop + h.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop - case msg.Code == GetPooledTransactionsMsg && p.version >= eth65: - // Decode the retrieval message - msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) - if _, err := msgStream.List(); err != nil { - return err - } - // Gather transactions until the fetch or network limits is reached - var ( - hash common.Hash - bytes int - hashes []common.Hash - txs []rlp.RawValue - ) - for bytes < softResponseLimit { - // Retrieve the hash of the next block - if err := msgStream.Decode(&hash); err == rlp.EOL { - break - } else if err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Retrieve the requested transaction, skipping if unknown to us - tx := pm.txpool.Get(hash) - if tx == nil { - continue - } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(tx); err != nil { - log.Error("Failed to encode transaction", "err", err) - } else { - hashes = append(hashes, hash) - txs = append(txs, encoded) - bytes += len(encoded) - } - } - return p.SendPooledTransactionsRLP(hashes, txs) + // Quit chainSync and txsync64. + // After this is done, no new peers will be accepted. + close(h.quitSync) + h.wg.Wait() - case msg.Code == TransactionMsg || (msg.Code == PooledTransactionsMsg && p.version >= eth65): - // Transactions arrived, make sure we have a valid and fresh chain to handle them - if atomic.LoadUint32(&pm.acceptTxs) == 0 { - break - } - // Transactions can be processed, parse all of them and deliver to the pool - var txs []*types.Transaction - if err := msg.Decode(&txs); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return errResp(ErrDecode, "transaction %d is nil", i) - } - p.MarkTransaction(tx.Hash()) - } - pm.txFetcher.Enqueue(p.id, txs, msg.Code == PooledTransactionsMsg) + // Disconnect existing sessions. + // This also closes the gate for any new registrations on the peer set. + // sessions which are already established but not added to h.peers yet + // will exit when they try to register. + h.peers.close() + h.peerWG.Wait() - default: - return errResp(ErrInvalidMsgCode, "%v", msg.Code) - } - return nil + log.Info("Ethereum protocol stopped") } // BroadcastBlock will either propagate a block to a subset of its peers, or // will only announce its availability (depending what's requested). -func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { +func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { hash := block.Hash() - peers := pm.peers.PeersWithoutBlock(hash) + peers := h.peers.ethPeersWithoutBlock(hash) // If propagation is requested, send to a subset of the peer if propagate { // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) var td *big.Int - if parent := pm.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { - td = new(big.Int).Add(block.Difficulty(), pm.blockchain.GetTd(block.ParentHash(), block.NumberU64()-1)) + if parent := h.chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { + td = new(big.Int).Add(block.Difficulty(), h.chain.GetTd(block.ParentHash(), block.NumberU64()-1)) } else { log.Error("Propagating dangling block", "number", block.Number(), "hash", hash) return @@ -843,7 +419,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { return } // Otherwise if the block is indeed in out own chain, announce it - if pm.blockchain.HasBlock(hash, block.NumberU64()) { + if h.chain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { peer.AsyncSendNewBlockHash(block) } @@ -853,15 +429,15 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // BroadcastTransactions will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propagate bool) { +func (h *handler) BroadcastTransactions(txs types.Transactions, propagate bool) { var ( - txset = make(map[*peer][]common.Hash) - annos = make(map[*peer][]common.Hash) + txset = make(map[*ethPeer][]common.Hash) + annos = make(map[*ethPeer][]common.Hash) ) // Broadcast transactions to a batch of peers not knowing about it if propagate { for _, tx := range txs { - peers := pm.peers.PeersWithoutTx(tx.Hash()) + peers := h.peers.ethPeersWithoutTransacion(tx.Hash()) // Send the block to a subset of our peers transfer := peers[:int(math.Sqrt(float64(len(peers))))] @@ -877,13 +453,13 @@ func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propaga } // Otherwise only broadcast the announcement to peers for _, tx := range txs { - peers := pm.peers.PeersWithoutTx(tx.Hash()) + peers := h.peers.ethPeersWithoutTransacion(tx.Hash()) for _, peer := range peers { annos[peer] = append(annos[peer], tx.Hash()) } } for peer, hashes := range annos { - if peer.version >= eth65 { + if peer.Version() >= eth.ETH65 { peer.AsyncSendPooledTransactionHashes(hashes) } else { peer.AsyncSendTransactions(hashes) @@ -892,56 +468,29 @@ func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propaga } // minedBroadcastLoop sends mined blocks to connected peers. -func (pm *ProtocolManager) minedBroadcastLoop() { - defer pm.wg.Done() +func (h *handler) minedBroadcastLoop() { + defer h.wg.Done() - for obj := range pm.minedBlockSub.Chan() { + for obj := range h.minedBlockSub.Chan() { if ev, ok := obj.Data.(core.NewMinedBlockEvent); ok { - pm.BroadcastBlock(ev.Block, true) // First propagate block to peers - pm.BroadcastBlock(ev.Block, false) // Only then announce to the rest + h.BroadcastBlock(ev.Block, true) // First propagate block to peers + h.BroadcastBlock(ev.Block, false) // Only then announce to the rest } } } // txBroadcastLoop announces new transactions to connected peers. -func (pm *ProtocolManager) txBroadcastLoop() { - defer pm.wg.Done() +func (h *handler) txBroadcastLoop() { + defer h.wg.Done() for { select { - case event := <-pm.txsCh: - // For testing purpose only, disable propagation - if pm.broadcastTxAnnouncesOnly { - pm.BroadcastTransactions(event.Txs, false) - continue - } - pm.BroadcastTransactions(event.Txs, true) // First propagate transactions to peers - pm.BroadcastTransactions(event.Txs, false) // Only then announce to the rest + case event := <-h.txsCh: + h.BroadcastTransactions(event.Txs, true) // First propagate transactions to peers + h.BroadcastTransactions(event.Txs, false) // Only then announce to the rest - case <-pm.txsSub.Err(): + case <-h.txsSub.Err(): return } } } - -// NodeInfo represents a short summary of the Ethereum sub-protocol metadata -// known about the host peer. -type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) - Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain - Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block - Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules - Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block -} - -// NodeInfo retrieves some protocol metadata about the running host node. -func (pm *ProtocolManager) NodeInfo() *NodeInfo { - currentBlock := pm.blockchain.CurrentBlock() - return &NodeInfo{ - Network: pm.networkID, - Difficulty: pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), - Genesis: pm.blockchain.Genesis().Hash(), - Config: pm.blockchain.Config(), - Head: currentBlock.Hash(), - } -} diff --git a/eth/handler_eth.go b/eth/handler_eth.go new file mode 100644 index 0000000000..84bdac659a --- /dev/null +++ b/eth/handler_eth.go @@ -0,0 +1,218 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "errors" + "fmt" + "math/big" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/trie" +) + +// ethHandler implements the eth.Backend interface to handle the various network +// packets that are sent as replies or broadcasts. +type ethHandler handler + +func (h *ethHandler) Chain() *core.BlockChain { return h.chain } +func (h *ethHandler) StateBloom() *trie.SyncBloom { return h.stateBloom } +func (h *ethHandler) TxPool() eth.TxPool { return h.txpool } + +// RunPeer is invoked when a peer joins on the `eth` protocol. +func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error { + return (*handler)(h).runEthPeer(peer, hand) +} + +// PeerInfo retrieves all known `eth` information about a peer. +func (h *ethHandler) PeerInfo(id enode.ID) interface{} { + if p := h.peers.ethPeer(id.String()); p != nil { + return p.info() + } + return nil +} + +// AcceptTxs retrieves whether transaction processing is enabled on the node +// or if inbound transactions should simply be dropped. +func (h *ethHandler) AcceptTxs() bool { + return atomic.LoadUint32(&h.acceptTxs) == 1 +} + +// Handle is invoked from a peer's message handler when it receives a new remote +// message that the handler couldn't consume and serve itself. +func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { + // Consume any broadcasts and announces, forwarding the rest to the downloader + switch packet := packet.(type) { + case *eth.BlockHeadersPacket: + return h.handleHeaders(peer, *packet) + + case *eth.BlockBodiesPacket: + txset, uncleset := packet.Unpack() + return h.handleBodies(peer, txset, uncleset) + + case *eth.NodeDataPacket: + if err := h.downloader.DeliverNodeData(peer.ID(), *packet); err != nil { + log.Debug("Failed to deliver node state data", "err", err) + } + return nil + + case *eth.ReceiptsPacket: + if err := h.downloader.DeliverReceipts(peer.ID(), *packet); err != nil { + log.Debug("Failed to deliver receipts", "err", err) + } + return nil + + case *eth.NewBlockHashesPacket: + hashes, numbers := packet.Unpack() + return h.handleBlockAnnounces(peer, hashes, numbers) + + case *eth.NewBlockPacket: + return h.handleBlockBroadcast(peer, packet.Block, packet.TD) + + case *eth.NewPooledTransactionHashesPacket: + return h.txFetcher.Notify(peer.ID(), *packet) + + case *eth.TransactionsPacket: + return h.txFetcher.Enqueue(peer.ID(), *packet, false) + + case *eth.PooledTransactionsPacket: + return h.txFetcher.Enqueue(peer.ID(), *packet, true) + + default: + return fmt.Errorf("unexpected eth packet type: %T", packet) + } +} + +// handleHeaders is invoked from a peer's message handler when it transmits a batch +// of headers for the local node to process. +func (h *ethHandler) handleHeaders(peer *eth.Peer, headers []*types.Header) error { + p := h.peers.ethPeer(peer.ID()) + if p == nil { + return errors.New("unregistered during callback") + } + // If no headers were received, but we're expencting a checkpoint header, consider it that + if len(headers) == 0 && p.syncDrop != nil { + // Stop the timer either way, decide later to drop or not + p.syncDrop.Stop() + p.syncDrop = nil + + // If we're doing a fast (or snap) sync, we must enforce the checkpoint block to avoid + // eclipse attacks. Unsynced nodes are welcome to connect after we're done + // joining the network + if atomic.LoadUint32(&h.fastSync) == 1 { + peer.Log().Warn("Dropping unsynced node during sync", "addr", peer.RemoteAddr(), "type", peer.Name()) + return errors.New("unsynced node cannot serve sync") + } + } + // Filter out any explicitly requested headers, deliver the rest to the downloader + filter := len(headers) == 1 + if filter { + // If it's a potential sync progress check, validate the content and advertised chain weight + if p.syncDrop != nil && headers[0].Number.Uint64() == h.checkpointNumber { + // Disable the sync drop timer + p.syncDrop.Stop() + p.syncDrop = nil + + // Validate the header and either drop the peer or continue + if headers[0].Hash() != h.checkpointHash { + return errors.New("checkpoint hash mismatch") + } + return nil + } + // Otherwise if it's a whitelisted block, validate against the set + if want, ok := h.whitelist[headers[0].Number.Uint64()]; ok { + if hash := headers[0].Hash(); want != hash { + peer.Log().Info("Whitelist mismatch, dropping peer", "number", headers[0].Number.Uint64(), "hash", hash, "want", want) + return errors.New("whitelist block mismatch") + } + peer.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", want) + } + // Irrelevant of the fork checks, send the header to the fetcher just in case + headers = h.blockFetcher.FilterHeaders(peer.ID(), headers, time.Now()) + } + if len(headers) > 0 || !filter { + err := h.downloader.DeliverHeaders(peer.ID(), headers) + if err != nil { + log.Debug("Failed to deliver headers", "err", err) + } + } + return nil +} + +// handleBodies is invoked from a peer's message handler when it transmits a batch +// of block bodies for the local node to process. +func (h *ethHandler) handleBodies(peer *eth.Peer, txs [][]*types.Transaction, uncles [][]*types.Header) error { + // Filter out any explicitly requested bodies, deliver the rest to the downloader + filter := len(txs) > 0 || len(uncles) > 0 + if filter { + txs, uncles = h.blockFetcher.FilterBodies(peer.ID(), txs, uncles, time.Now()) + } + if len(txs) > 0 || len(uncles) > 0 || !filter { + err := h.downloader.DeliverBodies(peer.ID(), txs, uncles) + if err != nil { + log.Debug("Failed to deliver bodies", "err", err) + } + } + return nil +} + +// handleBlockAnnounces is invoked from a peer's message handler when it transmits a +// batch of block announcements for the local node to process. +func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, numbers []uint64) error { + // Schedule all the unknown hashes for retrieval + var ( + unknownHashes = make([]common.Hash, 0, len(hashes)) + unknownNumbers = make([]uint64, 0, len(numbers)) + ) + for i := 0; i < len(hashes); i++ { + if !h.chain.HasBlock(hashes[i], numbers[i]) { + unknownHashes = append(unknownHashes, hashes[i]) + unknownNumbers = append(unknownNumbers, numbers[i]) + } + } + for i := 0; i < len(unknownHashes); i++ { + h.blockFetcher.Notify(peer.ID(), unknownHashes[i], unknownNumbers[i], time.Now(), peer.RequestOneHeader, peer.RequestBodies) + } + return nil +} + +// handleBlockBroadcast is invoked from a peer's message handler when it transmits a +// block broadcast for the local node to process. +func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td *big.Int) error { + // Schedule the block for import + h.blockFetcher.Enqueue(peer.ID(), block) + + // Assuming the block is importable by the peer, but possibly not yet done so, + // calculate the head hash and TD that the peer truly must have. + var ( + trueHead = block.ParentHash() + trueTD = new(big.Int).Sub(td, block.Difficulty()) + ) + // Update the peer's total difficulty if better than the previous + if _, td := peer.Head(); trueTD.Cmp(td) > 0 { + peer.SetHead(trueHead, trueTD) + h.chainSync.handlePeerEvent(peer) + } + return nil +} diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go new file mode 100644 index 0000000000..0e5c0c90ee --- /dev/null +++ b/eth/handler_eth_test.go @@ -0,0 +1,740 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "fmt" + "math/big" + "math/rand" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +// testEthHandler is a mock event handler to listen for inbound network requests +// on the `eth` protocol and convert them into a more easily testable form. +type testEthHandler struct { + blockBroadcasts event.Feed + txAnnounces event.Feed + txBroadcasts event.Feed +} + +func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") } +func (h *testEthHandler) StateBloom() *trie.SyncBloom { panic("no backing state bloom") } +func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") } +func (h *testEthHandler) AcceptTxs() bool { return true } +func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") } +func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") } + +func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { + switch packet := packet.(type) { + case *eth.NewBlockPacket: + h.blockBroadcasts.Send(packet.Block) + return nil + + case *eth.NewPooledTransactionHashesPacket: + h.txAnnounces.Send(([]common.Hash)(*packet)) + return nil + + case *eth.TransactionsPacket: + h.txBroadcasts.Send(([]*types.Transaction)(*packet)) + return nil + + case *eth.PooledTransactionsPacket: + h.txBroadcasts.Send(([]*types.Transaction)(*packet)) + return nil + + default: + panic(fmt.Sprintf("unexpected eth packet type in tests: %T", packet)) + } +} + +// Tests that peers are correctly accepted (or rejected) based on the advertised +// fork IDs in the protocol handshake. +func TestForkIDSplit64(t *testing.T) { testForkIDSplit(t, 64) } +func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, 65) } + +func testForkIDSplit(t *testing.T, protocol uint) { + t.Parallel() + + var ( + engine = ethash.NewFaker() + + configNoFork = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(1)} + configProFork = ¶ms.ChainConfig{ + HomesteadBlock: big.NewInt(1), + EIP150Block: big.NewInt(2), + EIP155Block: big.NewInt(2), + EIP158Block: big.NewInt(2), + ByzantiumBlock: big.NewInt(3), + } + dbNoFork = rawdb.NewMemoryDatabase() + dbProFork = rawdb.NewMemoryDatabase() + + gspecNoFork = &core.Genesis{Config: configNoFork} + gspecProFork = &core.Genesis{Config: configProFork} + + genesisNoFork = gspecNoFork.MustCommit(dbNoFork) + genesisProFork = gspecProFork.MustCommit(dbProFork) + + chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil) + chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil) + + blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) + blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) + + ethNoFork, _ = newHandler(&handlerConfig{ + Database: dbNoFork, + Chain: chainNoFork, + TxPool: newTestTxPool(), + Network: 1, + Sync: downloader.FullSync, + BloomCache: 1, + }) + ethProFork, _ = newHandler(&handlerConfig{ + Database: dbProFork, + Chain: chainProFork, + TxPool: newTestTxPool(), + Network: 1, + Sync: downloader.FullSync, + BloomCache: 1, + }) + ) + ethNoFork.Start(1000) + ethProFork.Start(1000) + + // Clean up everything after ourselves + defer chainNoFork.Stop() + defer chainProFork.Stop() + + defer ethNoFork.Stop() + defer ethProFork.Stop() + + // Both nodes should allow the other to connect (same genesis, next fork is the same) + p2pNoFork, p2pProFork := p2p.MsgPipe() + defer p2pNoFork.Close() + defer p2pProFork.Close() + + peerNoFork := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) + defer peerNoFork.Close() + defer peerProFork.Close() + + errc := make(chan error, 2) + go func(errc chan error) { + errc <- ethNoFork.runEthPeer(peerProFork, func(peer *eth.Peer) error { return nil }) + }(errc) + go func(errc chan error) { + errc <- ethProFork.runEthPeer(peerNoFork, func(peer *eth.Peer) error { return nil }) + }(errc) + + for i := 0; i < 2; i++ { + select { + case err := <-errc: + if err != nil { + t.Fatalf("frontier nofork <-> profork failed: %v", err) + } + case <-time.After(250 * time.Millisecond): + t.Fatalf("frontier nofork <-> profork handler timeout") + } + } + // Progress into Homestead. Fork's match, so we don't care what the future holds + chainNoFork.InsertChain(blocksNoFork[:1]) + chainProFork.InsertChain(blocksProFork[:1]) + + p2pNoFork, p2pProFork = p2p.MsgPipe() + defer p2pNoFork.Close() + defer p2pProFork.Close() + + peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) + defer peerNoFork.Close() + defer peerProFork.Close() + + errc = make(chan error, 2) + go func(errc chan error) { + errc <- ethNoFork.runEthPeer(peerProFork, func(peer *eth.Peer) error { return nil }) + }(errc) + go func(errc chan error) { + errc <- ethProFork.runEthPeer(peerNoFork, func(peer *eth.Peer) error { return nil }) + }(errc) + + for i := 0; i < 2; i++ { + select { + case err := <-errc: + if err != nil { + t.Fatalf("homestead nofork <-> profork failed: %v", err) + } + case <-time.After(250 * time.Millisecond): + t.Fatalf("homestead nofork <-> profork handler timeout") + } + } + // Progress into Spurious. Forks mismatch, signalling differing chains, reject + chainNoFork.InsertChain(blocksNoFork[1:2]) + chainProFork.InsertChain(blocksProFork[1:2]) + + p2pNoFork, p2pProFork = p2p.MsgPipe() + defer p2pNoFork.Close() + defer p2pProFork.Close() + + peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) + defer peerNoFork.Close() + defer peerProFork.Close() + + errc = make(chan error, 2) + go func(errc chan error) { + errc <- ethNoFork.runEthPeer(peerProFork, func(peer *eth.Peer) error { return nil }) + }(errc) + go func(errc chan error) { + errc <- ethProFork.runEthPeer(peerNoFork, func(peer *eth.Peer) error { return nil }) + }(errc) + + var successes int + for i := 0; i < 2; i++ { + select { + case err := <-errc: + if err == nil { + successes++ + if successes == 2 { // Only one side disconnects + t.Fatalf("fork ID rejection didn't happen") + } + } + case <-time.After(250 * time.Millisecond): + t.Fatalf("split peers not rejected") + } + } +} + +// Tests that received transactions are added to the local pool. +func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) } +func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, 65) } + +func testRecvTransactions(t *testing.T, protocol uint) { + t.Parallel() + + // Create a message handler, configure it to accept transactions and watch them + handler := newTestHandler() + defer handler.close() + + handler.handler.acceptTxs = 1 // mark synced to accept transactions + + txs := make(chan core.NewTxsEvent) + sub := handler.txpool.SubscribeNewTxsEvent(txs) + defer sub.Unsubscribe() + + // Create a source peer to send messages through and a sink handler to receive them + p2pSrc, p2pSink := p2p.MsgPipe() + defer p2pSrc.Close() + defer p2pSink.Close() + + src := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pSrc, handler.txpool) + sink := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pSink, handler.txpool) + defer src.Close() + defer sink.Close() + + go handler.handler.runEthPeer(sink, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(handler.handler), peer) + }) + // Run the handshake locally to avoid spinning up a source handler + var ( + genesis = handler.chain.Genesis() + head = handler.chain.CurrentBlock() + td = handler.chain.GetTd(head.Hash(), head.NumberU64()) + ) + if err := src.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { + t.Fatalf("failed to run protocol handshake") + } + // Send the transaction to the sink and verify that it's added to the tx pool + tx := types.NewTransaction(0, common.Address{}, big.NewInt(0), 100000, big.NewInt(0), nil) + tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + + if err := src.SendTransactions([]*types.Transaction{tx}); err != nil { + t.Fatalf("failed to send transaction: %v", err) + } + select { + case event := <-txs: + if len(event.Txs) != 1 { + t.Errorf("wrong number of added transactions: got %d, want 1", len(event.Txs)) + } else if event.Txs[0].Hash() != tx.Hash() { + t.Errorf("added wrong tx hash: got %v, want %v", event.Txs[0].Hash(), tx.Hash()) + } + case <-time.After(2 * time.Second): + t.Errorf("no NewTxsEvent received within 2 seconds") + } +} + +// This test checks that pending transactions are sent. +func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) } +func TestSendTransactions65(t *testing.T) { testSendTransactions(t, 65) } + +func testSendTransactions(t *testing.T, protocol uint) { + t.Parallel() + + // Create a message handler and fill the pool with big transactions + handler := newTestHandler() + defer handler.close() + + insert := make([]*types.Transaction, 100) + for nonce := range insert { + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) + tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + + insert[nonce] = tx + } + go handler.txpool.AddRemotes(insert) // Need goroutine to not block on feed + time.Sleep(250 * time.Millisecond) // Wait until tx events get out of the system (can't use events, tx broadcaster races with peer join) + + // Create a source handler to send messages through and a sink peer to receive them + p2pSrc, p2pSink := p2p.MsgPipe() + defer p2pSrc.Close() + defer p2pSink.Close() + + src := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pSrc, handler.txpool) + sink := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pSink, handler.txpool) + defer src.Close() + defer sink.Close() + + go handler.handler.runEthPeer(src, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(handler.handler), peer) + }) + // Run the handshake locally to avoid spinning up a source handler + var ( + genesis = handler.chain.Genesis() + head = handler.chain.CurrentBlock() + td = handler.chain.GetTd(head.Hash(), head.NumberU64()) + ) + if err := sink.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { + t.Fatalf("failed to run protocol handshake") + } + // After the handshake completes, the source handler should stream the sink + // the transactions, subscribe to all inbound network events + backend := new(testEthHandler) + + anns := make(chan []common.Hash) + annSub := backend.txAnnounces.Subscribe(anns) + defer annSub.Unsubscribe() + + bcasts := make(chan []*types.Transaction) + bcastSub := backend.txBroadcasts.Subscribe(bcasts) + defer bcastSub.Unsubscribe() + + go eth.Handle(backend, sink) + + // Make sure we get all the transactions on the correct channels + seen := make(map[common.Hash]struct{}) + for len(seen) < len(insert) { + switch protocol { + case 63, 64: + select { + case <-anns: + t.Errorf("tx announce received on pre eth/65") + case txs := <-bcasts: + for _, tx := range txs { + if _, ok := seen[tx.Hash()]; ok { + t.Errorf("duplicate transaction announced: %x", tx.Hash()) + } + seen[tx.Hash()] = struct{}{} + } + } + case 65: + select { + case hashes := <-anns: + for _, hash := range hashes { + if _, ok := seen[hash]; ok { + t.Errorf("duplicate transaction announced: %x", hash) + } + seen[hash] = struct{}{} + } + case <-bcasts: + t.Errorf("initial tx broadcast received on post eth/65") + } + + default: + panic("unsupported protocol, please extend test") + } + } + for _, tx := range insert { + if _, ok := seen[tx.Hash()]; !ok { + t.Errorf("missing transaction: %x", tx.Hash()) + } + } +} + +// Tests that transactions get propagated to all attached peers, either via direct +// broadcasts or via announcements/retrievals. +func TestTransactionPropagation64(t *testing.T) { testTransactionPropagation(t, 64) } +func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, 65) } + +func testTransactionPropagation(t *testing.T, protocol uint) { + t.Parallel() + + // Create a source handler to send transactions from and a number of sinks + // to receive them. We need multiple sinks since a one-to-one peering would + // broadcast all transactions without announcement. + source := newTestHandler() + defer source.close() + + sinks := make([]*testHandler, 10) + for i := 0; i < len(sinks); i++ { + sinks[i] = newTestHandler() + defer sinks[i].close() + + sinks[i].handler.acceptTxs = 1 // mark synced to accept transactions + } + // Interconnect all the sink handlers with the source handler + for i, sink := range sinks { + sink := sink // Closure for gorotuine below + + sourcePipe, sinkPipe := p2p.MsgPipe() + defer sourcePipe.Close() + defer sinkPipe.Close() + + sourcePeer := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{byte(i)}, "", nil), sourcePipe, source.txpool) + sinkPeer := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, sink.txpool) + defer sourcePeer.Close() + defer sinkPeer.Close() + + go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(source.handler), peer) + }) + go sink.handler.runEthPeer(sinkPeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(sink.handler), peer) + }) + } + // Subscribe to all the transaction pools + txChs := make([]chan core.NewTxsEvent, len(sinks)) + for i := 0; i < len(sinks); i++ { + txChs[i] = make(chan core.NewTxsEvent, 1024) + + sub := sinks[i].txpool.SubscribeNewTxsEvent(txChs[i]) + defer sub.Unsubscribe() + } + // Fill the source pool with transactions and wait for them at the sinks + txs := make([]*types.Transaction, 1024) + for nonce := range txs { + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), nil) + tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + + txs[nonce] = tx + } + source.txpool.AddRemotes(txs) + + // Iterate through all the sinks and ensure they all got the transactions + for i := range sinks { + for arrived := 0; arrived < len(txs); { + select { + case event := <-txChs[i]: + arrived += len(event.Txs) + case <-time.NewTimer(time.Second).C: + t.Errorf("sink %d: transaction propagation timed out: have %d, want %d", i, arrived, len(txs)) + } + } + } +} + +// Tests that post eth protocol handshake, clients perform a mutual checkpoint +// challenge to validate each other's chains. Hash mismatches, or missing ones +// during a fast sync should lead to the peer getting dropped. +func TestCheckpointChallenge(t *testing.T) { + tests := []struct { + syncmode downloader.SyncMode + checkpoint bool + timeout bool + empty bool + match bool + drop bool + }{ + // If checkpointing is not enabled locally, don't challenge and don't drop + {downloader.FullSync, false, false, false, false, false}, + {downloader.FastSync, false, false, false, false, false}, + + // If checkpointing is enabled locally and remote response is empty, only drop during fast sync + {downloader.FullSync, true, false, true, false, false}, + {downloader.FastSync, true, false, true, false, true}, // Special case, fast sync, unsynced peer + + // If checkpointing is enabled locally and remote response mismatches, always drop + {downloader.FullSync, true, false, false, false, true}, + {downloader.FastSync, true, false, false, false, true}, + + // If checkpointing is enabled locally and remote response matches, never drop + {downloader.FullSync, true, false, false, true, false}, + {downloader.FastSync, true, false, false, true, false}, + + // If checkpointing is enabled locally and remote times out, always drop + {downloader.FullSync, true, true, false, true, true}, + {downloader.FastSync, true, true, false, true, true}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("sync %v checkpoint %v timeout %v empty %v match %v", tt.syncmode, tt.checkpoint, tt.timeout, tt.empty, tt.match), func(t *testing.T) { + testCheckpointChallenge(t, tt.syncmode, tt.checkpoint, tt.timeout, tt.empty, tt.match, tt.drop) + }) + } +} + +func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpoint bool, timeout bool, empty bool, match bool, drop bool) { + // Reduce the checkpoint handshake challenge timeout + defer func(old time.Duration) { syncChallengeTimeout = old }(syncChallengeTimeout) + syncChallengeTimeout = 250 * time.Millisecond + + // Create a test handler and inject a CHT into it. The injection is a bit + // ugly, but it beats creating everything manually just to avoid reaching + // into the internals a bit. + handler := newTestHandler() + defer handler.close() + + if syncmode == downloader.FastSync { + atomic.StoreUint32(&handler.handler.fastSync, 1) + } else { + atomic.StoreUint32(&handler.handler.fastSync, 0) + } + var response *types.Header + if checkpoint { + number := (uint64(rand.Intn(500))+1)*params.CHTFrequency - 1 + response = &types.Header{Number: big.NewInt(int64(number)), Extra: []byte("valid")} + + handler.handler.checkpointNumber = number + handler.handler.checkpointHash = response.Hash() + } + // Create a challenger peer and a challenged one + p2pLocal, p2pRemote := p2p.MsgPipe() + defer p2pLocal.Close() + defer p2pRemote.Close() + + local := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{1}, "", nil), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{2}, "", nil), p2pRemote, handler.txpool) + defer local.Close() + defer remote.Close() + + go handler.handler.runEthPeer(local, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(handler.handler), peer) + }) + // Run the handshake locally to avoid spinning up a remote handler + var ( + genesis = handler.chain.Genesis() + head = handler.chain.CurrentBlock() + td = handler.chain.GetTd(head.Hash(), head.NumberU64()) + ) + if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { + t.Fatalf("failed to run protocol handshake") + } + // Connect a new peer and check that we receive the checkpoint challenge + if checkpoint { + if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { + t.Fatalf("challenge mismatch: %v", err) + } + // Create a block to reply to the challenge if no timeout is simulated + if !timeout { + if empty { + if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { + t.Fatalf("failed to answer challenge: %v", err) + } + } else if match { + if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { + t.Fatalf("failed to answer challenge: %v", err) + } + } else { + if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { + t.Fatalf("failed to answer challenge: %v", err) + } + } + } + } + // Wait until the test timeout passes to ensure proper cleanup + time.Sleep(syncChallengeTimeout + 300*time.Millisecond) + + // Verify that the remote peer is maintained or dropped + if drop { + if peers := handler.handler.peers.Len(); peers != 0 { + t.Fatalf("peer count mismatch: have %d, want %d", peers, 0) + } + } else { + if peers := handler.handler.peers.Len(); peers != 1 { + t.Fatalf("peer count mismatch: have %d, want %d", peers, 1) + } + } +} + +// Tests that blocks are broadcast to a sqrt number of peers only. +func TestBroadcastBlock1Peer(t *testing.T) { testBroadcastBlock(t, 1, 1) } +func TestBroadcastBlock2Peers(t *testing.T) { testBroadcastBlock(t, 2, 1) } +func TestBroadcastBlock3Peers(t *testing.T) { testBroadcastBlock(t, 3, 1) } +func TestBroadcastBlock4Peers(t *testing.T) { testBroadcastBlock(t, 4, 2) } +func TestBroadcastBlock5Peers(t *testing.T) { testBroadcastBlock(t, 5, 2) } +func TestBroadcastBlock8Peers(t *testing.T) { testBroadcastBlock(t, 9, 3) } +func TestBroadcastBlock12Peers(t *testing.T) { testBroadcastBlock(t, 12, 3) } +func TestBroadcastBlock16Peers(t *testing.T) { testBroadcastBlock(t, 16, 4) } +func TestBroadcastBloc26Peers(t *testing.T) { testBroadcastBlock(t, 26, 5) } +func TestBroadcastBlock100Peers(t *testing.T) { testBroadcastBlock(t, 100, 10) } + +func testBroadcastBlock(t *testing.T, peers, bcasts int) { + t.Parallel() + + // Create a source handler to broadcast blocks from and a number of sinks + // to receive them. + source := newTestHandlerWithBlocks(1) + defer source.close() + + sinks := make([]*testEthHandler, peers) + for i := 0; i < len(sinks); i++ { + sinks[i] = new(testEthHandler) + } + // Interconnect all the sink handlers with the source handler + var ( + genesis = source.chain.Genesis() + td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) + ) + for i, sink := range sinks { + sink := sink // Closure for gorotuine below + + sourcePipe, sinkPipe := p2p.MsgPipe() + defer sourcePipe.Close() + defer sinkPipe.Close() + + sourcePeer := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{byte(i)}, "", nil), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, nil) + defer sourcePeer.Close() + defer sinkPeer.Close() + + go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(source.handler), peer) + }) + if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { + t.Fatalf("failed to run protocol handshake") + } + go eth.Handle(sink, sinkPeer) + } + // Subscribe to all the transaction pools + blockChs := make([]chan *types.Block, len(sinks)) + for i := 0; i < len(sinks); i++ { + blockChs[i] = make(chan *types.Block, 1) + defer close(blockChs[i]) + + sub := sinks[i].blockBroadcasts.Subscribe(blockChs[i]) + defer sub.Unsubscribe() + } + // Initiate a block propagation across the peers + time.Sleep(100 * time.Millisecond) + source.handler.BroadcastBlock(source.chain.CurrentBlock(), true) + + // Iterate through all the sinks and ensure the correct number got the block + done := make(chan struct{}, peers) + for _, ch := range blockChs { + ch := ch + go func() { + <-ch + done <- struct{}{} + }() + } + var received int + for { + select { + case <-done: + received++ + + case <-time.After(100 * time.Millisecond): + if received != bcasts { + t.Errorf("broadcast count mismatch: have %d, want %d", received, bcasts) + } + return + } + } +} + +// Tests that a propagated malformed block (uncles or transactions don't match +// with the hashes in the header) gets discarded and not broadcast forward. +func TestBroadcastMalformedBlock64(t *testing.T) { testBroadcastMalformedBlock(t, 64) } +func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, 65) } + +func testBroadcastMalformedBlock(t *testing.T, protocol uint) { + t.Parallel() + + // Create a source handler to broadcast blocks from and a number of sinks + // to receive them. + source := newTestHandlerWithBlocks(1) + defer source.close() + + // Create a source handler to send messages through and a sink peer to receive them + p2pSrc, p2pSink := p2p.MsgPipe() + defer p2pSrc.Close() + defer p2pSink.Close() + + src := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pSrc, source.txpool) + sink := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pSink, source.txpool) + defer src.Close() + defer sink.Close() + + go source.handler.runEthPeer(src, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(source.handler), peer) + }) + // Run the handshake locally to avoid spinning up a sink handler + var ( + genesis = source.chain.Genesis() + td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) + ) + if err := sink.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { + t.Fatalf("failed to run protocol handshake") + } + // After the handshake completes, the source handler should stream the sink + // the blocks, subscribe to inbound network events + backend := new(testEthHandler) + + blocks := make(chan *types.Block, 1) + sub := backend.blockBroadcasts.Subscribe(blocks) + defer sub.Unsubscribe() + + go eth.Handle(backend, sink) + + // Create various combinations of malformed blocks + head := source.chain.CurrentBlock() + + malformedUncles := head.Header() + malformedUncles.UncleHash[0]++ + malformedTransactions := head.Header() + malformedTransactions.TxHash[0]++ + malformedEverything := head.Header() + malformedEverything.UncleHash[0]++ + malformedEverything.TxHash[0]++ + + // Try to broadcast all malformations and ensure they all get discarded + for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} { + block := types.NewBlockWithHeader(header).WithBody(head.Transactions(), head.Uncles()) + if err := src.SendNewBlock(block, big.NewInt(131136)); err != nil { + t.Fatalf("failed to broadcast block: %v", err) + } + select { + case <-blocks: + t.Fatalf("malformed block forwarded") + case <-time.After(100 * time.Millisecond): + } + } +} diff --git a/eth/handler_snap.go b/eth/handler_snap.go new file mode 100644 index 0000000000..25975bf60b --- /dev/null +++ b/eth/handler_snap.go @@ -0,0 +1,48 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// snapHandler implements the snap.Backend interface to handle the various network +// packets that are sent as replies or broadcasts. +type snapHandler handler + +func (h *snapHandler) Chain() *core.BlockChain { return h.chain } + +// RunPeer is invoked when a peer joins on the `snap` protocol. +func (h *snapHandler) RunPeer(peer *snap.Peer, hand snap.Handler) error { + return (*handler)(h).runSnapPeer(peer, hand) +} + +// PeerInfo retrieves all known `snap` information about a peer. +func (h *snapHandler) PeerInfo(id enode.ID) interface{} { + if p := h.peers.snapPeer(id.String()); p != nil { + return p.info() + } + return nil +} + +// Handle is invoked from a peer's message handler when it receives a new remote +// message that the handler couldn't consume and serve itself. +func (h *snapHandler) Handle(peer *snap.Peer, packet snap.Packet) error { + return h.downloader.DeliverSnapPacket(peer, packet) +} diff --git a/eth/handler_test.go b/eth/handler_test.go index fc6c6f2745..a90ef5c348 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -17,678 +17,154 @@ package eth import ( - "fmt" - "math" "math/big" - "math/rand" - "testing" - "time" + "sort" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" ) -// Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders63(t *testing.T) { testGetBlockHeaders(t, 63) } -func TestGetBlockHeaders64(t *testing.T) { testGetBlockHeaders(t, 64) } +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -func testGetBlockHeaders(t *testing.T, protocol int) { - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) - // Create a "random" unknown hash for testing - var unknown common.Hash - for i := range unknown { - unknown[i] = byte(i) - } - // Create a batch of tests for various scenarios - limit := uint64(downloader.MaxHeaderFetch) - tests := []struct { - query *getBlockHeadersData // The query to execute for header retrieval - expect []common.Hash // The hashes of the block whose headers are expected - }{ - // A single random block should be retrievable by hash and number too - { - &getBlockHeadersData{Origin: hashOrNumber{Hash: pm.blockchain.GetBlockByNumber(limit / 2).Hash()}, Amount: 1}, - []common.Hash{pm.blockchain.GetBlockByNumber(limit / 2).Hash()}, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 1}, - []common.Hash{pm.blockchain.GetBlockByNumber(limit / 2).Hash()}, - }, - // Multiple headers should be retrievable in both directions - { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 1).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 2).Hash(), - }, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3, Reverse: true}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 1).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 2).Hash(), - }, - }, - // Multiple headers with skip lists should be retrievable - { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 4).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 8).Hash(), - }, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3, Reverse: true}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 4).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 8).Hash(), - }, - }, - // The chain endpoints should be retrievable - { - &getBlockHeadersData{Origin: hashOrNumber{Number: 0}, Amount: 1}, - []common.Hash{pm.blockchain.GetBlockByNumber(0).Hash()}, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64()}, Amount: 1}, - []common.Hash{pm.blockchain.CurrentBlock().Hash()}, - }, - // Ensure protocol limits are honored - { - &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64() - 1}, Amount: limit + 10, Reverse: true}, - pm.blockchain.GetBlockHashesFromHash(pm.blockchain.CurrentBlock().Hash(), limit), - }, - // Check that requesting more than available is handled gracefully - { - &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64() - 4}, Skip: 3, Amount: 3}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64() - 4).Hash(), - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64()).Hash(), - }, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 3, Amount: 3, Reverse: true}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(4).Hash(), - pm.blockchain.GetBlockByNumber(0).Hash(), - }, - }, - // Check that requesting more than available is handled gracefully, even if mid skip - { - &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64() - 4}, Skip: 2, Amount: 3}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64() - 4).Hash(), - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64() - 1).Hash(), - }, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 2, Amount: 3, Reverse: true}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(4).Hash(), - pm.blockchain.GetBlockByNumber(1).Hash(), - }, - }, - // Check a corner case where requesting more can iterate past the endpoints - { - &getBlockHeadersData{Origin: hashOrNumber{Number: 2}, Amount: 5, Reverse: true}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(2).Hash(), - pm.blockchain.GetBlockByNumber(1).Hash(), - pm.blockchain.GetBlockByNumber(0).Hash(), - }, - }, - // Check a corner case where skipping overflow loops back into the chain start - { - &getBlockHeadersData{Origin: hashOrNumber{Hash: pm.blockchain.GetBlockByNumber(3).Hash()}, Amount: 2, Reverse: false, Skip: math.MaxUint64 - 1}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(3).Hash(), - }, - }, - // Check a corner case where skipping overflow loops back to the same header - { - &getBlockHeadersData{Origin: hashOrNumber{Hash: pm.blockchain.GetBlockByNumber(1).Hash()}, Amount: 2, Reverse: false, Skip: math.MaxUint64}, - []common.Hash{ - pm.blockchain.GetBlockByNumber(1).Hash(), - }, - }, - // Check that non existing headers aren't returned - { - &getBlockHeadersData{Origin: hashOrNumber{Hash: unknown}, Amount: 1}, - []common.Hash{}, - }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64() + 1}, Amount: 1}, - []common.Hash{}, - }, - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Collect the headers to expect in the response - headers := []*types.Header{} - for _, hash := range tt.expect { - headers = append(headers, pm.blockchain.GetBlockByHash(hash).Header()) - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x03, tt.query) - if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - // If the test used number origins, repeat with hashes as the too - if tt.query.Origin.Hash == (common.Hash{}) { - if origin := pm.blockchain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { - tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 +// testTxPool is a mock transaction pool that blindly accepts all transactions. +// Its goal is to get around setting up a valid statedb for the balance and nonce +// checks. +type testTxPool struct { + pool map[common.Hash]*types.Transaction // Hash map of collected transactions - p2p.Send(peer.app, 0x03, tt.query) - if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } - } - } + txFeed event.Feed // Notification feed to allow waiting for inclusion + lock sync.RWMutex // Protects the transaction pool } -// Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies63(t *testing.T) { testGetBlockBodies(t, 63) } -func TestGetBlockBodies64(t *testing.T) { testGetBlockBodies(t, 64) } - -func testGetBlockBodies(t *testing.T, protocol int) { - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, downloader.MaxBlockFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxBlockFetch - tests := []struct { - random int // Number of blocks to fetch randomly from the chain - explicit []common.Hash // Explicitly requested blocks - available []bool // Availability of explicitly requested blocks - expected int // Total number of existing blocks to expect - }{ - {1, nil, nil, 1}, // A single random block should be retrievable - {10, nil, nil, 10}, // Multiple random blocks should be retrievable - {limit, nil, nil, limit}, // The maximum possible blocks should be retrievable - {limit + 1, nil, nil, limit}, // No more than the possible block count should be returned - {0, []common.Hash{pm.blockchain.Genesis().Hash()}, []bool{true}, 1}, // The genesis block should be retrievable - {0, []common.Hash{pm.blockchain.CurrentBlock().Hash()}, []bool{true}, 1}, // The chains head block should be retrievable - {0, []common.Hash{{}}, []bool{false}, 0}, // A non existent block should not be returned - - // Existing and non-existing blocks interleaved should not cause problems - {0, []common.Hash{ - {}, - pm.blockchain.GetBlockByNumber(1).Hash(), - {}, - pm.blockchain.GetBlockByNumber(10).Hash(), - {}, - pm.blockchain.GetBlockByNumber(100).Hash(), - {}, - }, []bool{false, true, false, true, false, true, false}, 3}, - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Collect the hashes to request, and the response to expect - hashes, seen := []common.Hash{}, make(map[int64]bool) - bodies := []*blockBody{} - - for j := 0; j < tt.random; j++ { - for { - num := rand.Int63n(int64(pm.blockchain.CurrentBlock().NumberU64())) - if !seen[num] { - seen[num] = true - - block := pm.blockchain.GetBlockByNumber(uint64(num)) - hashes = append(hashes, block.Hash()) - if len(bodies) < tt.expected { - bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) - } - break - } - } - } - for j, hash := range tt.explicit { - hashes = append(hashes, hash) - if tt.available[j] && len(bodies) < tt.expected { - block := pm.blockchain.GetBlockByHash(hash) - bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) - } - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x05, hashes) - if err := p2p.ExpectMsg(peer.app, 0x06, bodies); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } +// newTestTxPool creates a mock transaction pool. +func newTestTxPool() *testTxPool { + return &testTxPool{ + pool: make(map[common.Hash]*types.Transaction), } } -// Tests that the node state database can be retrieved based on hashes. -func TestGetNodeData63(t *testing.T) { testGetNodeData(t, 63) } -func TestGetNodeData64(t *testing.T) { testGetNodeData(t, 64) } +// Has returns an indicator whether txpool has a transaction +// cached with the given hash. +func (p *testTxPool) Has(hash common.Hash) bool { + p.lock.Lock() + defer p.lock.Unlock() -func testGetNodeData(t *testing.T, protocol int) { - // Define three accounts to simulate transactions with - acc1Key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - acc2Key, _ := crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - acc1Addr := crypto.PubkeyToAddress(acc1Key.PublicKey) - acc2Addr := crypto.PubkeyToAddress(acc2Key.PublicKey) - - signer := types.HomesteadSigner{} - // Create a chain generator with some simple transactions (blatantly stolen from @fjl/chain_markets_test) - generator := func(i int, block *core.BlockGen) { - switch i { - case 0: - // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) - block.AddTx(tx) - case 1: - // In block 2, the test bank sends some more ether to account #1. - // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) - block.AddTx(tx1) - block.AddTx(tx2) - case 2: - // Block 3 is empty but was mined by account #2. - block.SetCoinbase(acc2Addr) - block.SetExtra([]byte("yeehaw")) - case 3: - // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). - b2 := block.PrevBlock(1).Header() - b2.Extra = []byte("foo") - block.AddUncle(b2) - b3 := block.PrevBlock(2).Header() - b3.Extra = []byte("foo") - block.AddUncle(b3) - } - } - // Assemble the test environment - pm, db := newTestProtocolManagerMust(t, downloader.FullSync, 4, generator, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Fetch for now the entire chain db - hashes := []common.Hash{} - - it := db.NewIterator(nil, nil) - for it.Next() { - if key := it.Key(); len(key) == common.HashLength { - hashes = append(hashes, common.BytesToHash(key)) - } - } - it.Release() - - p2p.Send(peer.app, 0x0d, hashes) - msg, err := peer.app.ReadMsg() - if err != nil { - t.Fatalf("failed to read node data response: %v", err) - } - if msg.Code != 0x0e { - t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, 0x0c) - } - var data [][]byte - if err := msg.Decode(&data); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - // Verify that all hashes correspond to the requested data, and reconstruct a state tree - for i, want := range hashes { - if hash := crypto.Keccak256Hash(data[i]); hash != want { - t.Errorf("data hash mismatch: have %x, want %x", hash, want) - } - } - statedb := rawdb.NewMemoryDatabase() - for i := 0; i < len(data); i++ { - statedb.Put(hashes[i].Bytes(), data[i]) - } - accounts := []common.Address{testBank, acc1Addr, acc2Addr} - for i := uint64(0); i <= pm.blockchain.CurrentBlock().NumberU64(); i++ { - trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), state.NewDatabase(statedb), nil) - - for j, acc := range accounts { - state, _ := pm.blockchain.State() - bw := state.GetBalance(acc) - bh := trie.GetBalance(acc) - - if (bw != nil && bh == nil) || (bw == nil && bh != nil) { - t.Errorf("test %d, account %d: balance mismatch: have %v, want %v", i, j, bh, bw) - } - if bw != nil && bh != nil && bw.Cmp(bw) != 0 { - t.Errorf("test %d, account %d: balance mismatch: have %v, want %v", i, j, bh, bw) - } - } - } + return p.pool[hash] != nil } -// Tests that the transaction receipts can be retrieved based on hashes. -func TestGetReceipt63(t *testing.T) { testGetReceipt(t, 63) } -func TestGetReceipt64(t *testing.T) { testGetReceipt(t, 64) } +// Get retrieves the transaction from local txpool with given +// tx hash. +func (p *testTxPool) Get(hash common.Hash) *types.Transaction { + p.lock.Lock() + defer p.lock.Unlock() -func testGetReceipt(t *testing.T, protocol int) { - // Define three accounts to simulate transactions with - acc1Key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - acc2Key, _ := crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - acc1Addr := crypto.PubkeyToAddress(acc1Key.PublicKey) - acc2Addr := crypto.PubkeyToAddress(acc2Key.PublicKey) - - signer := types.HomesteadSigner{} - // Create a chain generator with some simple transactions (blatantly stolen from @fjl/chain_markets_test) - generator := func(i int, block *core.BlockGen) { - switch i { - case 0: - // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) - block.AddTx(tx) - case 1: - // In block 2, the test bank sends some more ether to account #1. - // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) - block.AddTx(tx1) - block.AddTx(tx2) - case 2: - // Block 3 is empty but was mined by account #2. - block.SetCoinbase(acc2Addr) - block.SetExtra([]byte("yeehaw")) - case 3: - // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). - b2 := block.PrevBlock(1).Header() - b2.Extra = []byte("foo") - block.AddUncle(b2) - b3 := block.PrevBlock(2).Header() - b3.Extra = []byte("foo") - block.AddUncle(b3) - } - } - // Assemble the test environment - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 4, generator, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Collect the hashes to request, and the response to expect - hashes, receipts := []common.Hash{}, []types.Receipts{} - for i := uint64(0); i <= pm.blockchain.CurrentBlock().NumberU64(); i++ { - block := pm.blockchain.GetBlockByNumber(i) - - hashes = append(hashes, block.Hash()) - receipts = append(receipts, pm.blockchain.GetReceiptsByHash(block.Hash())) - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x0f, hashes) - if err := p2p.ExpectMsg(peer.app, 0x10, receipts); err != nil { - t.Errorf("receipts mismatch: %v", err) - } + return p.pool[hash] } -// Tests that post eth protocol handshake, clients perform a mutual checkpoint -// challenge to validate each other's chains. Hash mismatches, or missing ones -// during a fast sync should lead to the peer getting dropped. -func TestCheckpointChallenge(t *testing.T) { - tests := []struct { - syncmode downloader.SyncMode - checkpoint bool - timeout bool - empty bool - match bool - drop bool - }{ - // If checkpointing is not enabled locally, don't challenge and don't drop - {downloader.FullSync, false, false, false, false, false}, - {downloader.FastSync, false, false, false, false, false}, - - // If checkpointing is enabled locally and remote response is empty, only drop during fast sync - {downloader.FullSync, true, false, true, false, false}, - {downloader.FastSync, true, false, true, false, true}, // Special case, fast sync, unsynced peer - - // If checkpointing is enabled locally and remote response mismatches, always drop - {downloader.FullSync, true, false, false, false, true}, - {downloader.FastSync, true, false, false, false, true}, - - // If checkpointing is enabled locally and remote response matches, never drop - {downloader.FullSync, true, false, false, true, false}, - {downloader.FastSync, true, false, false, true, false}, +// AddRemotes appends a batch of transactions to the pool, and notifies any +// listeners if the addition channel is non nil +func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { + p.lock.Lock() + defer p.lock.Unlock() - // If checkpointing is enabled locally and remote times out, always drop - {downloader.FullSync, true, true, false, true, true}, - {downloader.FastSync, true, true, false, true, true}, - } - for _, tt := range tests { - t.Run(fmt.Sprintf("sync %v checkpoint %v timeout %v empty %v match %v", tt.syncmode, tt.checkpoint, tt.timeout, tt.empty, tt.match), func(t *testing.T) { - testCheckpointChallenge(t, tt.syncmode, tt.checkpoint, tt.timeout, tt.empty, tt.match, tt.drop) - }) + for _, tx := range txs { + p.pool[tx.Hash()] = tx } + p.txFeed.Send(core.NewTxsEvent{Txs: txs}) + return make([]error, len(txs)) } -func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpoint bool, timeout bool, empty bool, match bool, drop bool) { - // Reduce the checkpoint handshake challenge timeout - defer func(old time.Duration) { syncChallengeTimeout = old }(syncChallengeTimeout) - syncChallengeTimeout = 250 * time.Millisecond - - // Initialize a chain and generate a fake CHT if checkpointing is enabled - var ( - db = rawdb.NewMemoryDatabase() - config = new(params.ChainConfig) - ) - (&core.Genesis{Config: config}).MustCommit(db) // Commit genesis block - // If checkpointing is enabled, create and inject a fake CHT and the corresponding - // chllenge response. - var response *types.Header - var cht *params.TrustedCheckpoint - if checkpoint { - index := uint64(rand.Intn(500)) - number := (index+1)*params.CHTFrequency - 1 - response = &types.Header{Number: big.NewInt(int64(number)), Extra: []byte("valid")} +// Pending returns all the transactions known to the pool +func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) { + p.lock.RLock() + defer p.lock.RUnlock() - cht = ¶ms.TrustedCheckpoint{ - SectionIndex: index, - SectionHead: response.Hash(), - } + batches := make(map[common.Address]types.Transactions) + for _, tx := range p.pool { + from, _ := types.Sender(types.HomesteadSigner{}, tx) + batches[from] = append(batches[from], tx) } - // Create a checkpoint aware protocol manager - blockchain, err := core.NewBlockChain(db, nil, config, ethash.NewFaker(), vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("failed to create new blockchain: %v", err) - } - pm, err := NewProtocolManager(config, cht, syncmode, DefaultConfig.NetworkId, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, ethash.NewFaker(), blockchain, db, 1, nil) - if err != nil { - t.Fatalf("failed to start test protocol manager: %v", err) - } - pm.Start(1000) - defer pm.Stop() - - // Connect a new peer and check that we receive the checkpoint challenge - peer, _ := newTestPeer("peer", eth63, pm, true) - defer peer.close() - - if checkpoint { - challenge := &getBlockHeadersData{ - Origin: hashOrNumber{Number: response.Number.Uint64()}, - Amount: 1, - Skip: 0, - Reverse: false, - } - if err := p2p.ExpectMsg(peer.app, GetBlockHeadersMsg, challenge); err != nil { - t.Fatalf("challenge mismatch: %v", err) - } - // Create a block to reply to the challenge if no timeout is simulated - if !timeout { - if empty { - if err := p2p.Send(peer.app, BlockHeadersMsg, []*types.Header{}); err != nil { - t.Fatalf("failed to answer challenge: %v", err) - } - } else if match { - if err := p2p.Send(peer.app, BlockHeadersMsg, []*types.Header{response}); err != nil { - t.Fatalf("failed to answer challenge: %v", err) - } - } else { - if err := p2p.Send(peer.app, BlockHeadersMsg, []*types.Header{{Number: response.Number}}); err != nil { - t.Fatalf("failed to answer challenge: %v", err) - } - } - } - } - // Wait until the test timeout passes to ensure proper cleanup - time.Sleep(syncChallengeTimeout + 300*time.Millisecond) - - // Verify that the remote peer is maintained or dropped - if drop { - if peers := pm.peers.Len(); peers != 0 { - t.Fatalf("peer count mismatch: have %d, want %d", peers, 0) - } - } else { - if peers := pm.peers.Len(); peers != 1 { - t.Fatalf("peer count mismatch: have %d, want %d", peers, 1) - } + for _, batch := range batches { + sort.Sort(types.TxByNonce(batch)) } + return batches, nil } -func TestBroadcastBlock(t *testing.T) { - var tests = []struct { - totalPeers int - broadcastExpected int - }{ - {1, 1}, - {2, 1}, - {3, 1}, - {4, 2}, - {5, 2}, - {9, 3}, - {12, 3}, - {16, 4}, - {26, 5}, - {100, 10}, - } - for _, test := range tests { - testBroadcastBlock(t, test.totalPeers, test.broadcastExpected) - } +// SubscribeNewTxsEvent should return an event subscription of NewTxsEvent and +// send events to the given channel. +func (p *testTxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { + return p.txFeed.Subscribe(ch) } -func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { - var ( - evmux = new(event.TypeMux) - pow = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() - config = ¶ms.ChainConfig{} - gspec = &core.Genesis{Config: config} - genesis = gspec.MustCommit(db) - ) - blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("failed to create new blockchain: %v", err) - } - pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, evmux, &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, pow, blockchain, db, 1, nil) - if err != nil { - t.Fatalf("failed to start test protocol manager: %v", err) - } - pm.Start(1000) - defer pm.Stop() - var peers []*testPeer - for i := 0; i < totalPeers; i++ { - peer, _ := newTestPeer(fmt.Sprintf("peer %d", i), eth63, pm, true) - defer peer.close() - - peers = append(peers, peer) - } - chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {}) - pm.BroadcastBlock(chain[0], true /*propagate*/) - - errCh := make(chan error, totalPeers) - doneCh := make(chan struct{}, totalPeers) - for _, peer := range peers { - go func(p *testPeer) { - if err := p2p.ExpectMsg(p.app, NewBlockMsg, &newBlockData{Block: chain[0], TD: big.NewInt(131136)}); err != nil { - errCh <- err - } else { - doneCh <- struct{}{} - } - }(peer) - } - var received int - for { - select { - case <-doneCh: - received++ - if received > broadcastExpected { - // We can bail early here - t.Errorf("broadcast count mismatch: have %d > want %d", received, broadcastExpected) - return - } - case <-time.After(2 * time.Second): - if received != broadcastExpected { - t.Errorf("broadcast count mismatch: have %d, want %d", received, broadcastExpected) - } - return - case err = <-errCh: - t.Fatalf("broadcast failed: %v", err) - } - } +// testHandler is a live implementation of the Ethereum protocol handler, just +// preinitialized with some sane testing defaults and the transaction pool mocked +// out. +type testHandler struct { + db ethdb.Database + chain *core.BlockChain + txpool *testTxPool + handler *handler +} +// newTestHandler creates a new handler for testing purposes with no blocks. +func newTestHandler() *testHandler { + return newTestHandlerWithBlocks(0) } -// Tests that a propagated malformed block (uncles or transactions don't match -// with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock(t *testing.T) { - // Create a live node to test propagation with - var ( - engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() - config = ¶ms.ChainConfig{} - gspec = &core.Genesis{Config: config} - genesis = gspec.MustCommit(db) - ) - blockchain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("failed to create new blockchain: %v", err) +// newTestHandlerWithBlocks creates a new handler for testing purposes, with a +// given number of initial blocks. +func newTestHandlerWithBlocks(blocks int) *testHandler { + // Create a database pre-initialize with a genesis block + db := rawdb.NewMemoryDatabase() + (&core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, + }).MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + + bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, nil) + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + txpool := newTestTxPool() + + handler, _ := newHandler(&handlerConfig{ + Database: db, + Chain: chain, + TxPool: txpool, + Network: 1, + Sync: downloader.FastSync, + BloomCache: 1, + }) + handler.Start(1000) + + return &testHandler{ + db: db, + chain: chain, + txpool: txpool, + handler: handler, } - pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), engine, blockchain, db, 1, nil) - if err != nil { - t.Fatalf("failed to start test protocol manager: %v", err) - } - pm.Start(2) - defer pm.Stop() - - // Create two peers, one to send the malformed block with and one to check - // propagation - source, _ := newTestPeer("source", eth63, pm, true) - defer source.close() - - sink, _ := newTestPeer("sink", eth63, pm, true) - defer sink.close() - - // Create various combinations of malformed blocks - chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {}) - - malformedUncles := chain[0].Header() - malformedUncles.UncleHash[0]++ - malformedTransactions := chain[0].Header() - malformedTransactions.TxHash[0]++ - malformedEverything := chain[0].Header() - malformedEverything.UncleHash[0]++ - malformedEverything.TxHash[0]++ +} - // Keep listening to broadcasts and notify if any arrives - notify := make(chan struct{}, 1) - go func() { - if _, err := sink.app.ReadMsg(); err == nil { - notify <- struct{}{} - } - }() - // Try to broadcast all malformations and ensure they all get discarded - for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} { - block := types.NewBlockWithHeader(header).WithBody(chain[0].Transactions(), chain[0].Uncles()) - if err := p2p.Send(source.app, NewBlockMsg, []interface{}{block, big.NewInt(131136)}); err != nil { - t.Fatalf("failed to broadcast block: %v", err) - } - select { - case <-notify: - t.Fatalf("malformed block forwarded") - case <-time.After(100 * time.Millisecond): - } - } +// close tears down the handler and all its internal constructs. +func (b *testHandler) close() { + b.handler.Stop() + b.chain.Stop() } diff --git a/eth/helper_test.go b/eth/helper_test.go deleted file mode 100644 index c0bda181ea..0000000000 --- a/eth/helper_test.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// This file contains some shares testing functionality, common to multiple -// different files and modules being tested. - -package eth - -import ( - "crypto/ecdsa" - "crypto/rand" - "fmt" - "math/big" - "sort" - "sync" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/params" -) - -var ( - testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testBank = crypto.PubkeyToAddress(testBankKey.PublicKey) -) - -// newTestProtocolManager creates a new protocol manager for testing purposes, -// with the given number of blocks already known, and potential notification -// channels for different events. -func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, ethdb.Database, error) { - var ( - evmux = new(event.TypeMux) - engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}}, - } - genesis = gspec.MustCommit(db) - blockchain, _ = core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil) - ) - chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator) - if _, err := blockchain.InsertChain(chain); err != nil { - panic(err) - } - pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx, pool: make(map[common.Hash]*types.Transaction)}, engine, blockchain, db, 1, nil) - if err != nil { - return nil, nil, err - } - pm.Start(1000) - return pm, db, nil -} - -// newTestProtocolManagerMust creates a new protocol manager for testing purposes, -// with the given number of blocks already known, and potential notification -// channels for different events. In case of an error, the constructor force- -// fails the test. -func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, ethdb.Database) { - pm, db, err := newTestProtocolManager(mode, blocks, generator, newtx) - if err != nil { - t.Fatalf("Failed to create protocol manager: %v", err) - } - return pm, db -} - -// testTxPool is a fake, helper transaction pool for testing purposes -type testTxPool struct { - txFeed event.Feed - pool map[common.Hash]*types.Transaction // Hash map of collected transactions - added chan<- []*types.Transaction // Notification channel for new transactions - - lock sync.RWMutex // Protects the transaction pool -} - -// Has returns an indicator whether txpool has a transaction -// cached with the given hash. -func (p *testTxPool) Has(hash common.Hash) bool { - p.lock.Lock() - defer p.lock.Unlock() - - return p.pool[hash] != nil -} - -// Get retrieves the transaction from local txpool with given -// tx hash. -func (p *testTxPool) Get(hash common.Hash) *types.Transaction { - p.lock.Lock() - defer p.lock.Unlock() - - return p.pool[hash] -} - -// AddRemotes appends a batch of transactions to the pool, and notifies any -// listeners if the addition channel is non nil -func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { - p.lock.Lock() - defer p.lock.Unlock() - - for _, tx := range txs { - p.pool[tx.Hash()] = tx - } - if p.added != nil { - p.added <- txs - } - p.txFeed.Send(core.NewTxsEvent{Txs: txs}) - return make([]error, len(txs)) -} - -// Pending returns all the transactions known to the pool -func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - batches := make(map[common.Address]types.Transactions) - for _, tx := range p.pool { - from, _ := types.Sender(types.HomesteadSigner{}, tx) - batches[from] = append(batches[from], tx) - } - for _, batch := range batches { - sort.Sort(types.TxByNonce(batch)) - } - return batches, nil -} - -func (p *testTxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { - return p.txFeed.Subscribe(ch) -} - -// newTestTransaction create a new dummy transaction. -func newTestTransaction(from *ecdsa.PrivateKey, nonce uint64, datasize int) *types.Transaction { - tx := types.NewTransaction(nonce, common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, datasize)) - tx, _ = types.SignTx(tx, types.HomesteadSigner{}, from) - return tx -} - -// testPeer is a simulated peer to allow testing direct network calls. -type testPeer struct { - net p2p.MsgReadWriter // Network layer reader/writer to simulate remote messaging - app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side - *peer -} - -// newTestPeer creates a new peer registered at the given protocol manager. -func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*testPeer, <-chan error) { - // Create a message pipe to communicate through - app, net := p2p.MsgPipe() - - // Start the peer on a new thread - var id enode.ID - rand.Read(id[:]) - peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net, pm.txpool.Get) - errc := make(chan error, 1) - go func() { errc <- pm.runPeer(peer) }() - tp := &testPeer{app: app, net: net, peer: peer} - - // Execute any implicitly requested handshakes and return - if shake { - var ( - genesis = pm.blockchain.Genesis() - head = pm.blockchain.CurrentHeader() - td = pm.blockchain.GetTd(head.Hash(), head.Number.Uint64()) - ) - forkID := forkid.NewID(pm.blockchain.Config(), pm.blockchain.Genesis().Hash(), pm.blockchain.CurrentHeader().Number.Uint64()) - tp.handshake(nil, td, head.Hash(), genesis.Hash(), forkID, forkid.NewFilter(pm.blockchain)) - } - return tp, errc -} - -// handshake simulates a trivial handshake that expects the same state from the -// remote side as we are simulating locally. -func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) { - var msg interface{} - switch { - case p.version == eth63: - msg = &statusData63{ - ProtocolVersion: uint32(p.version), - NetworkId: DefaultConfig.NetworkId, - TD: td, - CurrentBlock: head, - GenesisBlock: genesis, - } - case p.version >= eth64: - msg = &statusData{ - ProtocolVersion: uint32(p.version), - NetworkID: DefaultConfig.NetworkId, - TD: td, - Head: head, - Genesis: genesis, - ForkID: forkID, - } - default: - panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) - } - if err := p2p.ExpectMsg(p.app, StatusMsg, msg); err != nil { - t.Fatalf("status recv: %v", err) - } - if err := p2p.Send(p.app, StatusMsg, msg); err != nil { - t.Fatalf("status send: %v", err) - } -} - -// close terminates the local side of the peer, notifying the remote protocol -// manager of termination. -func (p *testPeer) close() { - p.app.Close() -} diff --git a/eth/peer.go b/eth/peer.go index 21b82a19c5..6970c8afd3 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -17,806 +17,58 @@ package eth import ( - "errors" - "fmt" "math/big" "sync" "time" - mapset "github.com/deckarep/golang-set" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" ) -var ( - errClosed = errors.New("peer set is closed") - errAlreadyRegistered = errors.New("peer is already registered") - errNotRegistered = errors.New("peer is not registered") -) - -const ( - maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) - maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) - - // maxQueuedTxs is the maximum number of transactions to queue up before dropping - // older broadcasts. - maxQueuedTxs = 4096 - - // maxQueuedTxAnns is the maximum number of transaction announcements to queue up - // before dropping older announcements. - maxQueuedTxAnns = 4096 - - // maxQueuedBlocks is the maximum number of block propagations to queue up before - // dropping broadcasts. There's not much point in queueing stale blocks, so a few - // that might cover uncles should be enough. - maxQueuedBlocks = 4 - - // maxQueuedBlockAnns is the maximum number of block announcements to queue up before - // dropping broadcasts. Similarly to block propagations, there's no point to queue - // above some healthy uncle limit, so use that. - maxQueuedBlockAnns = 4 - - handshakeTimeout = 5 * time.Second -) - -// max is a helper function which returns the larger of the two given integers. -func max(a, b int) int { - if a > b { - return a - } - return b -} - -// PeerInfo represents a short summary of the Ethereum sub-protocol metadata known +// ethPeerInfo represents a short summary of the `eth` sub-protocol metadata known // about a connected peer. -type PeerInfo struct { - Version int `json:"version"` // Ethereum protocol version negotiated +type ethPeerInfo struct { + Version uint `json:"version"` // Ethereum protocol version negotiated Difficulty *big.Int `json:"difficulty"` // Total difficulty of the peer's blockchain - Head string `json:"head"` // SHA3 hash of the peer's best owned block -} - -// propEvent is a block propagation, waiting for its turn in the broadcast queue. -type propEvent struct { - block *types.Block - td *big.Int -} - -type peer struct { - id string - - *p2p.Peer - rw p2p.MsgReadWriter - - version int // Protocol version negotiated - syncDrop *time.Timer // Timed connection dropper if sync progress isn't validated in time - - head common.Hash - td *big.Int - lock sync.RWMutex - - knownBlocks mapset.Set // Set of block hashes known to be known by this peer - queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer - queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer - - knownTxs mapset.Set // Set of transaction hashes known to be known by this peer - txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests - txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests - getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool - - term chan struct{} // Termination channel to stop the broadcaster + Head string `json:"head"` // Hex hash of the peer's best owned block } -func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { - return &peer{ - Peer: p, - rw: rw, - version: version, - id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), - knownTxs: mapset.NewSet(), - knownBlocks: mapset.NewSet(), - queuedBlocks: make(chan *propEvent, maxQueuedBlocks), - queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), - txBroadcast: make(chan []common.Hash), - txAnnounce: make(chan []common.Hash), - getPooledTx: getPooledTx, - term: make(chan struct{}), - } -} - -// broadcastBlocks is a write loop that multiplexes blocks and block accouncements -// to the remote peer. The goal is to have an async writer that does not lock up -// node internals and at the same time rate limits queued data. -func (p *peer) broadcastBlocks(removePeer func(string)) { - for { - select { - case prop := <-p.queuedBlocks: - if err := p.SendNewBlock(prop.block, prop.td); err != nil { - removePeer(p.id) - return - } - p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) - - case block := <-p.queuedBlockAnns: - if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { - removePeer(p.id) - return - } - p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) - - case <-p.term: - return - } - } -} - -// broadcastTransactions is a write loop that schedules transaction broadcasts -// to the remote peer. The goal is to have an async writer that does not lock up -// node internals and at the same time rate limits queued data. -func (p *peer) broadcastTransactions(removePeer func(string)) { - var ( - queue []common.Hash // Queue of hashes to broadcast as full transactions - done chan struct{} // Non-nil if background broadcaster is running - fail = make(chan error, 1) // Channel used to receive network error - ) - for { - // If there's no in-flight broadcast running, check if a new one is needed - if done == nil && len(queue) > 0 { - // Pile transaction until we reach our allowed network limit - var ( - hashes []common.Hash - txs []*types.Transaction - size common.StorageSize - ) - for i := 0; i < len(queue) && size < txsyncPackSize; i++ { - if tx := p.getPooledTx(queue[i]); tx != nil { - txs = append(txs, tx) - size += tx.Size() - } - hashes = append(hashes, queue[i]) - } - queue = queue[:copy(queue, queue[len(hashes):])] - - // If there's anything available to transfer, fire up an async writer - if len(txs) > 0 { - done = make(chan struct{}) - go func() { - if err := p.sendTransactions(txs); err != nil { - fail <- err - return - } - close(done) - p.Log().Trace("Sent transactions", "count", len(txs)) - }() - } - } - // Transfer goroutine may or may not have been started, listen for events - select { - case hashes := <-p.txBroadcast: - // New batch of transactions to be broadcast, queue them (with cap) - queue = append(queue, hashes...) - if len(queue) > maxQueuedTxs { - // Fancy copy and resize to ensure buffer doesn't grow indefinitely - queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] - } - - case <-done: - done = nil - - case <-fail: - removePeer(p.id) - return +// ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. +type ethPeer struct { + *eth.Peer - case <-p.term: - return - } - } + syncDrop *time.Timer // Connection dropper if `eth` sync progress isn't validated in time + lock sync.RWMutex // Mutex protecting the internal fields } -// announceTransactions is a write loop that schedules transaction broadcasts -// to the remote peer. The goal is to have an async writer that does not lock up -// node internals and at the same time rate limits queued data. -func (p *peer) announceTransactions(removePeer func(string)) { - var ( - queue []common.Hash // Queue of hashes to announce as transaction stubs - done chan struct{} // Non-nil if background announcer is running - fail = make(chan error, 1) // Channel used to receive network error - ) - for { - // If there's no in-flight announce running, check if a new one is needed - if done == nil && len(queue) > 0 { - // Pile transaction hashes until we reach our allowed network limit - var ( - hashes []common.Hash - pending []common.Hash - size common.StorageSize - ) - for i := 0; i < len(queue) && size < txsyncPackSize; i++ { - if p.getPooledTx(queue[i]) != nil { - pending = append(pending, queue[i]) - size += common.HashLength - } - hashes = append(hashes, queue[i]) - } - queue = queue[:copy(queue, queue[len(hashes):])] - - // If there's anything available to transfer, fire up an async writer - if len(pending) > 0 { - done = make(chan struct{}) - go func() { - if err := p.sendPooledTransactionHashes(pending); err != nil { - fail <- err - return - } - close(done) - p.Log().Trace("Sent transaction announcements", "count", len(pending)) - }() - } - } - // Transfer goroutine may or may not have been started, listen for events - select { - case hashes := <-p.txAnnounce: - // New batch of transactions to be broadcast, queue them (with cap) - queue = append(queue, hashes...) - if len(queue) > maxQueuedTxAnns { - // Fancy copy and resize to ensure buffer doesn't grow indefinitely - queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxAnns:])] - } - - case <-done: - done = nil - - case <-fail: - removePeer(p.id) - return - - case <-p.term: - return - } - } -} - -// close signals the broadcast goroutine to terminate. -func (p *peer) close() { - close(p.term) -} - -// Info gathers and returns a collection of metadata known about a peer. -func (p *peer) Info() *PeerInfo { +// info gathers and returns some `eth` protocol metadata known about a peer. +func (p *ethPeer) info() *ethPeerInfo { hash, td := p.Head() - return &PeerInfo{ - Version: p.version, + return ðPeerInfo{ + Version: p.Version(), Difficulty: td, Head: hash.Hex(), } } -// Head retrieves a copy of the current head hash and total difficulty of the -// peer. -func (p *peer) Head() (hash common.Hash, td *big.Int) { - p.lock.RLock() - defer p.lock.RUnlock() - - copy(hash[:], p.head[:]) - return hash, new(big.Int).Set(p.td) -} - -// SetHead updates the head hash and total difficulty of the peer. -func (p *peer) SetHead(hash common.Hash, td *big.Int) { - p.lock.Lock() - defer p.lock.Unlock() - - copy(p.head[:], hash[:]) - p.td.Set(td) -} - -// MarkBlock marks a block as known for the peer, ensuring that the block will -// never be propagated to this particular peer. -func (p *peer) MarkBlock(hash common.Hash) { - // If we reached the memory allowance, drop a previously known block hash - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } - p.knownBlocks.Add(hash) -} - -// MarkTransaction marks a transaction as known for the peer, ensuring that it -// will never be propagated to this particular peer. -func (p *peer) MarkTransaction(hash common.Hash) { - // If we reached the memory allowance, drop a previously known transaction hash - for p.knownTxs.Cardinality() >= maxKnownTxs { - p.knownTxs.Pop() - } - p.knownTxs.Add(hash) -} - -// SendTransactions64 sends transactions to the peer and includes the hashes -// in its transaction hash set for future reference. -// -// This method is legacy support for initial transaction exchange in eth/64 and -// prior. For eth/65 and higher use SendPooledTransactionHashes. -func (p *peer) SendTransactions64(txs types.Transactions) error { - return p.sendTransactions(txs) -} - -// sendTransactions sends transactions to the peer and includes the hashes -// in its transaction hash set for future reference. -// -// This method is a helper used by the async transaction sender. Don't call it -// directly as the queueing (memory) and transmission (bandwidth) costs should -// not be managed directly. -func (p *peer) sendTransactions(txs types.Transactions) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { - p.knownTxs.Pop() - } - for _, tx := range txs { - p.knownTxs.Add(tx.Hash()) - } - return p2p.Send(p.rw, TransactionMsg, txs) -} - -// AsyncSendTransactions queues a list of transactions (by hash) to eventually -// propagate to a remote peer. The number of pending sends are capped (new ones -// will force old sends to be dropped) -func (p *peer) AsyncSendTransactions(hashes []common.Hash) { - select { - case p.txBroadcast <- hashes: - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - case <-p.term: - p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) - } -} - -// sendPooledTransactionHashes sends transaction hashes to the peer and includes -// them in its transaction hash set for future reference. -// -// This method is a helper used by the async transaction announcer. Don't call it -// directly as the queueing (memory) and transmission (bandwidth) costs should -// not be managed directly. -func (p *peer) sendPooledTransactionHashes(hashes []common.Hash) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) -} - -// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually -// announce to a remote peer. The number of pending sends are capped (new ones -// will force old sends to be dropped) -func (p *peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { - select { - case p.txAnnounce <- hashes: - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - case <-p.term: - p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) - } -} - -// SendPooledTransactionsRLP sends requested transactions to the peer and adds the -// hashes in its transaction hash set for future reference. -// -// Note, the method assumes the hashes are correct and correspond to the list of -// transactions being sent. -func (p *peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, PooledTransactionsMsg, txs) -} - -// SendNewBlockHashes announces the availability of a number of blocks through -// a hash notification. -func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { - // Mark all the block hashes as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() > max(0, maxKnownBlocks-len(hashes)) { - p.knownBlocks.Pop() - } - for _, hash := range hashes { - p.knownBlocks.Add(hash) - } - request := make(newBlockHashesData, len(hashes)) - for i := 0; i < len(hashes); i++ { - request[i].Hash = hashes[i] - request[i].Number = numbers[i] - } - return p2p.Send(p.rw, NewBlockHashesMsg, request) -} - -// AsyncSendNewBlockHash queues the availability of a block for propagation to a -// remote peer. If the peer's broadcast queue is full, the event is silently -// dropped. -func (p *peer) AsyncSendNewBlockHash(block *types.Block) { - select { - case p.queuedBlockAnns <- block: - // Mark all the block hash as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) - } -} - -// SendNewBlock propagates an entire block to a remote peer. -func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { - // Mark all the block hash as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } - p.knownBlocks.Add(block.Hash()) - return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td}) -} - -// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If -// the peer's broadcast queue is full, the event is silently dropped. -func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { - select { - case p.queuedBlocks <- &propEvent{block: block, td: td}: - // Mark all the block hash as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) - } -} - -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *peer) SendBlockHeaders(headers []*types.Header) error { - return p2p.Send(p.rw, BlockHeadersMsg, headers) -} - -// SendBlockBodies sends a batch of block contents to the remote peer. -func (p *peer) SendBlockBodies(bodies []*blockBody) error { - return p2p.Send(p.rw, BlockBodiesMsg, blockBodiesData(bodies)) -} - -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from -// an already RLP encoded format. -func (p *peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { - return p2p.Send(p.rw, BlockBodiesMsg, bodies) -} - -// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the -// hashes requested. -func (p *peer) SendNodeData(data [][]byte) error { - return p2p.Send(p.rw, NodeDataMsg, data) -} - -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the -// ones requested from an already RLP encoded format. -func (p *peer) SendReceiptsRLP(receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, receipts) -} - -// RequestOneHeader is a wrapper around the header query functions to fetch a -// single header. It is used solely by the fetcher. -func (p *peer) RequestOneHeader(hash common.Hash) error { - p.Log().Debug("Fetching single header", "hash", hash) - return p2p.Send(p.rw, GetBlockHeadersMsg, &getBlockHeadersData{Origin: hashOrNumber{Hash: hash}, Amount: uint64(1), Skip: uint64(0), Reverse: false}) -} - -// RequestHeadersByHash fetches a batch of blocks' headers corresponding to the -// specified header query, based on the hash of an origin block. -func (p *peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - return p2p.Send(p.rw, GetBlockHeadersMsg, &getBlockHeadersData{Origin: hashOrNumber{Hash: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse}) -} - -// RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the -// specified header query, based on the number of an origin block. -func (p *peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - return p2p.Send(p.rw, GetBlockHeadersMsg, &getBlockHeadersData{Origin: hashOrNumber{Number: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse}) -} - -// RequestBodies fetches a batch of blocks' bodies corresponding to the hashes -// specified. -func (p *peer) RequestBodies(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - return p2p.Send(p.rw, GetBlockBodiesMsg, hashes) -} - -// RequestNodeData fetches a batch of arbitrary data from a node's known state -// data, corresponding to the specified hashes. -func (p *peer) RequestNodeData(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - return p2p.Send(p.rw, GetNodeDataMsg, hashes) -} - -// RequestReceipts fetches a batch of transaction receipts from a remote node. -func (p *peer) RequestReceipts(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - return p2p.Send(p.rw, GetReceiptsMsg, hashes) -} - -// RequestTxs fetches a batch of transactions from a remote node. -func (p *peer) RequestTxs(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - return p2p.Send(p.rw, GetPooledTransactionsMsg, hashes) -} - -// Handshake executes the eth protocol handshake, negotiating version number, -// network IDs, difficulties, head and genesis blocks. -func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { - // Send out own handshake in a new thread - errc := make(chan error, 2) - - var ( - status63 statusData63 // safe to read after two values have been received from errc - status statusData // safe to read after two values have been received from errc - ) - go func() { - switch { - case p.version == eth63: - errc <- p2p.Send(p.rw, StatusMsg, &statusData63{ - ProtocolVersion: uint32(p.version), - NetworkId: network, - TD: td, - CurrentBlock: head, - GenesisBlock: genesis, - }) - case p.version >= eth64: - errc <- p2p.Send(p.rw, StatusMsg, &statusData{ - ProtocolVersion: uint32(p.version), - NetworkID: network, - TD: td, - Head: head, - Genesis: genesis, - ForkID: forkID, - }) - default: - panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) - } - }() - go func() { - switch { - case p.version == eth63: - errc <- p.readStatusLegacy(network, &status63, genesis) - case p.version >= eth64: - errc <- p.readStatus(network, &status, genesis, forkFilter) - default: - panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) - } - }() - timeout := time.NewTimer(handshakeTimeout) - defer timeout.Stop() - for i := 0; i < 2; i++ { - select { - case err := <-errc: - if err != nil { - return err - } - case <-timeout.C: - return p2p.DiscReadTimeout - } - } - switch { - case p.version == eth63: - p.td, p.head = status63.TD, status63.CurrentBlock - case p.version >= eth64: - p.td, p.head = status.TD, status.Head - default: - panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) - } - return nil -} - -func (p *peer) readStatusLegacy(network uint64, status *statusData63, genesis common.Hash) error { - msg, err := p.rw.ReadMsg() - if err != nil { - return err - } - if msg.Code != StatusMsg { - return errResp(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg) - } - if msg.Size > protocolMaxMsgSize { - return errResp(ErrMsgTooLarge, "%v > %v", msg.Size, protocolMaxMsgSize) - } - // Decode the handshake and make sure everything matches - if err := msg.Decode(&status); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - if status.GenesisBlock != genesis { - return errResp(ErrGenesisMismatch, "%x (!= %x)", status.GenesisBlock[:8], genesis[:8]) - } - if status.NetworkId != network { - return errResp(ErrNetworkIDMismatch, "%d (!= %d)", status.NetworkId, network) - } - if int(status.ProtocolVersion) != p.version { - return errResp(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, p.version) - } - return nil -} - -func (p *peer) readStatus(network uint64, status *statusData, genesis common.Hash, forkFilter forkid.Filter) error { - msg, err := p.rw.ReadMsg() - if err != nil { - return err - } - if msg.Code != StatusMsg { - return errResp(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg) - } - if msg.Size > protocolMaxMsgSize { - return errResp(ErrMsgTooLarge, "%v > %v", msg.Size, protocolMaxMsgSize) - } - // Decode the handshake and make sure everything matches - if err := msg.Decode(&status); err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - if status.NetworkID != network { - return errResp(ErrNetworkIDMismatch, "%d (!= %d)", status.NetworkID, network) - } - if int(status.ProtocolVersion) != p.version { - return errResp(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, p.version) - } - if status.Genesis != genesis { - return errResp(ErrGenesisMismatch, "%x (!= %x)", status.Genesis, genesis) - } - if err := forkFilter(status.ForkID); err != nil { - return errResp(ErrForkIDRejected, "%v", err) - } - return nil -} - -// String implements fmt.Stringer. -func (p *peer) String() string { - return fmt.Sprintf("Peer %s [%s]", p.id, - fmt.Sprintf("eth/%2d", p.version), - ) -} - -// peerSet represents the collection of active peers currently participating in -// the Ethereum sub-protocol. -type peerSet struct { - peers map[string]*peer - lock sync.RWMutex - closed bool -} - -// newPeerSet creates a new peer set to track the active participants. -func newPeerSet() *peerSet { - return &peerSet{ - peers: make(map[string]*peer), - } -} - -// Register injects a new peer into the working set, or returns an error if the -// peer is already known. If a new peer it registered, its broadcast loop is also -// started. -func (ps *peerSet) Register(p *peer, removePeer func(string)) error { - ps.lock.Lock() - defer ps.lock.Unlock() - - if ps.closed { - return errClosed - } - if _, ok := ps.peers[p.id]; ok { - return errAlreadyRegistered - } - ps.peers[p.id] = p - - go p.broadcastBlocks(removePeer) - go p.broadcastTransactions(removePeer) - if p.version >= eth65 { - go p.announceTransactions(removePeer) - } - return nil -} - -// Unregister removes a remote peer from the active set, disabling any further -// actions to/from that particular entity. -func (ps *peerSet) Unregister(id string) error { - ps.lock.Lock() - defer ps.lock.Unlock() - - p, ok := ps.peers[id] - if !ok { - return errNotRegistered - } - delete(ps.peers, id) - p.close() - - return nil -} - -// Peer retrieves the registered peer with the given id. -func (ps *peerSet) Peer(id string) *peer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return ps.peers[id] -} - -// Len returns if the current number of peers in the set. -func (ps *peerSet) Len() int { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return len(ps.peers) -} - -// PeersWithoutBlock retrieves a list of peers that do not have a given block in -// their set of known hashes. -func (ps *peerSet) PeersWithoutBlock(hash common.Hash) []*peer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - list := make([]*peer, 0, len(ps.peers)) - for _, p := range ps.peers { - if !p.knownBlocks.Contains(hash) { - list = append(list, p) - } - } - return list -} - -// PeersWithoutTx retrieves a list of peers that do not have a given transaction -// in their set of known hashes. -func (ps *peerSet) PeersWithoutTx(hash common.Hash) []*peer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - list := make([]*peer, 0, len(ps.peers)) - for _, p := range ps.peers { - if !p.knownTxs.Contains(hash) { - list = append(list, p) - } - } - return list +// snapPeerInfo represents a short summary of the `snap` sub-protocol metadata known +// about a connected peer. +type snapPeerInfo struct { + Version uint `json:"version"` // Snapshot protocol version negotiated } -// BestPeer retrieves the known peer with the currently highest total difficulty. -func (ps *peerSet) BestPeer() *peer { - ps.lock.RLock() - defer ps.lock.RUnlock() +// snapPeer is a wrapper around snap.Peer to maintain a few extra metadata. +type snapPeer struct { + *snap.Peer - var ( - bestPeer *peer - bestTd *big.Int - ) - for _, p := range ps.peers { - if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p, td - } - } - return bestPeer + ethDrop *time.Timer // Connection dropper if `eth` doesn't connect in time + lock sync.RWMutex // Mutex protecting the internal fields } -// Close disconnects all peers. -// No new peers can be registered after Close has returned. -func (ps *peerSet) Close() { - ps.lock.Lock() - defer ps.lock.Unlock() - - for _, p := range ps.peers { - p.Disconnect(p2p.DiscQuitting) +// info gathers and returns some `snap` protocol metadata known about a peer. +func (p *snapPeer) info() *snapPeerInfo { + return &snapPeerInfo{ + Version: p.Version(), } - ps.closed = true } diff --git a/eth/peerset.go b/eth/peerset.go new file mode 100644 index 0000000000..9b584ec320 --- /dev/null +++ b/eth/peerset.go @@ -0,0 +1,301 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "errors" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/p2p" +) + +var ( + // errPeerSetClosed is returned if a peer is attempted to be added or removed + // from the peer set after it has been terminated. + errPeerSetClosed = errors.New("peerset closed") + + // errPeerAlreadyRegistered is returned if a peer is attempted to be added + // to the peer set, but one with the same id already exists. + errPeerAlreadyRegistered = errors.New("peer already registered") + + // errPeerNotRegistered is returned if a peer is attempted to be removed from + // a peer set, but no peer with the given id exists. + errPeerNotRegistered = errors.New("peer not registered") + + // ethConnectTimeout is the `snap` timeout for `eth` to connect too. + ethConnectTimeout = 3 * time.Second +) + +// peerSet represents the collection of active peers currently participating in +// the `eth` or `snap` protocols. +type peerSet struct { + ethPeers map[string]*ethPeer // Peers connected on the `eth` protocol + snapPeers map[string]*snapPeer // Peers connected on the `snap` protocol + + ethJoinFeed event.Feed // Events when an `eth` peer successfully joins + ethDropFeed event.Feed // Events when an `eth` peer gets dropped + snapJoinFeed event.Feed // Events when a `snap` peer joins on both `eth` and `snap` + snapDropFeed event.Feed // Events when a `snap` peer gets dropped (only if fully joined) + + scope event.SubscriptionScope // Subscription group to unsubscribe everyone at once + + lock sync.RWMutex + closed bool +} + +// newPeerSet creates a new peer set to track the active participants. +func newPeerSet() *peerSet { + return &peerSet{ + ethPeers: make(map[string]*ethPeer), + snapPeers: make(map[string]*snapPeer), + } +} + +// subscribeEthJoin registers a subscription for peers joining (and completing +// the handshake) on the `eth` protocol. +func (ps *peerSet) subscribeEthJoin(ch chan<- *eth.Peer) event.Subscription { + return ps.scope.Track(ps.ethJoinFeed.Subscribe(ch)) +} + +// subscribeEthDrop registers a subscription for peers being dropped from the +// `eth` protocol. +func (ps *peerSet) subscribeEthDrop(ch chan<- *eth.Peer) event.Subscription { + return ps.scope.Track(ps.ethDropFeed.Subscribe(ch)) +} + +// subscribeSnapJoin registers a subscription for peers joining (and completing +// the `eth` join) on the `snap` protocol. +func (ps *peerSet) subscribeSnapJoin(ch chan<- *snap.Peer) event.Subscription { + return ps.scope.Track(ps.snapJoinFeed.Subscribe(ch)) +} + +// subscribeSnapDrop registers a subscription for peers being dropped from the +// `snap` protocol. +func (ps *peerSet) subscribeSnapDrop(ch chan<- *snap.Peer) event.Subscription { + return ps.scope.Track(ps.snapDropFeed.Subscribe(ch)) +} + +// registerEthPeer injects a new `eth` peer into the working set, or returns an +// error if the peer is already known. The peer is announced on the `eth` join +// feed and if it completes a pending `snap` peer, also on that feed. +func (ps *peerSet) registerEthPeer(peer *eth.Peer) error { + ps.lock.Lock() + if ps.closed { + ps.lock.Unlock() + return errPeerSetClosed + } + id := peer.ID() + if _, ok := ps.ethPeers[id]; ok { + ps.lock.Unlock() + return errPeerAlreadyRegistered + } + ps.ethPeers[id] = ðPeer{Peer: peer} + + snap, ok := ps.snapPeers[id] + ps.lock.Unlock() + + if ok { + // Previously dangling `snap` peer, stop it's timer since `eth` connected + snap.lock.Lock() + if snap.ethDrop != nil { + snap.ethDrop.Stop() + snap.ethDrop = nil + } + snap.lock.Unlock() + } + ps.ethJoinFeed.Send(peer) + if ok { + ps.snapJoinFeed.Send(snap.Peer) + } + return nil +} + +// unregisterEthPeer removes a remote peer from the active set, disabling any further +// actions to/from that particular entity. The drop is announced on the `eth` drop +// feed and also on the `snap` feed if the eth/snap duality was broken just now. +func (ps *peerSet) unregisterEthPeer(id string) error { + ps.lock.Lock() + eth, ok := ps.ethPeers[id] + if !ok { + ps.lock.Unlock() + return errPeerNotRegistered + } + delete(ps.ethPeers, id) + + snap, ok := ps.snapPeers[id] + ps.lock.Unlock() + + ps.ethDropFeed.Send(eth) + if ok { + ps.snapDropFeed.Send(snap) + } + return nil +} + +// registerSnapPeer injects a new `snap` peer into the working set, or returns +// an error if the peer is already known. The peer is announced on the `snap` +// join feed if it completes an existing `eth` peer. +// +// If the peer isn't yet connected on `eth` and fails to do so within a given +// amount of time, it is dropped. This enforces that `snap` is an extension to +// `eth`, not a standalone leeching protocol. +func (ps *peerSet) registerSnapPeer(peer *snap.Peer) error { + ps.lock.Lock() + if ps.closed { + ps.lock.Unlock() + return errPeerSetClosed + } + id := peer.ID() + if _, ok := ps.snapPeers[id]; ok { + ps.lock.Unlock() + return errPeerAlreadyRegistered + } + ps.snapPeers[id] = &snapPeer{Peer: peer} + + _, ok := ps.ethPeers[id] + if !ok { + // Dangling `snap` peer, start a timer to drop if `eth` doesn't connect + ps.snapPeers[id].ethDrop = time.AfterFunc(ethConnectTimeout, func() { + peer.Log().Warn("Snapshot peer missing eth, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) + peer.Disconnect(p2p.DiscUselessPeer) + }) + } + ps.lock.Unlock() + + if ok { + ps.snapJoinFeed.Send(peer) + } + return nil +} + +// unregisterSnapPeer removes a remote peer from the active set, disabling any +// further actions to/from that particular entity. The drop is announced on the +// `snap` drop feed. +func (ps *peerSet) unregisterSnapPeer(id string) error { + ps.lock.Lock() + peer, ok := ps.snapPeers[id] + if !ok { + ps.lock.Unlock() + return errPeerNotRegistered + } + delete(ps.snapPeers, id) + ps.lock.Unlock() + + peer.lock.Lock() + if peer.ethDrop != nil { + peer.ethDrop.Stop() + peer.ethDrop = nil + } + peer.lock.Unlock() + + ps.snapDropFeed.Send(peer) + return nil +} + +// ethPeer retrieves the registered `eth` peer with the given id. +func (ps *peerSet) ethPeer(id string) *ethPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return ps.ethPeers[id] +} + +// snapPeer retrieves the registered `snap` peer with the given id. +func (ps *peerSet) snapPeer(id string) *snapPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return ps.snapPeers[id] +} + +// ethPeersWithoutBlock retrieves a list of `eth` peers that do not have a given +// block in their set of known hashes so it might be propagated to them. +func (ps *peerSet) ethPeersWithoutBlock(hash common.Hash) []*ethPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + list := make([]*ethPeer, 0, len(ps.ethPeers)) + for _, p := range ps.ethPeers { + if !p.KnownBlock(hash) { + list = append(list, p) + } + } + return list +} + +// ethPeersWithoutTransacion retrieves a list of `eth` peers that do not have a +// given transaction in their set of known hashes. +func (ps *peerSet) ethPeersWithoutTransacion(hash common.Hash) []*ethPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + list := make([]*ethPeer, 0, len(ps.ethPeers)) + for _, p := range ps.ethPeers { + if !p.KnownTransaction(hash) { + list = append(list, p) + } + } + return list +} + +// Len returns if the current number of `eth` peers in the set. Since the `snap` +// peers are tied to the existnce of an `eth` connection, that will always be a +// subset of `eth`. +func (ps *peerSet) Len() int { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return len(ps.ethPeers) +} + +// ethPeerWithHighestTD retrieves the known peer with the currently highest total +// difficulty. +func (ps *peerSet) ethPeerWithHighestTD() *eth.Peer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + var ( + bestPeer *eth.Peer + bestTd *big.Int + ) + for _, p := range ps.ethPeers { + if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { + bestPeer, bestTd = p.Peer, td + } + } + return bestPeer +} + +// close disconnects all peers. +func (ps *peerSet) close() { + ps.lock.Lock() + defer ps.lock.Unlock() + + for _, p := range ps.ethPeers { + p.Disconnect(p2p.DiscQuitting) + } + for _, p := range ps.snapPeers { + p.Disconnect(p2p.DiscQuitting) + } + ps.closed = true +} diff --git a/eth/protocol.go b/eth/protocol.go deleted file mode 100644 index dc75d6b31a..0000000000 --- a/eth/protocol.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "fmt" - "io" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/rlp" -) - -// Constants to match up protocol versions and messages -const ( - eth63 = 63 - eth64 = 64 - eth65 = 65 -) - -// protocolName is the official short name of the protocol used during capability negotiation. -const protocolName = "eth" - -// ProtocolVersions are the supported versions of the eth protocol (first is primary). -var ProtocolVersions = []uint{eth65, eth64, eth63} - -// protocolLengths are the number of implemented message corresponding to different protocol versions. -var protocolLengths = map[uint]uint64{eth65: 17, eth64: 17, eth63: 17} - -const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message - -// eth protocol message codes -const ( - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 - - // New protocol message codes introduced in eth65 - // - // Previously these message ids were used by some legacy and unsupported - // eth protocols, reown them here. - NewPooledTransactionHashesMsg = 0x08 - GetPooledTransactionsMsg = 0x09 - PooledTransactionsMsg = 0x0a -) - -type errCode int - -const ( - ErrMsgTooLarge = iota - ErrDecode - ErrInvalidMsgCode - ErrProtocolVersionMismatch - ErrNetworkIDMismatch - ErrGenesisMismatch - ErrForkIDRejected - ErrNoStatusMsg - ErrExtraStatusMsg -) - -func (e errCode) String() string { - return errorToString[int(e)] -} - -// XXX change once legacy code is out -var errorToString = map[int]string{ - ErrMsgTooLarge: "Message too long", - ErrDecode: "Invalid message", - ErrInvalidMsgCode: "Invalid message code", - ErrProtocolVersionMismatch: "Protocol version mismatch", - ErrNetworkIDMismatch: "Network ID mismatch", - ErrGenesisMismatch: "Genesis mismatch", - ErrForkIDRejected: "Fork ID rejected", - ErrNoStatusMsg: "No status message", - ErrExtraStatusMsg: "Extra status message", -} - -type txPool interface { - // Has returns an indicator whether txpool has a transaction - // cached with the given hash. - Has(hash common.Hash) bool - - // Get retrieves the transaction from local txpool with given - // tx hash. - Get(hash common.Hash) *types.Transaction - - // AddRemotes should add the given transactions to the pool. - AddRemotes([]*types.Transaction) []error - - // Pending should return pending transactions. - // The slice should be modifiable by the caller. - Pending() (map[common.Address]types.Transactions, error) - - // SubscribeNewTxsEvent should return an event subscription of - // NewTxsEvent and send events to the given channel. - SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription -} - -// statusData63 is the network packet for the status message for eth/63. -type statusData63 struct { - ProtocolVersion uint32 - NetworkId uint64 - TD *big.Int - CurrentBlock common.Hash - GenesisBlock common.Hash -} - -// statusData is the network packet for the status message for eth/64 and later. -type statusData struct { - ProtocolVersion uint32 - NetworkID uint64 - TD *big.Int - Head common.Hash - Genesis common.Hash - ForkID forkid.ID -} - -// newBlockHashesData is the network packet for the block announcements. -type newBlockHashesData []struct { - Hash common.Hash // Hash of one particular block being announced - Number uint64 // Number of one particular block being announced -} - -// getBlockHeadersData represents a block header query. -type getBlockHeadersData struct { - Origin hashOrNumber // Block from which to retrieve headers - Amount uint64 // Maximum number of headers to retrieve - Skip uint64 // Blocks to skip between consecutive headers - Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) -} - -// hashOrNumber is a combined field for specifying an origin block. -type hashOrNumber struct { - Hash common.Hash // Block hash from which to retrieve headers (excludes Number) - Number uint64 // Block hash from which to retrieve headers (excludes Hash) -} - -// EncodeRLP is a specialized encoder for hashOrNumber to encode only one of the -// two contained union fields. -func (hn *hashOrNumber) EncodeRLP(w io.Writer) error { - if hn.Hash == (common.Hash{}) { - return rlp.Encode(w, hn.Number) - } - if hn.Number != 0 { - return fmt.Errorf("both origin hash (%x) and number (%d) provided", hn.Hash, hn.Number) - } - return rlp.Encode(w, hn.Hash) -} - -// DecodeRLP is a specialized decoder for hashOrNumber to decode the contents -// into either a block hash or a block number. -func (hn *hashOrNumber) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - origin, err := s.Raw() - if err == nil { - switch { - case size == 32: - err = rlp.DecodeBytes(origin, &hn.Hash) - case size <= 8: - err = rlp.DecodeBytes(origin, &hn.Number) - default: - err = fmt.Errorf("invalid input size %d for origin", size) - } - } - return err -} - -// newBlockData is the network packet for the block propagation message. -type newBlockData struct { - Block *types.Block - TD *big.Int -} - -// sanityCheck verifies that the values are reasonable, as a DoS protection -func (request *newBlockData) sanityCheck() error { - if err := request.Block.SanityCheck(); err != nil { - return err - } - //TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times - // larger, it will still fit within 100 bits - if tdlen := request.TD.BitLen(); tdlen > 100 { - return fmt.Errorf("too large block TD: bitlen %d", tdlen) - } - return nil -} - -// blockBody represents the data content of a single block. -type blockBody struct { - Transactions []*types.Transaction // Transactions contained within a block - Uncles []*types.Header // Uncles contained within a block -} - -// blockBodiesData is the network packet for block content distribution. -type blockBodiesData []*blockBody diff --git a/eth/protocol_test.go b/eth/protocol_test.go deleted file mode 100644 index 331dd05ce1..0000000000 --- a/eth/protocol_test.go +++ /dev/null @@ -1,459 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "fmt" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" -) - -func init() { - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) -} - -var testAccount, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - -// Tests that handshake failures are detected and reported correctly. -func TestStatusMsgErrors63(t *testing.T) { - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) - var ( - genesis = pm.blockchain.Genesis() - head = pm.blockchain.CurrentHeader() - td = pm.blockchain.GetTd(head.Hash(), head.Number.Uint64()) - ) - defer pm.Stop() - - tests := []struct { - code uint64 - data interface{} - wantError error - }{ - { - code: TransactionMsg, data: []interface{}{}, - wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), - }, - { - code: StatusMsg, data: statusData63{10, DefaultConfig.NetworkId, td, head.Hash(), genesis.Hash()}, - wantError: errResp(ErrProtocolVersionMismatch, "10 (!= %d)", 63), - }, - { - code: StatusMsg, data: statusData63{63, 999, td, head.Hash(), genesis.Hash()}, - wantError: errResp(ErrNetworkIDMismatch, "999 (!= %d)", DefaultConfig.NetworkId), - }, - { - code: StatusMsg, data: statusData63{63, DefaultConfig.NetworkId, td, head.Hash(), common.Hash{3}}, - wantError: errResp(ErrGenesisMismatch, "0300000000000000 (!= %x)", genesis.Hash().Bytes()[:8]), - }, - } - for i, test := range tests { - p, errc := newTestPeer("peer", 63, pm, false) - // The send call might hang until reset because - // the protocol might not read the payload. - go p2p.Send(p.app, test.code, test.data) - - select { - case err := <-errc: - if err == nil { - t.Errorf("test %d: protocol returned nil error, want %q", i, test.wantError) - } else if err.Error() != test.wantError.Error() { - t.Errorf("test %d: wrong error: got %q, want %q", i, err, test.wantError) - } - case <-time.After(2 * time.Second): - t.Errorf("protocol did not shut down within 2 seconds") - } - p.close() - } -} - -func TestStatusMsgErrors64(t *testing.T) { - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) - var ( - genesis = pm.blockchain.Genesis() - head = pm.blockchain.CurrentHeader() - td = pm.blockchain.GetTd(head.Hash(), head.Number.Uint64()) - forkID = forkid.NewID(pm.blockchain.Config(), pm.blockchain.Genesis().Hash(), pm.blockchain.CurrentHeader().Number.Uint64()) - ) - defer pm.Stop() - - tests := []struct { - code uint64 - data interface{} - wantError error - }{ - { - code: TransactionMsg, data: []interface{}{}, - wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), - }, - { - code: StatusMsg, data: statusData{10, DefaultConfig.NetworkId, td, head.Hash(), genesis.Hash(), forkID}, - wantError: errResp(ErrProtocolVersionMismatch, "10 (!= %d)", 64), - }, - { - code: StatusMsg, data: statusData{64, 999, td, head.Hash(), genesis.Hash(), forkID}, - wantError: errResp(ErrNetworkIDMismatch, "999 (!= %d)", DefaultConfig.NetworkId), - }, - { - code: StatusMsg, data: statusData{64, DefaultConfig.NetworkId, td, head.Hash(), common.Hash{3}, forkID}, - wantError: errResp(ErrGenesisMismatch, "0300000000000000000000000000000000000000000000000000000000000000 (!= %x)", genesis.Hash()), - }, - { - code: StatusMsg, data: statusData{64, DefaultConfig.NetworkId, td, head.Hash(), genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}}, - wantError: errResp(ErrForkIDRejected, forkid.ErrLocalIncompatibleOrStale.Error()), - }, - } - for i, test := range tests { - p, errc := newTestPeer("peer", 64, pm, false) - // The send call might hang until reset because - // the protocol might not read the payload. - go p2p.Send(p.app, test.code, test.data) - - select { - case err := <-errc: - if err == nil { - t.Errorf("test %d: protocol returned nil error, want %q", i, test.wantError) - } else if err.Error() != test.wantError.Error() { - t.Errorf("test %d: wrong error: got %q, want %q", i, err, test.wantError) - } - case <-time.After(2 * time.Second): - t.Errorf("protocol did not shut down within 2 seconds") - } - p.close() - } -} - -func TestForkIDSplit(t *testing.T) { - var ( - engine = ethash.NewFaker() - - configNoFork = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(1)} - configProFork = ¶ms.ChainConfig{ - HomesteadBlock: big.NewInt(1), - EIP150Block: big.NewInt(2), - EIP155Block: big.NewInt(2), - EIP158Block: big.NewInt(2), - ByzantiumBlock: big.NewInt(3), - } - dbNoFork = rawdb.NewMemoryDatabase() - dbProFork = rawdb.NewMemoryDatabase() - - gspecNoFork = &core.Genesis{Config: configNoFork} - gspecProFork = &core.Genesis{Config: configProFork} - - genesisNoFork = gspecNoFork.MustCommit(dbNoFork) - genesisProFork = gspecProFork.MustCommit(dbProFork) - - chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil) - chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil) - - blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) - blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) - - ethNoFork, _ = NewProtocolManager(configNoFork, nil, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainNoFork, dbNoFork, 1, nil) - ethProFork, _ = NewProtocolManager(configProFork, nil, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainProFork, dbProFork, 1, nil) - ) - ethNoFork.Start(1000) - ethProFork.Start(1000) - - // Both nodes should allow the other to connect (same genesis, next fork is the same) - p2pNoFork, p2pProFork := p2p.MsgPipe() - peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) - peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) - - errc := make(chan error, 2) - go func() { errc <- ethNoFork.handle(peerProFork) }() - go func() { errc <- ethProFork.handle(peerNoFork) }() - - select { - case err := <-errc: - t.Fatalf("frontier nofork <-> profork failed: %v", err) - case <-time.After(250 * time.Millisecond): - p2pNoFork.Close() - p2pProFork.Close() - } - // Progress into Homestead. Fork's match, so we don't care what the future holds - chainNoFork.InsertChain(blocksNoFork[:1]) - chainProFork.InsertChain(blocksProFork[:1]) - - p2pNoFork, p2pProFork = p2p.MsgPipe() - peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) - peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) - - errc = make(chan error, 2) - go func() { errc <- ethNoFork.handle(peerProFork) }() - go func() { errc <- ethProFork.handle(peerNoFork) }() - - select { - case err := <-errc: - t.Fatalf("homestead nofork <-> profork failed: %v", err) - case <-time.After(250 * time.Millisecond): - p2pNoFork.Close() - p2pProFork.Close() - } - // Progress into Spurious. Forks mismatch, signalling differing chains, reject - chainNoFork.InsertChain(blocksNoFork[1:2]) - chainProFork.InsertChain(blocksProFork[1:2]) - - p2pNoFork, p2pProFork = p2p.MsgPipe() - peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) - peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) - - errc = make(chan error, 2) - go func() { errc <- ethNoFork.handle(peerProFork) }() - go func() { errc <- ethProFork.handle(peerNoFork) }() - - select { - case err := <-errc: - if want := errResp(ErrForkIDRejected, forkid.ErrLocalIncompatibleOrStale.Error()); err.Error() != want.Error() { - t.Fatalf("fork ID rejection error mismatch: have %v, want %v", err, want) - } - case <-time.After(250 * time.Millisecond): - t.Fatalf("split peers not rejected") - } -} - -// This test checks that received transactions are added to the local pool. -func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) } -func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) } -func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, 65) } - -func testRecvTransactions(t *testing.T, protocol int) { - txAdded := make(chan []*types.Transaction) - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, txAdded) - pm.acceptTxs = 1 // mark synced to accept transactions - p, _ := newTestPeer("peer", protocol, pm, true) - defer pm.Stop() - defer p.close() - - tx := newTestTransaction(testAccount, 0, 0) - if err := p2p.Send(p.app, TransactionMsg, []interface{}{tx}); err != nil { - t.Fatalf("send error: %v", err) - } - select { - case added := <-txAdded: - if len(added) != 1 { - t.Errorf("wrong number of added transactions: got %d, want 1", len(added)) - } else if added[0].Hash() != tx.Hash() { - t.Errorf("added wrong tx hash: got %v, want %v", added[0].Hash(), tx.Hash()) - } - case <-time.After(2 * time.Second): - t.Errorf("no NewTxsEvent received within 2 seconds") - } -} - -// This test checks that pending transactions are sent. -func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) } -func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) } -func TestSendTransactions65(t *testing.T) { testSendTransactions(t, 65) } - -func testSendTransactions(t *testing.T, protocol int) { - pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) - defer pm.Stop() - - // Fill the pool with big transactions (use a subscription to wait until all - // the transactions are announced to avoid spurious events causing extra - // broadcasts). - const txsize = txsyncPackSize / 10 - alltxs := make([]*types.Transaction, 100) - for nonce := range alltxs { - alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize) - } - pm.txpool.AddRemotes(alltxs) - time.Sleep(100 * time.Millisecond) // Wait until new tx even gets out of the system (lame) - - // Connect several peers. They should all receive the pending transactions. - var wg sync.WaitGroup - checktxs := func(p *testPeer) { - defer wg.Done() - defer p.close() - seen := make(map[common.Hash]bool) - for _, tx := range alltxs { - seen[tx.Hash()] = false - } - for n := 0; n < len(alltxs) && !t.Failed(); { - var forAllHashes func(callback func(hash common.Hash)) - switch protocol { - case 63: - fallthrough - case 64: - msg, err := p.app.ReadMsg() - if err != nil { - t.Errorf("%v: read error: %v", p.Peer, err) - continue - } else if msg.Code != TransactionMsg { - t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) - continue - } - var txs []*types.Transaction - if err := msg.Decode(&txs); err != nil { - t.Errorf("%v: %v", p.Peer, err) - continue - } - forAllHashes = func(callback func(hash common.Hash)) { - for _, tx := range txs { - callback(tx.Hash()) - } - } - case 65: - msg, err := p.app.ReadMsg() - if err != nil { - t.Errorf("%v: read error: %v", p.Peer, err) - continue - } else if msg.Code != NewPooledTransactionHashesMsg { - t.Errorf("%v: got code %d, want NewPooledTransactionHashesMsg", p.Peer, msg.Code) - continue - } - var hashes []common.Hash - if err := msg.Decode(&hashes); err != nil { - t.Errorf("%v: %v", p.Peer, err) - continue - } - forAllHashes = func(callback func(hash common.Hash)) { - for _, h := range hashes { - callback(h) - } - } - } - forAllHashes(func(hash common.Hash) { - seentx, want := seen[hash] - if seentx { - t.Errorf("%v: got tx more than once: %x", p.Peer, hash) - } - if !want { - t.Errorf("%v: got unexpected tx: %x", p.Peer, hash) - } - seen[hash] = true - n++ - }) - } - } - for i := 0; i < 3; i++ { - p, _ := newTestPeer(fmt.Sprintf("peer #%d", i), protocol, pm, true) - wg.Add(1) - go checktxs(p) - } - wg.Wait() -} - -func TestTransactionPropagation(t *testing.T) { testSyncTransaction(t, true) } -func TestTransactionAnnouncement(t *testing.T) { testSyncTransaction(t, false) } - -func testSyncTransaction(t *testing.T, propagtion bool) { - // Create a protocol manager for transaction fetcher and sender - pmFetcher, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil) - defer pmFetcher.Stop() - pmSender, _ := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil) - pmSender.broadcastTxAnnouncesOnly = !propagtion - defer pmSender.Stop() - - // Sync up the two peers - io1, io2 := p2p.MsgPipe() - - go pmSender.handle(pmSender.newPeer(65, p2p.NewPeer(enode.ID{}, "sender", nil), io2, pmSender.txpool.Get)) - go pmFetcher.handle(pmFetcher.newPeer(65, p2p.NewPeer(enode.ID{}, "fetcher", nil), io1, pmFetcher.txpool.Get)) - - time.Sleep(250 * time.Millisecond) - pmFetcher.doSync(peerToSyncOp(downloader.FullSync, pmFetcher.peers.BestPeer())) - atomic.StoreUint32(&pmFetcher.acceptTxs, 1) - - newTxs := make(chan core.NewTxsEvent, 1024) - sub := pmFetcher.txpool.SubscribeNewTxsEvent(newTxs) - defer sub.Unsubscribe() - - // Fill the pool with new transactions - alltxs := make([]*types.Transaction, 1024) - for nonce := range alltxs { - alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), 0) - } - pmSender.txpool.AddRemotes(alltxs) - - var got int -loop: - for { - select { - case ev := <-newTxs: - got += len(ev.Txs) - if got == 1024 { - break loop - } - case <-time.NewTimer(time.Second).C: - t.Fatal("Failed to retrieve all transaction") - } - } -} - -// Tests that the custom union field encoder and decoder works correctly. -func TestGetBlockHeadersDataEncodeDecode(t *testing.T) { - // Create a "random" hash for testing - var hash common.Hash - for i := range hash { - hash[i] = byte(i) - } - // Assemble some table driven tests - tests := []struct { - packet *getBlockHeadersData - fail bool - }{ - // Providing the origin as either a hash or a number should both work - {fail: false, packet: &getBlockHeadersData{Origin: hashOrNumber{Number: 314}}}, - {fail: false, packet: &getBlockHeadersData{Origin: hashOrNumber{Hash: hash}}}, - - // Providing arbitrary query field should also work - {fail: false, packet: &getBlockHeadersData{Origin: hashOrNumber{Number: 314}, Amount: 314, Skip: 1, Reverse: true}}, - {fail: false, packet: &getBlockHeadersData{Origin: hashOrNumber{Hash: hash}, Amount: 314, Skip: 1, Reverse: true}}, - - // Providing both the origin hash and origin number must fail - {fail: true, packet: &getBlockHeadersData{Origin: hashOrNumber{Hash: hash, Number: 314}}}, - } - // Iterate over each of the tests and try to encode and then decode - for i, tt := range tests { - bytes, err := rlp.EncodeToBytes(tt.packet) - if err != nil && !tt.fail { - t.Fatalf("test %d: failed to encode packet: %v", i, err) - } else if err == nil && tt.fail { - t.Fatalf("test %d: encode should have failed", i) - } - if !tt.fail { - packet := new(getBlockHeadersData) - if err := rlp.DecodeBytes(bytes, packet); err != nil { - t.Fatalf("test %d: failed to decode packet: %v", i, err) - } - if packet.Origin.Hash != tt.packet.Origin.Hash || packet.Origin.Number != tt.packet.Origin.Number || packet.Amount != tt.packet.Amount || - packet.Skip != tt.packet.Skip || packet.Reverse != tt.packet.Reverse { - t.Fatalf("test %d: encode decode mismatch: have %+v, want %+v", i, packet, tt.packet) - } - } - } -} diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go new file mode 100644 index 0000000000..2349398fae --- /dev/null +++ b/eth/protocols/eth/broadcast.go @@ -0,0 +1,195 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +const ( + // This is the target size for the packs of transactions or announcements. A + // pack can get larger than this if a single transactions exceeds this size. + maxTxPacketSize = 100 * 1024 +) + +// blockPropagation is a block propagation event, waiting for its turn in the +// broadcast queue. +type blockPropagation struct { + block *types.Block + td *big.Int +} + +// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *Peer) broadcastBlocks() { + for { + select { + case prop := <-p.queuedBlocks: + if err := p.SendNewBlock(prop.block, prop.td); err != nil { + return + } + p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) + + case block := <-p.queuedBlockAnns: + if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { + return + } + p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) + + case <-p.term: + return + } + } +} + +// broadcastTransactions is a write loop that schedules transaction broadcasts +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *Peer) broadcastTransactions() { + var ( + queue []common.Hash // Queue of hashes to broadcast as full transactions + done chan struct{} // Non-nil if background broadcaster is running + fail = make(chan error, 1) // Channel used to receive network error + failed bool // Flag whether a send failed, discard everything onward + ) + for { + // If there's no in-flight broadcast running, check if a new one is needed + if done == nil && len(queue) > 0 { + // Pile transaction until we reach our allowed network limit + var ( + hashes []common.Hash + txs []*types.Transaction + size common.StorageSize + ) + for i := 0; i < len(queue) && size < maxTxPacketSize; i++ { + if tx := p.txpool.Get(queue[i]); tx != nil { + txs = append(txs, tx) + size += tx.Size() + } + hashes = append(hashes, queue[i]) + } + queue = queue[:copy(queue, queue[len(hashes):])] + + // If there's anything available to transfer, fire up an async writer + if len(txs) > 0 { + done = make(chan struct{}) + go func() { + if err := p.SendTransactions(txs); err != nil { + fail <- err + return + } + close(done) + p.Log().Trace("Sent transactions", "count", len(txs)) + }() + } + } + // Transfer goroutine may or may not have been started, listen for events + select { + case hashes := <-p.txBroadcast: + // If the connection failed, discard all transaction events + if failed { + continue + } + // New batch of transactions to be broadcast, queue them (with cap) + queue = append(queue, hashes...) + if len(queue) > maxQueuedTxs { + // Fancy copy and resize to ensure buffer doesn't grow indefinitely + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] + } + + case <-done: + done = nil + + case <-fail: + failed = true + + case <-p.term: + return + } + } +} + +// announceTransactions is a write loop that schedules transaction broadcasts +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *Peer) announceTransactions() { + var ( + queue []common.Hash // Queue of hashes to announce as transaction stubs + done chan struct{} // Non-nil if background announcer is running + fail = make(chan error, 1) // Channel used to receive network error + failed bool // Flag whether a send failed, discard everything onward + ) + for { + // If there's no in-flight announce running, check if a new one is needed + if done == nil && len(queue) > 0 { + // Pile transaction hashes until we reach our allowed network limit + var ( + hashes []common.Hash + pending []common.Hash + size common.StorageSize + ) + for i := 0; i < len(queue) && size < maxTxPacketSize; i++ { + if p.txpool.Get(queue[i]) != nil { + pending = append(pending, queue[i]) + size += common.HashLength + } + hashes = append(hashes, queue[i]) + } + queue = queue[:copy(queue, queue[len(hashes):])] + + // If there's anything available to transfer, fire up an async writer + if len(pending) > 0 { + done = make(chan struct{}) + go func() { + if err := p.sendPooledTransactionHashes(pending); err != nil { + fail <- err + return + } + close(done) + p.Log().Trace("Sent transaction announcements", "count", len(pending)) + }() + } + } + // Transfer goroutine may or may not have been started, listen for events + select { + case hashes := <-p.txAnnounce: + // If the connection failed, discard all transaction events + if failed { + continue + } + // New batch of transactions to be broadcast, queue them (with cap) + queue = append(queue, hashes...) + if len(queue) > maxQueuedTxAnns { + // Fancy copy and resize to ensure buffer doesn't grow indefinitely + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] + } + + case <-done: + done = nil + + case <-fail: + failed = true + + case <-p.term: + return + } + } +} diff --git a/eth/protocols/eth/discovery.go b/eth/protocols/eth/discovery.go new file mode 100644 index 0000000000..025479b423 --- /dev/null +++ b/eth/protocols/eth/discovery.go @@ -0,0 +1,65 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" +) + +// enrEntry is the ENR entry which advertises `eth` protocol on the discovery. +type enrEntry struct { + ForkID forkid.ID // Fork identifier per EIP-2124 + + // Ignore additional fields (for forward compatibility). + Rest []rlp.RawValue `rlp:"tail"` +} + +// ENRKey implements enr.Entry. +func (e enrEntry) ENRKey() string { + return "eth" +} + +// StartENRUpdater starts the `eth` ENR updater loop, which listens for chain +// head events and updates the requested node record whenever a fork is passed. +func StartENRUpdater(chain *core.BlockChain, ln *enode.LocalNode) { + var newHead = make(chan core.ChainHeadEvent, 10) + sub := chain.SubscribeChainHeadEvent(newHead) + + go func() { + defer sub.Unsubscribe() + for { + select { + case <-newHead: + ln.Set(currentENREntry(chain)) + case <-sub.Err(): + // Would be nice to sync with Stop, but there is no + // good way to do that. + return + } + } + }() +} + +// currentENREntry constructs an `eth` ENR entry based on the current state of the chain. +func currentENREntry(chain *core.BlockChain) *enrEntry { + return &enrEntry{ + ForkID: forkid.NewID(chain.Config(), chain.Genesis().Hash(), chain.CurrentHeader().Number.Uint64()), + } +} diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go new file mode 100644 index 0000000000..25ddcd93ec --- /dev/null +++ b/eth/protocols/eth/handler.go @@ -0,0 +1,512 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +const ( + // softResponseLimit is the target maximum size of replies to data retrievals. + softResponseLimit = 2 * 1024 * 1024 + + // estHeaderSize is the approximate size of an RLP encoded block header. + estHeaderSize = 500 + + // maxHeadersServe is the maximum number of block headers to serve. This number + // is there to limit the number of disk lookups. + maxHeadersServe = 1024 + + // maxBodiesServe is the maximum number of block bodies to serve. This number + // is mostly there to limit the number of disk lookups. With 24KB block sizes + // nowadays, the practical limit will always be softResponseLimit. + maxBodiesServe = 1024 + + // maxNodeDataServe is the maximum number of state trie nodes to serve. This + // number is there to limit the number of disk lookups. + maxNodeDataServe = 1024 + + // maxReceiptsServe is the maximum number of block receipts to serve. This + // number is mostly there to limit the number of disk lookups. With block + // containing 200+ transactions nowadays, the practical limit will always + // be softResponseLimit. + maxReceiptsServe = 1024 +) + +// Handler is a callback to invoke from an outside runner after the boilerplate +// exchanges have passed. +type Handler func(peer *Peer) error + +// Backend defines the data retrieval methods to serve remote requests and the +// callback methods to invoke on remote deliveries. +type Backend interface { + // Chain retrieves the blockchain object to serve data. + Chain() *core.BlockChain + + // StateBloom retrieves the bloom filter - if any - for state trie nodes. + StateBloom() *trie.SyncBloom + + // TxPool retrieves the transaction pool object to serve data. + TxPool() TxPool + + // AcceptTxs retrieves whether transaction processing is enabled on the node + // or if inbound transactions should simply be dropped. + AcceptTxs() bool + + // RunPeer is invoked when a peer joins on the `eth` protocol. The handler + // should do any peer maintenance work, handshakes and validations. If all + // is passed, control should be given back to the `handler` to process the + // inbound messages going forward. + RunPeer(peer *Peer, handler Handler) error + + // PeerInfo retrieves all known `eth` information about a peer. + PeerInfo(id enode.ID) interface{} + + // Handle is a callback to be invoked when a data packet is received from + // the remote peer. Only packets not consumed by the protocol handler will + // be forwarded to the backend. + Handle(peer *Peer, packet Packet) error +} + +// TxPool defines the methods needed by the protocol handler to serve transactions. +type TxPool interface { + // Get retrieves the the transaction from the local txpool with the given hash. + Get(hash common.Hash) *types.Transaction +} + +// MakeProtocols constructs the P2P protocol definitions for `eth`. +func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2p.Protocol { + protocols := make([]p2p.Protocol, len(protocolVersions)) + for i, version := range protocolVersions { + version := version // Closure + + protocols[i] = p2p.Protocol{ + Name: protocolName, + Version: version, + Length: protocolLengths[version], + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := NewPeer(version, p, rw, backend.TxPool()) + defer peer.Close() + + return backend.RunPeer(peer, func(peer *Peer) error { + return Handle(backend, peer) + }) + }, + NodeInfo: func() interface{} { + return nodeInfo(backend.Chain(), network) + }, + PeerInfo: func(id enode.ID) interface{} { + return backend.PeerInfo(id) + }, + Attributes: []enr.Entry{currentENREntry(backend.Chain())}, + DialCandidates: dnsdisc, + } + } + return protocols +} + +// NodeInfo represents a short summary of the `eth` sub-protocol metadata +// known about the host peer. +type NodeInfo struct { + Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain + Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block + Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules + Head common.Hash `json:"head"` // Hex hash of the host's best owned block +} + +// nodeInfo retrieves some `eth` protocol metadata about the running host node. +func nodeInfo(chain *core.BlockChain, network uint64) *NodeInfo { + head := chain.CurrentBlock() + return &NodeInfo{ + Network: network, + Difficulty: chain.GetTd(head.Hash(), head.NumberU64()), + Genesis: chain.Genesis().Hash(), + Config: chain.Config(), + Head: head.Hash(), + } +} + +// Handle is invoked whenever an `eth` connection is made that successfully passes +// the protocol handshake. This method will keep processing messages until the +// connection is torn down. +func Handle(backend Backend, peer *Peer) error { + for { + if err := handleMessage(backend, peer); err != nil { + peer.Log().Debug("Message handling failed in `eth`", "err", err) + return err + } + } +} + +// handleMessage is invoked whenever an inbound message is received from a remote +// peer. The remote connection is torn down upon returning any error. +func handleMessage(backend Backend, peer *Peer) error { + // Read the next message from the remote peer, and ensure it's fully consumed + msg, err := peer.rw.ReadMsg() + if err != nil { + return err + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + defer msg.Discard() + + // Handle the message depending on its contents + switch { + case msg.Code == StatusMsg: + // Status messages should never arrive after the handshake + return fmt.Errorf("%w: uncontrolled status message", errExtraStatusMsg) + + // Block header query, collect the requested headers and reply + case msg.Code == GetBlockHeadersMsg: + // Decode the complex header query + var query GetBlockHeadersPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + hashMode := query.Origin.Hash != (common.Hash{}) + first := true + maxNonCanonical := uint64(100) + + // Gather headers until the fetch or network limits is reached + var ( + bytes common.StorageSize + headers []*types.Header + unknown bool + lookups int + ) + for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit && + len(headers) < maxHeadersServe && lookups < 2*maxHeadersServe { + lookups++ + // Retrieve the next header satisfying the query + var origin *types.Header + if hashMode { + if first { + first = false + origin = backend.Chain().GetHeaderByHash(query.Origin.Hash) + if origin != nil { + query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = backend.Chain().GetHeader(query.Origin.Hash, query.Origin.Number) + } + } else { + origin = backend.Chain().GetHeaderByNumber(query.Origin.Number) + } + if origin == nil { + break + } + headers = append(headers, origin) + bytes += estHeaderSize + + // Advance to the next header of the query + switch { + case hashMode && query.Reverse: + // Hash based traversal towards the genesis block + ancestor := query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + query.Origin.Hash, query.Origin.Number = backend.Chain().GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) + unknown = (query.Origin.Hash == common.Hash{}) + } + case hashMode && !query.Reverse: + // Hash based traversal towards the leaf block + var ( + current = origin.Number.Uint64() + next = current + query.Skip + 1 + ) + if next <= current { + infos, _ := json.MarshalIndent(peer.Peer.Info(), "", " ") + peer.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) + unknown = true + } else { + if header := backend.Chain().GetHeaderByNumber(next); header != nil { + nextHash := header.Hash() + expOldHash, _ := backend.Chain().GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) + if expOldHash == query.Origin.Hash { + query.Origin.Hash, query.Origin.Number = nextHash, next + } else { + unknown = true + } + } else { + unknown = true + } + } + case query.Reverse: + // Number based traversal towards the genesis block + if query.Origin.Number >= query.Skip+1 { + query.Origin.Number -= query.Skip + 1 + } else { + unknown = true + } + + case !query.Reverse: + // Number based traversal towards the leaf block + query.Origin.Number += query.Skip + 1 + } + } + return peer.SendBlockHeaders(headers) + + case msg.Code == BlockHeadersMsg: + // A batch of headers arrived to one of our previous requests + res := new(BlockHeadersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + + case msg.Code == GetBlockBodiesMsg: + // Decode the block body retrieval message + var query GetBlockBodiesPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Gather blocks until the fetch or network limits is reached + var ( + bytes int + bodies []rlp.RawValue + ) + for lookups, hash := range query { + if bytes >= softResponseLimit || len(bodies) >= maxBodiesServe || + lookups >= 2*maxBodiesServe { + break + } + if data := backend.Chain().GetBodyRLP(hash); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) + } + } + return peer.SendBlockBodiesRLP(bodies) + + case msg.Code == BlockBodiesMsg: + // A batch of block bodies arrived to one of our previous requests + res := new(BlockBodiesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + + case msg.Code == GetNodeDataMsg: + // Decode the trie node data retrieval message + var query GetNodeDataPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Gather state data until the fetch or network limits is reached + var ( + bytes int + nodes [][]byte + ) + for lookups, hash := range query { + if bytes >= softResponseLimit || len(nodes) >= maxNodeDataServe || + lookups >= 2*maxNodeDataServe { + break + } + // Retrieve the requested state entry + if bloom := backend.StateBloom(); bloom != nil && !bloom.Contains(hash[:]) { + // Only lookup the trie node if there's chance that we actually have it + continue + } + entry, err := backend.Chain().TrieNode(hash) + if len(entry) == 0 || err != nil { + // Read the contract code with prefix only to save unnecessary lookups. + entry, err = backend.Chain().ContractCodeWithPrefix(hash) + } + if err == nil && len(entry) > 0 { + nodes = append(nodes, entry) + bytes += len(entry) + } + } + return peer.SendNodeData(nodes) + + case msg.Code == NodeDataMsg: + // A batch of node state data arrived to one of our previous requests + res := new(NodeDataPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + + case msg.Code == GetReceiptsMsg: + // Decode the block receipts retrieval message + var query GetReceiptsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Gather state data until the fetch or network limits is reached + var ( + bytes int + receipts []rlp.RawValue + ) + for lookups, hash := range query { + if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || + lookups >= 2*maxReceiptsServe { + break + } + // Retrieve the requested block's receipts + results := backend.Chain().GetReceiptsByHash(hash) + if results == nil { + if header := backend.Chain().GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + continue + } + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(results); err != nil { + log.Error("Failed to encode receipt", "err", err) + } else { + receipts = append(receipts, encoded) + bytes += len(encoded) + } + } + return peer.SendReceiptsRLP(receipts) + + case msg.Code == ReceiptsMsg: + // A batch of receipts arrived to one of our previous requests + res := new(ReceiptsPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + + case msg.Code == NewBlockHashesMsg: + // A batch of new block announcements just arrived + ann := new(NewBlockHashesPacket) + if err := msg.Decode(ann); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Mark the hashes as present at the remote node + for _, block := range *ann { + peer.markBlock(block.Hash) + } + // Deliver them all to the backend for queuing + return backend.Handle(peer, ann) + + case msg.Code == NewBlockMsg: + // Retrieve and decode the propagated block + ann := new(NewBlockPacket) + if err := msg.Decode(ann); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { + log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) + break // TODO(karalabe): return error eventually, but wait a few releases + } + if hash := types.DeriveSha(ann.Block.Transactions(), trie.NewStackTrie(nil)); hash != ann.Block.TxHash() { + log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) + break // TODO(karalabe): return error eventually, but wait a few releases + } + if err := ann.sanityCheck(); err != nil { + return err + } + ann.Block.ReceivedAt = msg.ReceivedAt + ann.Block.ReceivedFrom = peer + + // Mark the peer as owning the block + peer.markBlock(ann.Block.Hash()) + + return backend.Handle(peer, ann) + + case msg.Code == NewPooledTransactionHashesMsg && peer.version >= ETH65: + // New transaction announcement arrived, make sure we have + // a valid and fresh chain to handle them + if !backend.AcceptTxs() { + break + } + ann := new(NewPooledTransactionHashesPacket) + if err := msg.Decode(ann); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Schedule all the unknown hashes for retrieval + for _, hash := range *ann { + peer.markTransaction(hash) + } + return backend.Handle(peer, ann) + + case msg.Code == GetPooledTransactionsMsg && peer.version >= ETH65: + // Decode the pooled transactions retrieval message + var query GetPooledTransactionsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Gather transactions until the fetch or network limits is reached + var ( + bytes int + hashes []common.Hash + txs []rlp.RawValue + ) + for _, hash := range query { + if bytes >= softResponseLimit { + break + } + // Retrieve the requested transaction, skipping if unknown to us + tx := backend.TxPool().Get(hash) + if tx == nil { + continue + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(tx); err != nil { + log.Error("Failed to encode transaction", "err", err) + } else { + hashes = append(hashes, hash) + txs = append(txs, encoded) + bytes += len(encoded) + } + } + return peer.SendPooledTransactionsRLP(hashes, txs) + + case msg.Code == TransactionsMsg || (msg.Code == PooledTransactionsMsg && peer.version >= ETH65): + // Transactions arrived, make sure we have a valid and fresh chain to handle them + if !backend.AcceptTxs() { + break + } + // Transactions can be processed, parse all of them and deliver to the pool + var txs []*types.Transaction + if err := msg.Decode(&txs); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + for i, tx := range txs { + // Validate and mark the remote transaction + if tx == nil { + return fmt.Errorf("%w: transaction %d is nil", errDecode, i) + } + peer.markTransaction(tx.Hash()) + } + if msg.Code == PooledTransactionsMsg { + return backend.Handle(peer, (*PooledTransactionsPacket)(&txs)) + } + return backend.Handle(peer, (*TransactionsPacket)(&txs)) + + default: + return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) + } + return nil +} diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go new file mode 100644 index 0000000000..65c4a10b0a --- /dev/null +++ b/eth/protocols/eth/handler_test.go @@ -0,0 +1,519 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "math" + "math/big" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +// testBackend is a mock implementation of the live Ethereum message handler. Its +// purpose is to allow testing the request/reply workflows and wire serialization +// in the `eth` protocol without actually doing any data processing. +type testBackend struct { + db ethdb.Database + chain *core.BlockChain + txpool *core.TxPool +} + +// newTestBackend creates an empty chain and wraps it into a mock backend. +func newTestBackend(blocks int) *testBackend { + return newTestBackendWithGenerator(blocks, nil) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func newTestBackendWithGenerator(blocks int, generator func(int, *core.BlockGen)) *testBackend { + // Create a database pre-initialize with a genesis block + db := rawdb.NewMemoryDatabase() + (&core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, + }).MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + + bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + txconfig := core.DefaultTxPoolConfig + txconfig.Journal = "" // Don't litter the disk with test journals + + return &testBackend{ + db: db, + chain: chain, + txpool: core.NewTxPool(txconfig, params.TestChainConfig, chain), + } +} + +// close tears down the transaction pool and chain behind the mock backend. +func (b *testBackend) close() { + b.txpool.Stop() + b.chain.Stop() +} + +func (b *testBackend) Chain() *core.BlockChain { return b.chain } +func (b *testBackend) StateBloom() *trie.SyncBloom { return nil } +func (b *testBackend) TxPool() TxPool { return b.txpool } + +func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { + // Normally the backend would do peer mainentance and handshakes. All that + // is omitted and we will just give control back to the handler. + return handler(peer) +} +func (b *testBackend) PeerInfo(enode.ID) interface{} { panic("not implemented") } + +func (b *testBackend) AcceptTxs() bool { + panic("data processing tests should be done in the handler package") +} +func (b *testBackend) Handle(*Peer, Packet) error { + panic("data processing tests should be done in the handler package") +} + +// Tests that block headers can be retrieved from a remote chain based on user queries. +func TestGetBlockHeaders64(t *testing.T) { testGetBlockHeaders(t, 64) } +func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, 65) } + +func testGetBlockHeaders(t *testing.T, protocol uint) { + t.Parallel() + + backend := newTestBackend(maxHeadersServe + 15) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + // Create a "random" unknown hash for testing + var unknown common.Hash + for i := range unknown { + unknown[i] = byte(i) + } + // Create a batch of tests for various scenarios + limit := uint64(maxHeadersServe) + tests := []struct { + query *GetBlockHeadersPacket // The query to execute for header retrieval + expect []common.Hash // The hashes of the block whose headers are expected + }{ + // A single random block should be retrievable by hash and number too + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: backend.chain.GetBlockByNumber(limit / 2).Hash()}, Amount: 1}, + []common.Hash{backend.chain.GetBlockByNumber(limit / 2).Hash()}, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: limit / 2}, Amount: 1}, + []common.Hash{backend.chain.GetBlockByNumber(limit / 2).Hash()}, + }, + // Multiple headers should be retrievable in both directions + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: limit / 2}, Amount: 3}, + []common.Hash{ + backend.chain.GetBlockByNumber(limit / 2).Hash(), + backend.chain.GetBlockByNumber(limit/2 + 1).Hash(), + backend.chain.GetBlockByNumber(limit/2 + 2).Hash(), + }, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: limit / 2}, Amount: 3, Reverse: true}, + []common.Hash{ + backend.chain.GetBlockByNumber(limit / 2).Hash(), + backend.chain.GetBlockByNumber(limit/2 - 1).Hash(), + backend.chain.GetBlockByNumber(limit/2 - 2).Hash(), + }, + }, + // Multiple headers with skip lists should be retrievable + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3}, + []common.Hash{ + backend.chain.GetBlockByNumber(limit / 2).Hash(), + backend.chain.GetBlockByNumber(limit/2 + 4).Hash(), + backend.chain.GetBlockByNumber(limit/2 + 8).Hash(), + }, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3, Reverse: true}, + []common.Hash{ + backend.chain.GetBlockByNumber(limit / 2).Hash(), + backend.chain.GetBlockByNumber(limit/2 - 4).Hash(), + backend.chain.GetBlockByNumber(limit/2 - 8).Hash(), + }, + }, + // The chain endpoints should be retrievable + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: 0}, Amount: 1}, + []common.Hash{backend.chain.GetBlockByNumber(0).Hash()}, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: backend.chain.CurrentBlock().NumberU64()}, Amount: 1}, + []common.Hash{backend.chain.CurrentBlock().Hash()}, + }, + // Ensure protocol limits are honored + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: backend.chain.CurrentBlock().NumberU64() - 1}, Amount: limit + 10, Reverse: true}, + backend.chain.GetBlockHashesFromHash(backend.chain.CurrentBlock().Hash(), limit), + }, + // Check that requesting more than available is handled gracefully + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: backend.chain.CurrentBlock().NumberU64() - 4}, Skip: 3, Amount: 3}, + []common.Hash{ + backend.chain.GetBlockByNumber(backend.chain.CurrentBlock().NumberU64() - 4).Hash(), + backend.chain.GetBlockByNumber(backend.chain.CurrentBlock().NumberU64()).Hash(), + }, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: 4}, Skip: 3, Amount: 3, Reverse: true}, + []common.Hash{ + backend.chain.GetBlockByNumber(4).Hash(), + backend.chain.GetBlockByNumber(0).Hash(), + }, + }, + // Check that requesting more than available is handled gracefully, even if mid skip + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: backend.chain.CurrentBlock().NumberU64() - 4}, Skip: 2, Amount: 3}, + []common.Hash{ + backend.chain.GetBlockByNumber(backend.chain.CurrentBlock().NumberU64() - 4).Hash(), + backend.chain.GetBlockByNumber(backend.chain.CurrentBlock().NumberU64() - 1).Hash(), + }, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: 4}, Skip: 2, Amount: 3, Reverse: true}, + []common.Hash{ + backend.chain.GetBlockByNumber(4).Hash(), + backend.chain.GetBlockByNumber(1).Hash(), + }, + }, + // Check a corner case where requesting more can iterate past the endpoints + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: 2}, Amount: 5, Reverse: true}, + []common.Hash{ + backend.chain.GetBlockByNumber(2).Hash(), + backend.chain.GetBlockByNumber(1).Hash(), + backend.chain.GetBlockByNumber(0).Hash(), + }, + }, + // Check a corner case where skipping overflow loops back into the chain start + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: backend.chain.GetBlockByNumber(3).Hash()}, Amount: 2, Reverse: false, Skip: math.MaxUint64 - 1}, + []common.Hash{ + backend.chain.GetBlockByNumber(3).Hash(), + }, + }, + // Check a corner case where skipping overflow loops back to the same header + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: backend.chain.GetBlockByNumber(1).Hash()}, Amount: 2, Reverse: false, Skip: math.MaxUint64}, + []common.Hash{ + backend.chain.GetBlockByNumber(1).Hash(), + }, + }, + // Check that non existing headers aren't returned + { + &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: unknown}, Amount: 1}, + []common.Hash{}, + }, { + &GetBlockHeadersPacket{Origin: HashOrNumber{Number: backend.chain.CurrentBlock().NumberU64() + 1}, Amount: 1}, + []common.Hash{}, + }, + } + // Run each of the tests and verify the results against the chain + for i, tt := range tests { + // Collect the headers to expect in the response + var headers []*types.Header + for _, hash := range tt.expect { + headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) + } + // Send the hash request and verify the response + p2p.Send(peer.app, 0x03, tt.query) + if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + // If the test used number origins, repeat with hashes as the too + if tt.query.Origin.Hash == (common.Hash{}) { + if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { + tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 + + p2p.Send(peer.app, 0x03, tt.query) + if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } + } + } +} + +// Tests that block contents can be retrieved from a remote chain based on their hashes. +func TestGetBlockBodies64(t *testing.T) { testGetBlockBodies(t, 64) } +func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, 65) } + +func testGetBlockBodies(t *testing.T, protocol uint) { + t.Parallel() + + backend := newTestBackend(maxBodiesServe + 15) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + // Create a batch of tests for various scenarios + limit := maxBodiesServe + tests := []struct { + random int // Number of blocks to fetch randomly from the chain + explicit []common.Hash // Explicitly requested blocks + available []bool // Availability of explicitly requested blocks + expected int // Total number of existing blocks to expect + }{ + {1, nil, nil, 1}, // A single random block should be retrievable + {10, nil, nil, 10}, // Multiple random blocks should be retrievable + {limit, nil, nil, limit}, // The maximum possible blocks should be retrievable + {limit + 1, nil, nil, limit}, // No more than the possible block count should be returned + {0, []common.Hash{backend.chain.Genesis().Hash()}, []bool{true}, 1}, // The genesis block should be retrievable + {0, []common.Hash{backend.chain.CurrentBlock().Hash()}, []bool{true}, 1}, // The chains head block should be retrievable + {0, []common.Hash{{}}, []bool{false}, 0}, // A non existent block should not be returned + + // Existing and non-existing blocks interleaved should not cause problems + {0, []common.Hash{ + {}, + backend.chain.GetBlockByNumber(1).Hash(), + {}, + backend.chain.GetBlockByNumber(10).Hash(), + {}, + backend.chain.GetBlockByNumber(100).Hash(), + {}, + }, []bool{false, true, false, true, false, true, false}, 3}, + } + // Run each of the tests and verify the results against the chain + for i, tt := range tests { + // Collect the hashes to request, and the response to expectva + var ( + hashes []common.Hash + bodies []*BlockBody + seen = make(map[int64]bool) + ) + for j := 0; j < tt.random; j++ { + for { + num := rand.Int63n(int64(backend.chain.CurrentBlock().NumberU64())) + if !seen[num] { + seen[num] = true + + block := backend.chain.GetBlockByNumber(uint64(num)) + hashes = append(hashes, block.Hash()) + if len(bodies) < tt.expected { + bodies = append(bodies, &BlockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) + } + break + } + } + } + for j, hash := range tt.explicit { + hashes = append(hashes, hash) + if tt.available[j] && len(bodies) < tt.expected { + block := backend.chain.GetBlockByHash(hash) + bodies = append(bodies, &BlockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) + } + } + // Send the hash request and verify the response + p2p.Send(peer.app, 0x05, hashes) + if err := p2p.ExpectMsg(peer.app, 0x06, bodies); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } + } +} + +// Tests that the state trie nodes can be retrieved based on hashes. +func TestGetNodeData64(t *testing.T) { testGetNodeData(t, 64) } +func TestGetNodeData65(t *testing.T) { testGetNodeData(t, 65) } + +func testGetNodeData(t *testing.T, protocol uint) { + t.Parallel() + + // Define three accounts to simulate transactions with + acc1Key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + acc2Key, _ := crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + acc1Addr := crypto.PubkeyToAddress(acc1Key.PublicKey) + acc2Addr := crypto.PubkeyToAddress(acc2Key.PublicKey) + + signer := types.HomesteadSigner{} + // Create a chain generator with some simple transactions (blatantly stolen from @fjl/chain_markets_test) + generator := func(i int, block *core.BlockGen) { + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // acc1Addr passes it on to account #2. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + block.AddTx(tx1) + block.AddTx(tx2) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(acc2Addr) + block.SetExtra([]byte("yeehaw")) + case 3: + // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). + b2 := block.PrevBlock(1).Header() + b2.Extra = []byte("foo") + block.AddUncle(b2) + b3 := block.PrevBlock(2).Header() + b3.Extra = []byte("foo") + block.AddUncle(b3) + } + } + // Assemble the test environment + backend := newTestBackendWithGenerator(4, generator) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + // Fetch for now the entire chain db + var hashes []common.Hash + + it := backend.db.NewIterator(nil, nil) + for it.Next() { + if key := it.Key(); len(key) == common.HashLength { + hashes = append(hashes, common.BytesToHash(key)) + } + } + it.Release() + + p2p.Send(peer.app, 0x0d, hashes) + msg, err := peer.app.ReadMsg() + if err != nil { + t.Fatalf("failed to read node data response: %v", err) + } + if msg.Code != 0x0e { + t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, 0x0c) + } + var data [][]byte + if err := msg.Decode(&data); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + // Verify that all hashes correspond to the requested data, and reconstruct a state tree + for i, want := range hashes { + if hash := crypto.Keccak256Hash(data[i]); hash != want { + t.Errorf("data hash mismatch: have %x, want %x", hash, want) + } + } + statedb := rawdb.NewMemoryDatabase() + for i := 0; i < len(data); i++ { + statedb.Put(hashes[i].Bytes(), data[i]) + } + accounts := []common.Address{testAddr, acc1Addr, acc2Addr} + for i := uint64(0); i <= backend.chain.CurrentBlock().NumberU64(); i++ { + trie, _ := state.New(backend.chain.GetBlockByNumber(i).Root(), state.NewDatabase(statedb), nil) + + for j, acc := range accounts { + state, _ := backend.chain.State() + bw := state.GetBalance(acc) + bh := trie.GetBalance(acc) + + if (bw != nil && bh == nil) || (bw == nil && bh != nil) { + t.Errorf("test %d, account %d: balance mismatch: have %v, want %v", i, j, bh, bw) + } + if bw != nil && bh != nil && bw.Cmp(bw) != 0 { + t.Errorf("test %d, account %d: balance mismatch: have %v, want %v", i, j, bh, bw) + } + } + } +} + +// Tests that the transaction receipts can be retrieved based on hashes. +func TestGetBlockReceipts64(t *testing.T) { testGetBlockReceipts(t, 64) } +func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, 65) } + +func testGetBlockReceipts(t *testing.T, protocol uint) { + t.Parallel() + + // Define three accounts to simulate transactions with + acc1Key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + acc2Key, _ := crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + acc1Addr := crypto.PubkeyToAddress(acc1Key.PublicKey) + acc2Addr := crypto.PubkeyToAddress(acc2Key.PublicKey) + + signer := types.HomesteadSigner{} + // Create a chain generator with some simple transactions (blatantly stolen from @fjl/chain_markets_test) + generator := func(i int, block *core.BlockGen) { + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // acc1Addr passes it on to account #2. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + block.AddTx(tx1) + block.AddTx(tx2) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(acc2Addr) + block.SetExtra([]byte("yeehaw")) + case 3: + // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). + b2 := block.PrevBlock(1).Header() + b2.Extra = []byte("foo") + block.AddUncle(b2) + b3 := block.PrevBlock(2).Header() + b3.Extra = []byte("foo") + block.AddUncle(b3) + } + } + // Assemble the test environment + backend := newTestBackendWithGenerator(4, generator) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + // Collect the hashes to request, and the response to expect + var ( + hashes []common.Hash + receipts []types.Receipts + ) + for i := uint64(0); i <= backend.chain.CurrentBlock().NumberU64(); i++ { + block := backend.chain.GetBlockByNumber(i) + + hashes = append(hashes, block.Hash()) + receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) + } + // Send the hash request and verify the response + p2p.Send(peer.app, 0x0f, hashes) + if err := p2p.ExpectMsg(peer.app, 0x10, receipts); err != nil { + t.Errorf("receipts mismatch: %v", err) + } +} diff --git a/eth/protocols/eth/handshake.go b/eth/protocols/eth/handshake.go new file mode 100644 index 0000000000..57a4e0bc34 --- /dev/null +++ b/eth/protocols/eth/handshake.go @@ -0,0 +1,107 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/p2p" +) + +const ( + // handshakeTimeout is the maximum allowed time for the `eth` handshake to + // complete before dropping the connection.= as malicious. + handshakeTimeout = 5 * time.Second +) + +// Handshake executes the eth protocol handshake, negotiating version number, +// network IDs, difficulties, head and genesis blocks. +func (p *Peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { + // Send out own handshake in a new thread + errc := make(chan error, 2) + + var status StatusPacket // safe to read after two values have been received from errc + + go func() { + errc <- p2p.Send(p.rw, StatusMsg, &StatusPacket{ + ProtocolVersion: uint32(p.version), + NetworkID: network, + TD: td, + Head: head, + Genesis: genesis, + ForkID: forkID, + }) + }() + go func() { + errc <- p.readStatus(network, &status, genesis, forkFilter) + }() + timeout := time.NewTimer(handshakeTimeout) + defer timeout.Stop() + for i := 0; i < 2; i++ { + select { + case err := <-errc: + if err != nil { + return err + } + case <-timeout.C: + return p2p.DiscReadTimeout + } + } + p.td, p.head = status.TD, status.Head + + // TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times + // larger, it will still fit within 100 bits + if tdlen := p.td.BitLen(); tdlen > 100 { + return fmt.Errorf("too large total difficulty: bitlen %d", tdlen) + } + return nil +} + +// readStatus reads the remote handshake message. +func (p *Peer) readStatus(network uint64, status *StatusPacket, genesis common.Hash, forkFilter forkid.Filter) error { + msg, err := p.rw.ReadMsg() + if err != nil { + return err + } + if msg.Code != StatusMsg { + return fmt.Errorf("%w: first msg has code %x (!= %x)", errNoStatusMsg, msg.Code, StatusMsg) + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + // Decode the handshake and make sure everything matches + if err := msg.Decode(&status); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if status.NetworkID != network { + return fmt.Errorf("%w: %d (!= %d)", errNetworkIDMismatch, status.NetworkID, network) + } + if uint(status.ProtocolVersion) != p.version { + return fmt.Errorf("%w: %d (!= %d)", errProtocolVersionMismatch, status.ProtocolVersion, p.version) + } + if status.Genesis != genesis { + return fmt.Errorf("%w: %x (!= %x)", errGenesisMismatch, status.Genesis, genesis) + } + if err := forkFilter(status.ForkID); err != nil { + return fmt.Errorf("%w: %v", errForkIDRejected, err) + } + return nil +} diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go new file mode 100644 index 0000000000..65f9a00064 --- /dev/null +++ b/eth/protocols/eth/handshake_test.go @@ -0,0 +1,91 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "errors" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// Tests that handshake failures are detected and reported correctly. +func TestHandshake64(t *testing.T) { testHandshake(t, 64) } +func TestHandshake65(t *testing.T) { testHandshake(t, 65) } + +func testHandshake(t *testing.T, protocol uint) { + t.Parallel() + + // Create a test backend only to have some valid genesis chain + backend := newTestBackend(3) + defer backend.close() + + var ( + genesis = backend.chain.Genesis() + head = backend.chain.CurrentBlock() + td = backend.chain.GetTd(head.Hash(), head.NumberU64()) + forkID = forkid.NewID(backend.chain.Config(), backend.chain.Genesis().Hash(), backend.chain.CurrentHeader().Number.Uint64()) + ) + tests := []struct { + code uint64 + data interface{} + want error + }{ + { + code: TransactionsMsg, data: []interface{}{}, + want: errNoStatusMsg, + }, + { + code: StatusMsg, data: StatusPacket{10, 1, td, head.Hash(), genesis.Hash(), forkID}, + want: errProtocolVersionMismatch, + }, + { + code: StatusMsg, data: StatusPacket{uint32(protocol), 999, td, head.Hash(), genesis.Hash(), forkID}, + want: errNetworkIDMismatch, + }, + { + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, td, head.Hash(), common.Hash{3}, forkID}, + want: errGenesisMismatch, + }, + { + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, td, head.Hash(), genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}}, + want: errForkIDRejected, + }, + } + for i, test := range tests { + // Create the two peers to shake with each other + app, net := p2p.MsgPipe() + defer app.Close() + defer net.Close() + + peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil) + defer peer.Close() + + // Send the junk test with one peer, check the handshake failure + go p2p.Send(app, test.code, test.data) + + err := peer.Handshake(1, td, head.Hash(), genesis.Hash(), forkID, forkid.NewFilter(backend.chain)) + if err == nil { + t.Errorf("test %d: protocol returned nil error, want %q", i, test.want) + } else if !errors.Is(err, test.want) { + t.Errorf("test %d: wrong error: got %q, want %q", i, err, test.want) + } + } +} diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go new file mode 100644 index 0000000000..735ef78ce7 --- /dev/null +++ b/eth/protocols/eth/peer.go @@ -0,0 +1,429 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "math/big" + "sync" + + mapset "github.com/deckarep/golang-set" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" +) + +const ( + // maxKnownTxs is the maximum transactions hashes to keep in the known list + // before starting to randomly evict them. + maxKnownTxs = 32768 + + // maxKnownBlocks is the maximum block hashes to keep in the known list + // before starting to randomly evict them. + maxKnownBlocks = 1024 + + // maxQueuedTxs is the maximum number of transactions to queue up before dropping + // older broadcasts. + maxQueuedTxs = 4096 + + // maxQueuedTxAnns is the maximum number of transaction announcements to queue up + // before dropping older announcements. + maxQueuedTxAnns = 4096 + + // maxQueuedBlocks is the maximum number of block propagations to queue up before + // dropping broadcasts. There's not much point in queueing stale blocks, so a few + // that might cover uncles should be enough. + maxQueuedBlocks = 4 + + // maxQueuedBlockAnns is the maximum number of block announcements to queue up before + // dropping broadcasts. Similarly to block propagations, there's no point to queue + // above some healthy uncle limit, so use that. + maxQueuedBlockAnns = 4 +) + +// max is a helper function which returns the larger of the two given integers. +func max(a, b int) int { + if a > b { + return a + } + return b +} + +// Peer is a collection of relevant information we have about a `eth` peer. +type Peer struct { + id string // Unique ID for the peer, cached + + *p2p.Peer // The embedded P2P package peer + rw p2p.MsgReadWriter // Input/output streams for snap + version uint // Protocol version negotiated + + head common.Hash // Latest advertised head block hash + td *big.Int // Latest advertised head block total difficulty + + knownBlocks mapset.Set // Set of block hashes known to be known by this peer + queuedBlocks chan *blockPropagation // Queue of blocks to broadcast to the peer + queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer + + txpool TxPool // Transaction pool used by the broadcasters for liveness checks + knownTxs mapset.Set // Set of transaction hashes known to be known by this peer + txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests + txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + + term chan struct{} // Termination channel to stop the broadcasters + lock sync.RWMutex // Mutex protecting the internal fields +} + +// NewPeer create a wrapper for a network connection and negotiated protocol +// version. +func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { + peer := &Peer{ + id: p.ID().String(), + Peer: p, + rw: rw, + version: version, + knownTxs: mapset.NewSet(), + knownBlocks: mapset.NewSet(), + queuedBlocks: make(chan *blockPropagation, maxQueuedBlocks), + queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), + txBroadcast: make(chan []common.Hash), + txAnnounce: make(chan []common.Hash), + txpool: txpool, + term: make(chan struct{}), + } + // Start up all the broadcasters + go peer.broadcastBlocks() + go peer.broadcastTransactions() + if version >= ETH65 { + go peer.announceTransactions() + } + return peer +} + +// Close signals the broadcast goroutine to terminate. Only ever call this if +// you created the peer yourself via NewPeer. Otherwise let whoever created it +// clean it up! +func (p *Peer) Close() { + close(p.term) +} + +// ID retrieves the peer's unique identifier. +func (p *Peer) ID() string { + return p.id +} + +// Version retrieves the peer's negoatiated `eth` protocol version. +func (p *Peer) Version() uint { + return p.version +} + +// Head retrieves the current head hash and total difficulty of the peer. +func (p *Peer) Head() (hash common.Hash, td *big.Int) { + p.lock.RLock() + defer p.lock.RUnlock() + + copy(hash[:], p.head[:]) + return hash, new(big.Int).Set(p.td) +} + +// SetHead updates the head hash and total difficulty of the peer. +func (p *Peer) SetHead(hash common.Hash, td *big.Int) { + p.lock.Lock() + defer p.lock.Unlock() + + copy(p.head[:], hash[:]) + p.td.Set(td) +} + +// KnownBlock returns whether peer is known to already have a block. +func (p *Peer) KnownBlock(hash common.Hash) bool { + return p.knownBlocks.Contains(hash) +} + +// KnownTransaction returns whether peer is known to already have a transaction. +func (p *Peer) KnownTransaction(hash common.Hash) bool { + return p.knownTxs.Contains(hash) +} + +// markBlock marks a block as known for the peer, ensuring that the block will +// never be propagated to this particular peer. +func (p *Peer) markBlock(hash common.Hash) { + // If we reached the memory allowance, drop a previously known block hash + for p.knownBlocks.Cardinality() >= maxKnownBlocks { + p.knownBlocks.Pop() + } + p.knownBlocks.Add(hash) +} + +// markTransaction marks a transaction as known for the peer, ensuring that it +// will never be propagated to this particular peer. +func (p *Peer) markTransaction(hash common.Hash) { + // If we reached the memory allowance, drop a previously known transaction hash + for p.knownTxs.Cardinality() >= maxKnownTxs { + p.knownTxs.Pop() + } + p.knownTxs.Add(hash) +} + +// SendTransactions sends transactions to the peer and includes the hashes +// in its transaction hash set for future reference. +// +// This method is a helper used by the async transaction sender. Don't call it +// directly as the queueing (memory) and transmission (bandwidth) costs should +// not be managed directly. +// +// The reasons this is public is to allow packages using this protocol to write +// tests that directly send messages without having to do the asyn queueing. +func (p *Peer) SendTransactions(txs types.Transactions) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { + p.knownTxs.Pop() + } + for _, tx := range txs { + p.knownTxs.Add(tx.Hash()) + } + return p2p.Send(p.rw, TransactionsMsg, txs) +} + +// AsyncSendTransactions queues a list of transactions (by hash) to eventually +// propagate to a remote peer. The number of pending sends are capped (new ones +// will force old sends to be dropped) +func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { + select { + case p.txBroadcast <- hashes: + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + case <-p.term: + p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) + } +} + +// sendPooledTransactionHashes sends transaction hashes to the peer and includes +// them in its transaction hash set for future reference. +// +// This method is a helper used by the async transaction announcer. Don't call it +// directly as the queueing (memory) and transmission (bandwidth) costs should +// not be managed directly. +func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket(hashes)) +} + +// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually +// announce to a remote peer. The number of pending sends are capped (new ones +// will force old sends to be dropped) +func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { + select { + case p.txAnnounce <- hashes: + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + case <-p.term: + p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) + } +} + +// SendPooledTransactionsRLP sends requested transactions to the peer and adds the +// hashes in its transaction hash set for future reference. +// +// Note, the method assumes the hashes are correct and correspond to the list of +// transactions being sent. +func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding +} + +// SendNewBlockHashes announces the availability of a number of blocks through +// a hash notification. +func (p *Peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { + // Mark all the block hashes as known, but ensure we don't overflow our limits + for p.knownBlocks.Cardinality() > max(0, maxKnownBlocks-len(hashes)) { + p.knownBlocks.Pop() + } + for _, hash := range hashes { + p.knownBlocks.Add(hash) + } + request := make(NewBlockHashesPacket, len(hashes)) + for i := 0; i < len(hashes); i++ { + request[i].Hash = hashes[i] + request[i].Number = numbers[i] + } + return p2p.Send(p.rw, NewBlockHashesMsg, request) +} + +// AsyncSendNewBlockHash queues the availability of a block for propagation to a +// remote peer. If the peer's broadcast queue is full, the event is silently +// dropped. +func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { + select { + case p.queuedBlockAnns <- block: + // Mark all the block hash as known, but ensure we don't overflow our limits + for p.knownBlocks.Cardinality() >= maxKnownBlocks { + p.knownBlocks.Pop() + } + p.knownBlocks.Add(block.Hash()) + default: + p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) + } +} + +// SendNewBlock propagates an entire block to a remote peer. +func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { + // Mark all the block hash as known, but ensure we don't overflow our limits + for p.knownBlocks.Cardinality() >= maxKnownBlocks { + p.knownBlocks.Pop() + } + p.knownBlocks.Add(block.Hash()) + return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{block, td}) +} + +// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If +// the peer's broadcast queue is full, the event is silently dropped. +func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { + select { + case p.queuedBlocks <- &blockPropagation{block: block, td: td}: + // Mark all the block hash as known, but ensure we don't overflow our limits + for p.knownBlocks.Cardinality() >= maxKnownBlocks { + p.knownBlocks.Pop() + } + p.knownBlocks.Add(block.Hash()) + default: + p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) + } +} + +// SendBlockHeaders sends a batch of block headers to the remote peer. +func (p *Peer) SendBlockHeaders(headers []*types.Header) error { + return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) +} + +// SendBlockBodies sends a batch of block contents to the remote peer. +func (p *Peer) SendBlockBodies(bodies []*BlockBody) error { + return p2p.Send(p.rw, BlockBodiesMsg, BlockBodiesPacket(bodies)) +} + +// SendBlockBodiesRLP sends a batch of block contents to the remote peer from +// an already RLP encoded format. +func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { + return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding +} + +// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the +// hashes requested. +func (p *Peer) SendNodeData(data [][]byte) error { + return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) +} + +// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the +// ones requested from an already RLP encoded format. +func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { + return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding +} + +// RequestOneHeader is a wrapper around the header query functions to fetch a +// single header. It is used solely by the fetcher. +func (p *Peer) RequestOneHeader(hash common.Hash) error { + p.Log().Debug("Fetching single header", "hash", hash) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + }) +} + +// RequestHeadersByHash fetches a batch of blocks' headers corresponding to the +// specified header query, based on the hash of an origin block. +func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { + p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }) +} + +// RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the +// specified header query, based on the number of an origin block. +func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }) +} + +// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side +// of the RequestHeadersByNumber operation. +func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + req := &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) +} + +// RequestBodies fetches a batch of blocks' bodies corresponding to the hashes +// specified. +func (p *Peer) RequestBodies(hashes []common.Hash) error { + p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) + return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) +} + +// RequestNodeData fetches a batch of arbitrary data from a node's known state +// data, corresponding to the specified hashes. +func (p *Peer) RequestNodeData(hashes []common.Hash) error { + p.Log().Debug("Fetching batch of state data", "count", len(hashes)) + return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) +} + +// RequestReceipts fetches a batch of transaction receipts from a remote node. +func (p *Peer) RequestReceipts(hashes []common.Hash) error { + p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) + return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) +} + +// RequestTxs fetches a batch of transactions from a remote node. +func (p *Peer) RequestTxs(hashes []common.Hash) error { + p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) +} diff --git a/eth/protocols/eth/peer_test.go b/eth/protocols/eth/peer_test.go new file mode 100644 index 0000000000..70e9959f82 --- /dev/null +++ b/eth/protocols/eth/peer_test.go @@ -0,0 +1,61 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains some shares testing functionality, common to multiple +// different files and modules being tested. + +package eth + +import ( + "crypto/rand" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// testPeer is a simulated peer to allow testing direct network calls. +type testPeer struct { + *Peer + + net p2p.MsgReadWriter // Network layer reader/writer to simulate remote messaging + app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side +} + +// newTestPeer creates a new peer registered at the given data backend. +func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + // Start the peer on a new thread + var id enode.ID + rand.Read(id[:]) + + peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool()) + errc := make(chan error, 1) + go func() { + errc <- backend.RunPeer(peer, func(peer *Peer) error { + return Handle(backend, peer) + }) + }() + return &testPeer{app: app, net: net, Peer: peer}, errc +} + +// close terminates the local side of the peer, notifying the remote protocol +// manager of termination. +func (p *testPeer) close() { + p.Peer.Close() + p.app.Close() +} diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go new file mode 100644 index 0000000000..63d3494ec4 --- /dev/null +++ b/eth/protocols/eth/protocol.go @@ -0,0 +1,279 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "errors" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Constants to match up protocol versions and messages +const ( + ETH64 = 64 + ETH65 = 65 +) + +// protocolName is the official short name of the `eth` protocol used during +// devp2p capability negotiation. +const protocolName = "eth" + +// protocolVersions are the supported versions of the `eth` protocol (first +// is primary). +var protocolVersions = []uint{ETH65, ETH64} + +// protocolLengths are the number of implemented message corresponding to +// different protocol versions. +var protocolLengths = map[uint]uint64{ETH65: 17, ETH64: 17} + +// maxMessageSize is the maximum cap on the size of a protocol message. +const maxMessageSize = 10 * 1024 * 1024 + +const ( + // Protocol messages in eth/64 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 + + // Protocol messages overloaded in eth/65 + NewPooledTransactionHashesMsg = 0x08 + GetPooledTransactionsMsg = 0x09 + PooledTransactionsMsg = 0x0a +) + +var ( + errNoStatusMsg = errors.New("no status message") + errMsgTooLarge = errors.New("message too long") + errDecode = errors.New("invalid message") + errInvalidMsgCode = errors.New("invalid message code") + errProtocolVersionMismatch = errors.New("protocol version mismatch") + errNetworkIDMismatch = errors.New("network ID mismatch") + errGenesisMismatch = errors.New("genesis mismatch") + errForkIDRejected = errors.New("fork ID rejected") + errExtraStatusMsg = errors.New("extra status message") +) + +// Packet represents a p2p message in the `eth` protocol. +type Packet interface { + Name() string // Name returns a string corresponding to the message type. + Kind() byte // Kind returns the message type. +} + +// StatusPacket is the network packet for the status message for eth/64 and later. +type StatusPacket struct { + ProtocolVersion uint32 + NetworkID uint64 + TD *big.Int + Head common.Hash + Genesis common.Hash + ForkID forkid.ID +} + +// NewBlockHashesPacket is the network packet for the block announcements. +type NewBlockHashesPacket []struct { + Hash common.Hash // Hash of one particular block being announced + Number uint64 // Number of one particular block being announced +} + +// Unpack retrieves the block hashes and numbers from the announcement packet +// and returns them in a split flat format that's more consistent with the +// internal data structures. +func (p *NewBlockHashesPacket) Unpack() ([]common.Hash, []uint64) { + var ( + hashes = make([]common.Hash, len(*p)) + numbers = make([]uint64, len(*p)) + ) + for i, body := range *p { + hashes[i], numbers[i] = body.Hash, body.Number + } + return hashes, numbers +} + +// TransactionsPacket is the network packet for broadcasting new transactions. +type TransactionsPacket []*types.Transaction + +// GetBlockHeadersPacket represents a block header query. +type GetBlockHeadersPacket struct { + Origin HashOrNumber // Block from which to retrieve headers + Amount uint64 // Maximum number of headers to retrieve + Skip uint64 // Blocks to skip between consecutive headers + Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) +} + +// HashOrNumber is a combined field for specifying an origin block. +type HashOrNumber struct { + Hash common.Hash // Block hash from which to retrieve headers (excludes Number) + Number uint64 // Block hash from which to retrieve headers (excludes Hash) +} + +// EncodeRLP is a specialized encoder for HashOrNumber to encode only one of the +// two contained union fields. +func (hn *HashOrNumber) EncodeRLP(w io.Writer) error { + if hn.Hash == (common.Hash{}) { + return rlp.Encode(w, hn.Number) + } + if hn.Number != 0 { + return fmt.Errorf("both origin hash (%x) and number (%d) provided", hn.Hash, hn.Number) + } + return rlp.Encode(w, hn.Hash) +} + +// DecodeRLP is a specialized decoder for HashOrNumber to decode the contents +// into either a block hash or a block number. +func (hn *HashOrNumber) DecodeRLP(s *rlp.Stream) error { + _, size, _ := s.Kind() + origin, err := s.Raw() + if err == nil { + switch { + case size == 32: + err = rlp.DecodeBytes(origin, &hn.Hash) + case size <= 8: + err = rlp.DecodeBytes(origin, &hn.Number) + default: + err = fmt.Errorf("invalid input size %d for origin", size) + } + } + return err +} + +// BlockHeadersPacket represents a block header response. +type BlockHeadersPacket []*types.Header + +// NewBlockPacket is the network packet for the block propagation message. +type NewBlockPacket struct { + Block *types.Block + TD *big.Int +} + +// sanityCheck verifies that the values are reasonable, as a DoS protection +func (request *NewBlockPacket) sanityCheck() error { + if err := request.Block.SanityCheck(); err != nil { + return err + } + //TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times + // larger, it will still fit within 100 bits + if tdlen := request.TD.BitLen(); tdlen > 100 { + return fmt.Errorf("too large block TD: bitlen %d", tdlen) + } + return nil +} + +// GetBlockBodiesPacket represents a block body query. +type GetBlockBodiesPacket []common.Hash + +// BlockBodiesPacket is the network packet for block content distribution. +type BlockBodiesPacket []*BlockBody + +// BlockBody represents the data content of a single block. +type BlockBody struct { + Transactions []*types.Transaction // Transactions contained within a block + Uncles []*types.Header // Uncles contained within a block +} + +// Unpack retrieves the transactions and uncles from the range packet and returns +// them in a split flat format that's more consistent with the internal data structures. +func (p *BlockBodiesPacket) Unpack() ([][]*types.Transaction, [][]*types.Header) { + var ( + txset = make([][]*types.Transaction, len(*p)) + uncleset = make([][]*types.Header, len(*p)) + ) + for i, body := range *p { + txset[i], uncleset[i] = body.Transactions, body.Uncles + } + return txset, uncleset +} + +// GetNodeDataPacket represents a trie node data query. +type GetNodeDataPacket []common.Hash + +// NodeDataPacket is the network packet for trie node data distribution. +type NodeDataPacket [][]byte + +// GetReceiptsPacket represents a block receipts query. +type GetReceiptsPacket []common.Hash + +// ReceiptsPacket is the network packet for block receipts distribution. +type ReceiptsPacket [][]*types.Receipt + +// NewPooledTransactionHashesPacket represents a transaction announcement packet. +type NewPooledTransactionHashesPacket []common.Hash + +// GetPooledTransactionsPacket represents a transaction query. +type GetPooledTransactionsPacket []common.Hash + +// PooledTransactionsPacket is the network packet for transaction distribution. +type PooledTransactionsPacket []*types.Transaction + +func (*StatusPacket) Name() string { return "Status" } +func (*StatusPacket) Kind() byte { return StatusMsg } + +func (*NewBlockHashesPacket) Name() string { return "NewBlockHashes" } +func (*NewBlockHashesPacket) Kind() byte { return NewBlockHashesMsg } + +func (*TransactionsPacket) Name() string { return "Transactions" } +func (*TransactionsPacket) Kind() byte { return TransactionsMsg } + +func (*GetBlockHeadersPacket) Name() string { return "GetBlockHeaders" } +func (*GetBlockHeadersPacket) Kind() byte { return GetBlockHeadersMsg } + +func (*BlockHeadersPacket) Name() string { return "BlockHeaders" } +func (*BlockHeadersPacket) Kind() byte { return BlockHeadersMsg } + +func (*GetBlockBodiesPacket) Name() string { return "GetBlockBodies" } +func (*GetBlockBodiesPacket) Kind() byte { return GetBlockBodiesMsg } + +func (*BlockBodiesPacket) Name() string { return "BlockBodies" } +func (*BlockBodiesPacket) Kind() byte { return BlockBodiesMsg } + +func (*NewBlockPacket) Name() string { return "NewBlock" } +func (*NewBlockPacket) Kind() byte { return NewBlockMsg } + +func (*GetNodeDataPacket) Name() string { return "GetNodeData" } +func (*GetNodeDataPacket) Kind() byte { return GetNodeDataMsg } + +func (*NodeDataPacket) Name() string { return "NodeData" } +func (*NodeDataPacket) Kind() byte { return NodeDataMsg } + +func (*GetReceiptsPacket) Name() string { return "GetReceipts" } +func (*GetReceiptsPacket) Kind() byte { return GetReceiptsMsg } + +func (*ReceiptsPacket) Name() string { return "Receipts" } +func (*ReceiptsPacket) Kind() byte { return ReceiptsMsg } + +func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } +func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } + +func (*GetPooledTransactionsPacket) Name() string { return "GetPooledTransactions" } +func (*GetPooledTransactionsPacket) Kind() byte { return GetPooledTransactionsMsg } + +func (*PooledTransactionsPacket) Name() string { return "PooledTransactions" } +func (*PooledTransactionsPacket) Kind() byte { return PooledTransactionsMsg } diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go new file mode 100644 index 0000000000..056ea56480 --- /dev/null +++ b/eth/protocols/eth/protocol_test.go @@ -0,0 +1,68 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +// Tests that the custom union field encoder and decoder works correctly. +func TestGetBlockHeadersDataEncodeDecode(t *testing.T) { + // Create a "random" hash for testing + var hash common.Hash + for i := range hash { + hash[i] = byte(i) + } + // Assemble some table driven tests + tests := []struct { + packet *GetBlockHeadersPacket + fail bool + }{ + // Providing the origin as either a hash or a number should both work + {fail: false, packet: &GetBlockHeadersPacket{Origin: HashOrNumber{Number: 314}}}, + {fail: false, packet: &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: hash}}}, + + // Providing arbitrary query field should also work + {fail: false, packet: &GetBlockHeadersPacket{Origin: HashOrNumber{Number: 314}, Amount: 314, Skip: 1, Reverse: true}}, + {fail: false, packet: &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: hash}, Amount: 314, Skip: 1, Reverse: true}}, + + // Providing both the origin hash and origin number must fail + {fail: true, packet: &GetBlockHeadersPacket{Origin: HashOrNumber{Hash: hash, Number: 314}}}, + } + // Iterate over each of the tests and try to encode and then decode + for i, tt := range tests { + bytes, err := rlp.EncodeToBytes(tt.packet) + if err != nil && !tt.fail { + t.Fatalf("test %d: failed to encode packet: %v", i, err) + } else if err == nil && tt.fail { + t.Fatalf("test %d: encode should have failed", i) + } + if !tt.fail { + packet := new(GetBlockHeadersPacket) + if err := rlp.DecodeBytes(bytes, packet); err != nil { + t.Fatalf("test %d: failed to decode packet: %v", i, err) + } + if packet.Origin.Hash != tt.packet.Origin.Hash || packet.Origin.Number != tt.packet.Origin.Number || packet.Amount != tt.packet.Amount || + packet.Skip != tt.packet.Skip || packet.Reverse != tt.packet.Reverse { + t.Fatalf("test %d: encode decode mismatch: have %+v, want %+v", i, packet, tt.packet) + } + } + } +} diff --git a/eth/protocols/snap/discovery.go b/eth/protocols/snap/discovery.go new file mode 100644 index 0000000000..684ec7e632 --- /dev/null +++ b/eth/protocols/snap/discovery.go @@ -0,0 +1,32 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "github.com/ethereum/go-ethereum/rlp" +) + +// enrEntry is the ENR entry which advertises `snap` protocol on the discovery. +type enrEntry struct { + // Ignore additional fields (for forward compatibility). + Rest []rlp.RawValue `rlp:"tail"` +} + +// ENRKey implements enr.Entry. +func (e enrEntry) ENRKey() string { + return "snap" +} diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go new file mode 100644 index 0000000000..36322e648b --- /dev/null +++ b/eth/protocols/snap/handler.go @@ -0,0 +1,490 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +const ( + // softResponseLimit is the target maximum size of replies to data retrievals. + softResponseLimit = 2 * 1024 * 1024 + + // maxCodeLookups is the maximum number of bytecodes to serve. This number is + // there to limit the number of disk lookups. + maxCodeLookups = 1024 + + // stateLookupSlack defines the ratio by how much a state response can exceed + // the requested limit in order to try and avoid breaking up contracts into + // multiple packages and proving them. + stateLookupSlack = 0.1 + + // maxTrieNodeLookups is the maximum number of state trie nodes to serve. This + // number is there to limit the number of disk lookups. + maxTrieNodeLookups = 1024 +) + +// Handler is a callback to invoke from an outside runner after the boilerplate +// exchanges have passed. +type Handler func(peer *Peer) error + +// Backend defines the data retrieval methods to serve remote requests and the +// callback methods to invoke on remote deliveries. +type Backend interface { + // Chain retrieves the blockchain object to serve data. + Chain() *core.BlockChain + + // RunPeer is invoked when a peer joins on the `eth` protocol. The handler + // should do any peer maintenance work, handshakes and validations. If all + // is passed, control should be given back to the `handler` to process the + // inbound messages going forward. + RunPeer(peer *Peer, handler Handler) error + + // PeerInfo retrieves all known `snap` information about a peer. + PeerInfo(id enode.ID) interface{} + + // Handle is a callback to be invoked when a data packet is received from + // the remote peer. Only packets not consumed by the protocol handler will + // be forwarded to the backend. + Handle(peer *Peer, packet Packet) error +} + +// MakeProtocols constructs the P2P protocol definitions for `snap`. +func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { + protocols := make([]p2p.Protocol, len(protocolVersions)) + for i, version := range protocolVersions { + version := version // Closure + + protocols[i] = p2p.Protocol{ + Name: protocolName, + Version: version, + Length: protocolLengths[version], + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + return backend.RunPeer(newPeer(version, p, rw), func(peer *Peer) error { + return handle(backend, peer) + }) + }, + NodeInfo: func() interface{} { + return nodeInfo(backend.Chain()) + }, + PeerInfo: func(id enode.ID) interface{} { + return backend.PeerInfo(id) + }, + Attributes: []enr.Entry{&enrEntry{}}, + DialCandidates: dnsdisc, + } + } + return protocols +} + +// handle is the callback invoked to manage the life cycle of a `snap` peer. +// When this function terminates, the peer is disconnected. +func handle(backend Backend, peer *Peer) error { + for { + if err := handleMessage(backend, peer); err != nil { + peer.Log().Debug("Message handling failed in `snap`", "err", err) + return err + } + } +} + +// handleMessage is invoked whenever an inbound message is received from a +// remote peer on the `spap` protocol. The remote connection is torn down upon +// returning any error. +func handleMessage(backend Backend, peer *Peer) error { + // Read the next message from the remote peer, and ensure it's fully consumed + msg, err := peer.rw.ReadMsg() + if err != nil { + return err + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + defer msg.Discard() + + // Handle the message depending on its contents + switch { + case msg.Code == GetAccountRangeMsg: + // Decode the account retrieval request + var req GetAccountRangePacket + if err := msg.Decode(&req); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if req.Bytes > softResponseLimit { + req.Bytes = softResponseLimit + } + // Retrieve the requested state and bail out if non existent + tr, err := trie.New(req.Root, backend.Chain().StateCache().TrieDB()) + if err != nil { + return p2p.Send(peer.rw, AccountRangeMsg, &AccountRangePacket{ID: req.ID}) + } + it, err := backend.Chain().Snapshots().AccountIterator(req.Root, req.Origin) + if err != nil { + return p2p.Send(peer.rw, AccountRangeMsg, &AccountRangePacket{ID: req.ID}) + } + // Iterate over the requested range and pile accounts up + var ( + accounts []*AccountData + size uint64 + last common.Hash + ) + for it.Next() && size < req.Bytes { + hash, account := it.Hash(), common.CopyBytes(it.Account()) + + // Track the returned interval for the Merkle proofs + last = hash + + // Assemble the reply item + size += uint64(common.HashLength + len(account)) + accounts = append(accounts, &AccountData{ + Hash: hash, + Body: account, + }) + // If we've exceeded the request threshold, abort + if bytes.Compare(hash[:], req.Limit[:]) >= 0 { + break + } + } + it.Release() + + // Generate the Merkle proofs for the first and last account + proof := light.NewNodeSet() + if err := tr.Prove(req.Origin[:], 0, proof); err != nil { + log.Warn("Failed to prove account range", "origin", req.Origin, "err", err) + return p2p.Send(peer.rw, AccountRangeMsg, &AccountRangePacket{ID: req.ID}) + } + if last != (common.Hash{}) { + if err := tr.Prove(last[:], 0, proof); err != nil { + log.Warn("Failed to prove account range", "last", last, "err", err) + return p2p.Send(peer.rw, AccountRangeMsg, &AccountRangePacket{ID: req.ID}) + } + } + var proofs [][]byte + for _, blob := range proof.NodeList() { + proofs = append(proofs, blob) + } + // Send back anything accumulated + return p2p.Send(peer.rw, AccountRangeMsg, &AccountRangePacket{ + ID: req.ID, + Accounts: accounts, + Proof: proofs, + }) + + case msg.Code == AccountRangeMsg: + // A range of accounts arrived to one of our previous requests + res := new(AccountRangePacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Ensure the range is monotonically increasing + for i := 1; i < len(res.Accounts); i++ { + if bytes.Compare(res.Accounts[i-1].Hash[:], res.Accounts[i].Hash[:]) >= 0 { + return fmt.Errorf("accounts not monotonically increasing: #%d [%x] vs #%d [%x]", i-1, res.Accounts[i-1].Hash[:], i, res.Accounts[i].Hash[:]) + } + } + return backend.Handle(peer, res) + + case msg.Code == GetStorageRangesMsg: + // Decode the storage retrieval request + var req GetStorageRangesPacket + if err := msg.Decode(&req); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if req.Bytes > softResponseLimit { + req.Bytes = softResponseLimit + } + // TODO(karalabe): Do we want to enforce > 0 accounts and 1 account if origin is set? + // TODO(karalabe): - Logging locally is not ideal as remote faulst annoy the local user + // TODO(karalabe): - Dropping the remote peer is less flexible wrt client bugs (slow is better than non-functional) + + // Calculate the hard limit at which to abort, even if mid storage trie + hardLimit := uint64(float64(req.Bytes) * (1 + stateLookupSlack)) + + // Retrieve storage ranges until the packet limit is reached + var ( + slots [][]*StorageData + proofs [][]byte + size uint64 + ) + for _, account := range req.Accounts { + // If we've exceeded the requested data limit, abort without opening + // a new storage range (that we'd need to prove due to exceeded size) + if size >= req.Bytes { + break + } + // The first account might start from a different origin and end sooner + var origin common.Hash + if len(req.Origin) > 0 { + origin, req.Origin = common.BytesToHash(req.Origin), nil + } + var limit = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + if len(req.Limit) > 0 { + limit, req.Limit = common.BytesToHash(req.Limit), nil + } + // Retrieve the requested state and bail out if non existent + it, err := backend.Chain().Snapshots().StorageIterator(req.Root, account, origin) + if err != nil { + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) + } + // Iterate over the requested range and pile slots up + var ( + storage []*StorageData + last common.Hash + ) + for it.Next() && size < hardLimit { + hash, slot := it.Hash(), common.CopyBytes(it.Slot()) + + // Track the returned interval for the Merkle proofs + last = hash + + // Assemble the reply item + size += uint64(common.HashLength + len(slot)) + storage = append(storage, &StorageData{ + Hash: hash, + Body: slot, + }) + // If we've exceeded the request threshold, abort + if bytes.Compare(hash[:], limit[:]) >= 0 { + break + } + } + slots = append(slots, storage) + it.Release() + + // Generate the Merkle proofs for the first and last storage slot, but + // only if the response was capped. If the entire storage trie included + // in the response, no need for any proofs. + if origin != (common.Hash{}) || size >= hardLimit { + // Request started at a non-zero hash or was capped prematurely, add + // the endpoint Merkle proofs + accTrie, err := trie.New(req.Root, backend.Chain().StateCache().TrieDB()) + if err != nil { + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) + } + var acc state.Account + if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) + } + stTrie, err := trie.New(acc.Root, backend.Chain().StateCache().TrieDB()) + if err != nil { + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) + } + proof := light.NewNodeSet() + if err := stTrie.Prove(origin[:], 0, proof); err != nil { + log.Warn("Failed to prove storage range", "origin", req.Origin, "err", err) + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) + } + if last != (common.Hash{}) { + if err := stTrie.Prove(last[:], 0, proof); err != nil { + log.Warn("Failed to prove storage range", "last", last, "err", err) + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) + } + } + for _, blob := range proof.NodeList() { + proofs = append(proofs, blob) + } + // Proof terminates the reply as proofs are only added if a node + // refuses to serve more data (exception when a contract fetch is + // finishing, but that's that). + break + } + } + // Send back anything accumulated + return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ + ID: req.ID, + Slots: slots, + Proof: proofs, + }) + + case msg.Code == StorageRangesMsg: + // A range of storage slots arrived to one of our previous requests + res := new(StorageRangesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Ensure the ranges ae monotonically increasing + for i, slots := range res.Slots { + for j := 1; j < len(slots); j++ { + if bytes.Compare(slots[j-1].Hash[:], slots[j].Hash[:]) >= 0 { + return fmt.Errorf("storage slots not monotonically increasing for account #%d: #%d [%x] vs #%d [%x]", i, j-1, slots[j-1].Hash[:], j, slots[j].Hash[:]) + } + } + } + return backend.Handle(peer, res) + + case msg.Code == GetByteCodesMsg: + // Decode bytecode retrieval request + var req GetByteCodesPacket + if err := msg.Decode(&req); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if req.Bytes > softResponseLimit { + req.Bytes = softResponseLimit + } + if len(req.Hashes) > maxCodeLookups { + req.Hashes = req.Hashes[:maxCodeLookups] + } + // Retrieve bytecodes until the packet size limit is reached + var ( + codes [][]byte + bytes uint64 + ) + for _, hash := range req.Hashes { + if hash == emptyCode { + // Peers should not request the empty code, but if they do, at + // least sent them back a correct response without db lookups + codes = append(codes, []byte{}) + } else if blob, err := backend.Chain().ContractCode(hash); err == nil { + codes = append(codes, blob) + bytes += uint64(len(blob)) + } + if bytes > req.Bytes { + break + } + } + // Send back anything accumulated + return p2p.Send(peer.rw, ByteCodesMsg, &ByteCodesPacket{ + ID: req.ID, + Codes: codes, + }) + + case msg.Code == ByteCodesMsg: + // A batch of byte codes arrived to one of our previous requests + res := new(ByteCodesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + + case msg.Code == GetTrieNodesMsg: + // Decode trie node retrieval request + var req GetTrieNodesPacket + if err := msg.Decode(&req); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if req.Bytes > softResponseLimit { + req.Bytes = softResponseLimit + } + // Make sure we have the state associated with the request + triedb := backend.Chain().StateCache().TrieDB() + + accTrie, err := trie.NewSecure(req.Root, triedb) + if err != nil { + // We don't have the requested state available, bail out + return p2p.Send(peer.rw, TrieNodesMsg, &TrieNodesPacket{ID: req.ID}) + } + snap := backend.Chain().Snapshots().Snapshot(req.Root) + if snap == nil { + // We don't have the requested state snapshotted yet, bail out. + // In reality we could still serve using the account and storage + // tries only, but let's protect the node a bit while it's doing + // snapshot generation. + return p2p.Send(peer.rw, TrieNodesMsg, &TrieNodesPacket{ID: req.ID}) + } + // Retrieve trie nodes until the packet size limit is reached + var ( + nodes [][]byte + bytes uint64 + loads int // Trie hash expansions to cound database reads + ) + for _, pathset := range req.Paths { + switch len(pathset) { + case 0: + // Ensure we penalize invalid requests + return fmt.Errorf("%w: zero-item pathset requested", errBadRequest) + + case 1: + // If we're only retrieving an account trie node, fetch it directly + blob, resolved, err := accTrie.TryGetNode(pathset[0]) + loads += resolved // always account database reads, even for failures + if err != nil { + break + } + nodes = append(nodes, blob) + bytes += uint64(len(blob)) + + default: + // Storage slots requested, open the storage trie and retrieve from there + account, err := snap.Account(common.BytesToHash(pathset[0])) + loads++ // always account database reads, even for failures + if err != nil { + break + } + stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) + loads++ // always account database reads, even for failures + if err != nil { + break + } + for _, path := range pathset[1:] { + blob, resolved, err := stTrie.TryGetNode(path) + loads += resolved // always account database reads, even for failures + if err != nil { + break + } + nodes = append(nodes, blob) + bytes += uint64(len(blob)) + + // Sanity check limits to avoid DoS on the store trie loads + if bytes > req.Bytes || loads > maxTrieNodeLookups { + break + } + } + } + // Abort request processing if we've exceeded our limits + if bytes > req.Bytes || loads > maxTrieNodeLookups { + break + } + } + // Send back anything accumulated + return p2p.Send(peer.rw, TrieNodesMsg, &TrieNodesPacket{ + ID: req.ID, + Nodes: nodes, + }) + + case msg.Code == TrieNodesMsg: + // A batch of trie nodes arrived to one of our previous requests + res := new(TrieNodesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) + + default: + return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) + } +} + +// NodeInfo represents a short summary of the `snap` sub-protocol metadata +// known about the host peer. +type NodeInfo struct{} + +// nodeInfo retrieves some `snap` protocol metadata about the running host node. +func nodeInfo(chain *core.BlockChain) *NodeInfo { + return &NodeInfo{} +} diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go new file mode 100644 index 0000000000..73eaaadd09 --- /dev/null +++ b/eth/protocols/snap/peer.go @@ -0,0 +1,111 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" +) + +// Peer is a collection of relevant information we have about a `snap` peer. +type Peer struct { + id string // Unique ID for the peer, cached + + *p2p.Peer // The embedded P2P package peer + rw p2p.MsgReadWriter // Input/output streams for snap + version uint // Protocol version negotiated + + logger log.Logger // Contextual logger with the peer id injected +} + +// newPeer create a wrapper for a network connection and negotiated protocol +// version. +func newPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { + id := p.ID().String() + return &Peer{ + id: id, + Peer: p, + rw: rw, + version: version, + logger: log.New("peer", id[:8]), + } +} + +// ID retrieves the peer's unique identifier. +func (p *Peer) ID() string { + return p.id +} + +// Version retrieves the peer's negoatiated `snap` protocol version. +func (p *Peer) Version() uint { + return p.version +} + +// RequestAccountRange fetches a batch of accounts rooted in a specific account +// trie, starting with the origin. +func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes uint64) error { + p.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) + return p2p.Send(p.rw, GetAccountRangeMsg, &GetAccountRangePacket{ + ID: id, + Root: root, + Origin: origin, + Limit: limit, + Bytes: bytes, + }) +} + +// RequestStorageRange fetches a batch of storage slots belonging to one or more +// accounts. If slots from only one accout is requested, an origin marker may also +// be used to retrieve from there. +func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { + if len(accounts) == 1 && origin != nil { + p.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) + } else { + p.logger.Trace("Fetching ranges of small storage slots", "reqid", id, "root", root, "accounts", len(accounts), "first", accounts[0], "bytes", common.StorageSize(bytes)) + } + return p2p.Send(p.rw, GetStorageRangesMsg, &GetStorageRangesPacket{ + ID: id, + Root: root, + Accounts: accounts, + Origin: origin, + Limit: limit, + Bytes: bytes, + }) +} + +// RequestByteCodes fetches a batch of bytecodes by hash. +func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { + p.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) + return p2p.Send(p.rw, GetByteCodesMsg, &GetByteCodesPacket{ + ID: id, + Hashes: hashes, + Bytes: bytes, + }) +} + +// RequestTrieNodes fetches a batch of account or storage trie nodes rooted in +// a specificstate trie. +func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { + p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) + return p2p.Send(p.rw, GetTrieNodesMsg, &GetTrieNodesPacket{ + ID: id, + Root: root, + Paths: paths, + Bytes: bytes, + }) +} diff --git a/eth/protocols/snap/protocol.go b/eth/protocols/snap/protocol.go new file mode 100644 index 0000000000..a1e4349691 --- /dev/null +++ b/eth/protocols/snap/protocol.go @@ -0,0 +1,218 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/rlp" +) + +// Constants to match up protocol versions and messages +const ( + snap1 = 1 +) + +// protocolName is the official short name of the `snap` protocol used during +// devp2p capability negotiation. +const protocolName = "snap" + +// protocolVersions are the supported versions of the `snap` protocol (first +// is primary). +var protocolVersions = []uint{snap1} + +// protocolLengths are the number of implemented message corresponding to +// different protocol versions. +var protocolLengths = map[uint]uint64{snap1: 8} + +// maxMessageSize is the maximum cap on the size of a protocol message. +const maxMessageSize = 10 * 1024 * 1024 + +const ( + GetAccountRangeMsg = 0x00 + AccountRangeMsg = 0x01 + GetStorageRangesMsg = 0x02 + StorageRangesMsg = 0x03 + GetByteCodesMsg = 0x04 + ByteCodesMsg = 0x05 + GetTrieNodesMsg = 0x06 + TrieNodesMsg = 0x07 +) + +var ( + errMsgTooLarge = errors.New("message too long") + errDecode = errors.New("invalid message") + errInvalidMsgCode = errors.New("invalid message code") + errBadRequest = errors.New("bad request") +) + +// Packet represents a p2p message in the `snap` protocol. +type Packet interface { + Name() string // Name returns a string corresponding to the message type. + Kind() byte // Kind returns the message type. +} + +// GetAccountRangePacket represents an account query. +type GetAccountRangePacket struct { + ID uint64 // Request ID to match up responses with + Root common.Hash // Root hash of the account trie to serve + Origin common.Hash // Hash of the first account to retrieve + Limit common.Hash // Hash of the last account to retrieve + Bytes uint64 // Soft limit at which to stop returning data +} + +// AccountRangePacket represents an account query response. +type AccountRangePacket struct { + ID uint64 // ID of the request this is a response for + Accounts []*AccountData // List of consecutive accounts from the trie + Proof [][]byte // List of trie nodes proving the account range +} + +// AccountData represents a single account in a query response. +type AccountData struct { + Hash common.Hash // Hash of the account + Body rlp.RawValue // Account body in slim format +} + +// Unpack retrieves the accounts from the range packet and converts from slim +// wire representation to consensus format. The returned data is RLP encoded +// since it's expected to be serialized to disk without further interpretation. +// +// Note, this method does a round of RLP decoding and reencoding, so only use it +// once and cache the results if need be. Ideally discard the packet afterwards +// to not double the memory use. +func (p *AccountRangePacket) Unpack() ([]common.Hash, [][]byte, error) { + var ( + hashes = make([]common.Hash, len(p.Accounts)) + accounts = make([][]byte, len(p.Accounts)) + ) + for i, acc := range p.Accounts { + val, err := snapshot.FullAccountRLP(acc.Body) + if err != nil { + return nil, nil, fmt.Errorf("invalid account %x: %v", acc.Body, err) + } + hashes[i], accounts[i] = acc.Hash, val + } + return hashes, accounts, nil +} + +// GetStorageRangesPacket represents an storage slot query. +type GetStorageRangesPacket struct { + ID uint64 // Request ID to match up responses with + Root common.Hash // Root hash of the account trie to serve + Accounts []common.Hash // Account hashes of the storage tries to serve + Origin []byte // Hash of the first storage slot to retrieve (large contract mode) + Limit []byte // Hash of the last storage slot to retrieve (large contract mode) + Bytes uint64 // Soft limit at which to stop returning data +} + +// StorageRangesPacket represents a storage slot query response. +type StorageRangesPacket struct { + ID uint64 // ID of the request this is a response for + Slots [][]*StorageData // Lists of consecutive storage slots for the requested accounts + Proof [][]byte // Merkle proofs for the *last* slot range, if it's incomplete +} + +// StorageData represents a single storage slot in a query response. +type StorageData struct { + Hash common.Hash // Hash of the storage slot + Body []byte // Data content of the slot +} + +// Unpack retrieves the storage slots from the range packet and returns them in +// a split flat format that's more consistent with the internal data structures. +func (p *StorageRangesPacket) Unpack() ([][]common.Hash, [][][]byte) { + var ( + hashset = make([][]common.Hash, len(p.Slots)) + slotset = make([][][]byte, len(p.Slots)) + ) + for i, slots := range p.Slots { + hashset[i] = make([]common.Hash, len(slots)) + slotset[i] = make([][]byte, len(slots)) + for j, slot := range slots { + hashset[i][j] = slot.Hash + slotset[i][j] = slot.Body + } + } + return hashset, slotset +} + +// GetByteCodesPacket represents a contract bytecode query. +type GetByteCodesPacket struct { + ID uint64 // Request ID to match up responses with + Hashes []common.Hash // Code hashes to retrieve the code for + Bytes uint64 // Soft limit at which to stop returning data +} + +// ByteCodesPacket represents a contract bytecode query response. +type ByteCodesPacket struct { + ID uint64 // ID of the request this is a response for + Codes [][]byte // Requested contract bytecodes +} + +// GetTrieNodesPacket represents a state trie node query. +type GetTrieNodesPacket struct { + ID uint64 // Request ID to match up responses with + Root common.Hash // Root hash of the account trie to serve + Paths []TrieNodePathSet // Trie node hashes to retrieve the nodes for + Bytes uint64 // Soft limit at which to stop returning data +} + +// TrieNodePathSet is a list of trie node paths to retrieve. A naive way to +// represent trie nodes would be a simple list of `account || storage` path +// segments concatenated, but that would be very wasteful on the network. +// +// Instead, this array special cases the first element as the path in the +// account trie and the remaining elements as paths in the storage trie. To +// address an account node, the slice should have a length of 1 consisting +// of only the account path. There's no need to be able to address both an +// account node and a storage node in the same request as it cannot happen +// that a slot is accessed before the account path is fully expanded. +type TrieNodePathSet [][]byte + +// TrieNodesPacket represents a state trie node query response. +type TrieNodesPacket struct { + ID uint64 // ID of the request this is a response for + Nodes [][]byte // Requested state trie nodes +} + +func (*GetAccountRangePacket) Name() string { return "GetAccountRange" } +func (*GetAccountRangePacket) Kind() byte { return GetAccountRangeMsg } + +func (*AccountRangePacket) Name() string { return "AccountRange" } +func (*AccountRangePacket) Kind() byte { return AccountRangeMsg } + +func (*GetStorageRangesPacket) Name() string { return "GetStorageRanges" } +func (*GetStorageRangesPacket) Kind() byte { return GetStorageRangesMsg } + +func (*StorageRangesPacket) Name() string { return "StorageRanges" } +func (*StorageRangesPacket) Kind() byte { return StorageRangesMsg } + +func (*GetByteCodesPacket) Name() string { return "GetByteCodes" } +func (*GetByteCodesPacket) Kind() byte { return GetByteCodesMsg } + +func (*ByteCodesPacket) Name() string { return "ByteCodes" } +func (*ByteCodesPacket) Kind() byte { return ByteCodesMsg } + +func (*GetTrieNodesPacket) Name() string { return "GetTrieNodes" } +func (*GetTrieNodesPacket) Kind() byte { return GetTrieNodesMsg } + +func (*TrieNodesPacket) Name() string { return "TrieNodes" } +func (*TrieNodesPacket) Kind() byte { return TrieNodesMsg } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go new file mode 100644 index 0000000000..679b328283 --- /dev/null +++ b/eth/protocols/snap/sync.go @@ -0,0 +1,2481 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "math/big" + "math/rand" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "golang.org/x/crypto/sha3" +) + +var ( + // emptyRoot is the known root hash of an empty trie. + emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + + // emptyCode is the known hash of the empty EVM bytecode. + emptyCode = crypto.Keccak256Hash(nil) +) + +const ( + // maxRequestSize is the maximum number of bytes to request from a remote peer. + maxRequestSize = 512 * 1024 + + // maxStorageSetRequestCountis th maximum number of contracts to request the + // storage of in a single query. If this number is too low, we're not filling + // responses fully and waste round trip times. If it's too high, we're capping + // responses and waste bandwidth. + maxStorageSetRequestCount = maxRequestSize / 1024 + + // maxCodeRequestCount is the maximum number of bytecode blobs to request in a + // single query. If this number is too low, we're not filling responses fully + // and waste round trip times. If it's too high, we're capping responses and + // waste bandwidth. + // + // Depoyed bytecodes are currently capped at 24KB, so the minimum request + // size should be maxRequestSize / 24K. Assuming that most contracts do not + // come close to that, requesting 4x should be a good approximation. + maxCodeRequestCount = maxRequestSize / (24 * 1024) * 4 + + // maxTrieRequestCount is the maximum number of trie node blobs to request in + // a single query. If this number is too low, we're not filling responses fully + // and waste round trip times. If it's too high, we're capping responses and + // waste bandwidth. + maxTrieRequestCount = 512 + + // requestTimeout is the maximum time a peer is allowed to spend on serving + // a single network request. + requestTimeout = 10 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? + + // accountConcurrency is the number of chunks to split the account trie into + // to allow concurrent retrievals. + accountConcurrency = 16 + + // storageConcurrency is the number of chunks to split the a large contract + // storage trie into to allow concurrent retrievals. + storageConcurrency = 16 +) + +// accountRequest tracks a pending account range request to ensure responses are +// to actual requests and to validate any security constraints. +// +// Concurrency note: account requests and responses are handled concurrently from +// the main runloop to allow Merkle proof verifications on the peer's thread and +// to drop on invalid response. The request struct must contain all the data to +// construct the response without accessing runloop internals (i.e. task). That +// is only included to allow the runloop to match a response to the task being +// synced without having yet another set of maps. +type accountRequest struct { + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped + + origin common.Hash // First account requested to allow continuation checks + limit common.Hash // Last account requested to allow non-overlapping chunking + + task *accountTask // Task which this request is filling (only access fields through the runloop!!) +} + +// accountResponse is an already Merkle-verified remote response to an account +// range request. It contains the subtrie for the requested account range and +// the database that's going to be filled with the internal nodes on commit. +type accountResponse struct { + task *accountTask // Task which this request is filling + + hashes []common.Hash // Account hashes in the returned range + accounts []*state.Account // Expanded accounts in the returned range + + nodes ethdb.KeyValueStore // Database containing the reconstructed trie nodes + trie *trie.Trie // Reconstructed trie to reject incomplete account paths + + bounds map[common.Hash]struct{} // Boundary nodes to avoid persisting incomplete accounts + overflow *light.NodeSet // Overflow nodes to avoid persisting across chunk boundaries + + cont bool // Whether the account range has a continuation +} + +// bytecodeRequest tracks a pending bytecode request to ensure responses are to +// actual requests and to validate any security constraints. +// +// Concurrency note: bytecode requests and responses are handled concurrently from +// the main runloop to allow Keccak256 hash verifications on the peer's thread and +// to drop on invalid response. The request struct must contain all the data to +// construct the response without accessing runloop internals (i.e. task). That +// is only included to allow the runloop to match a response to the task being +// synced without having yet another set of maps. +type bytecodeRequest struct { + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped + + hashes []common.Hash // Bytecode hashes to validate responses + task *accountTask // Task which this request is filling (only access fields through the runloop!!) +} + +// bytecodeResponse is an already verified remote response to a bytecode request. +type bytecodeResponse struct { + task *accountTask // Task which this request is filling + + hashes []common.Hash // Hashes of the bytecode to avoid double hashing + codes [][]byte // Actual bytecodes to store into the database (nil = missing) +} + +// storageRequest tracks a pending storage ranges request to ensure responses are +// to actual requests and to validate any security constraints. +// +// Concurrency note: storage requests and responses are handled concurrently from +// the main runloop to allow Merkel proof verifications on the peer's thread and +// to drop on invalid response. The request struct must contain all the data to +// construct the response without accessing runloop internals (i.e. tasks). That +// is only included to allow the runloop to match a response to the task being +// synced without having yet another set of maps. +type storageRequest struct { + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped + + accounts []common.Hash // Account hashes to validate responses + roots []common.Hash // Storage roots to validate responses + + origin common.Hash // First storage slot requested to allow continuation checks + limit common.Hash // Last storage slot requested to allow non-overlapping chunking + + mainTask *accountTask // Task which this response belongs to (only access fields through the runloop!!) + subTask *storageTask // Task which this response is filling (only access fields through the runloop!!) +} + +// storageResponse is an already Merkle-verified remote response to a storage +// range request. It contains the subtries for the requested storage ranges and +// the databases that's going to be filled with the internal nodes on commit. +type storageResponse struct { + mainTask *accountTask // Task which this response belongs to + subTask *storageTask // Task which this response is filling + + accounts []common.Hash // Account hashes requested, may be only partially filled + roots []common.Hash // Storage roots requested, may be only partially filled + + hashes [][]common.Hash // Storage slot hashes in the returned range + slots [][][]byte // Storage slot values in the returned range + nodes []ethdb.KeyValueStore // Database containing the reconstructed trie nodes + tries []*trie.Trie // Reconstructed tries to reject overflown slots + + // Fields relevant for the last account only + bounds map[common.Hash]struct{} // Boundary nodes to avoid persisting (incomplete) + overflow *light.NodeSet // Overflow nodes to avoid persisting across chunk boundaries + cont bool // Whether the last storage range has a continuation +} + +// trienodeHealRequest tracks a pending state trie request to ensure responses +// are to actual requests and to validate any security constraints. +// +// Concurrency note: trie node requests and responses are handled concurrently from +// the main runloop to allow Keccak256 hash verifications on the peer's thread and +// to drop on invalid response. The request struct must contain all the data to +// construct the response without accessing runloop internals (i.e. task). That +// is only included to allow the runloop to match a response to the task being +// synced without having yet another set of maps. +type trienodeHealRequest struct { + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped + + hashes []common.Hash // Trie node hashes to validate responses + paths []trie.SyncPath // Trie node paths requested for rescheduling + + task *healTask // Task which this request is filling (only access fields through the runloop!!) +} + +// trienodeHealResponse is an already verified remote response to a trie node request. +type trienodeHealResponse struct { + task *healTask // Task which this request is filling + + hashes []common.Hash // Hashes of the trie nodes to avoid double hashing + paths []trie.SyncPath // Trie node paths requested for rescheduling missing ones + nodes [][]byte // Actual trie nodes to store into the database (nil = missing) +} + +// bytecodeHealRequest tracks a pending bytecode request to ensure responses are to +// actual requests and to validate any security constraints. +// +// Concurrency note: bytecode requests and responses are handled concurrently from +// the main runloop to allow Keccak256 hash verifications on the peer's thread and +// to drop on invalid response. The request struct must contain all the data to +// construct the response without accessing runloop internals (i.e. task). That +// is only included to allow the runloop to match a response to the task being +// synced without having yet another set of maps. +type bytecodeHealRequest struct { + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped + + hashes []common.Hash // Bytecode hashes to validate responses + task *healTask // Task which this request is filling (only access fields through the runloop!!) +} + +// bytecodeHealResponse is an already verified remote response to a bytecode request. +type bytecodeHealResponse struct { + task *healTask // Task which this request is filling + + hashes []common.Hash // Hashes of the bytecode to avoid double hashing + codes [][]byte // Actual bytecodes to store into the database (nil = missing) +} + +// accountTask represents the sync task for a chunk of the account snapshot. +type accountTask struct { + // These fields get serialized to leveldb on shutdown + Next common.Hash // Next account to sync in this interval + Last common.Hash // Last account to sync in this interval + SubTasks map[common.Hash][]*storageTask // Storage intervals needing fetching for large contracts + + // These fields are internals used during runtime + req *accountRequest // Pending request to fill this task + res *accountResponse // Validate response filling this task + pend int // Number of pending subtasks for this round + + needCode []bool // Flags whether the filling accounts need code retrieval + needState []bool // Flags whether the filling accounts need storage retrieval + needHeal []bool // Flags whether the filling accounts's state was chunked and need healing + + codeTasks map[common.Hash]struct{} // Code hashes that need retrieval + stateTasks map[common.Hash]common.Hash // Account hashes->roots that need full state retrieval + + done bool // Flag whether the task can be removed +} + +// storageTask represents the sync task for a chunk of the storage snapshot. +type storageTask struct { + Next common.Hash // Next account to sync in this interval + Last common.Hash // Last account to sync in this interval + + // These fields are internals used during runtime + root common.Hash // Storage root hash for this instance + req *storageRequest // Pending request to fill this task + done bool // Flag whether the task can be removed +} + +// healTask represents the sync task for healing the snap-synced chunk boundaries. +type healTask struct { + scheduler *trie.Sync // State trie sync scheduler defining the tasks + + trieTasks map[common.Hash]trie.SyncPath // Set of trie node tasks currently queued for retrieval + codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval +} + +// syncProgress is a database entry to allow suspending and resuming a snapshot state +// sync. Opposed to full and fast sync, there is no way to restart a suspended +// snap sync without prior knowledge of the suspension point. +type syncProgress struct { + Tasks []*accountTask // The suspended account tasks (contract tasks within) + + // Status report during syncing phase + AccountSynced uint64 // Number of accounts downloaded + AccountBytes common.StorageSize // Number of account trie bytes persisted to disk + BytecodeSynced uint64 // Number of bytecodes downloaded + BytecodeBytes common.StorageSize // Number of bytecode bytes downloaded + StorageSynced uint64 // Number of storage slots downloaded + StorageBytes common.StorageSize // Number of storage trie bytes persisted to disk + + // Status report during healing phase + TrienodeHealSynced uint64 // Number of state trie nodes downloaded + TrienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk + TrienodeHealDups uint64 // Number of state trie nodes already processed + TrienodeHealNops uint64 // Number of state trie nodes not requested + BytecodeHealSynced uint64 // Number of bytecodes downloaded + BytecodeHealBytes common.StorageSize // Number of bytecodes persisted to disk + BytecodeHealDups uint64 // Number of bytecodes already processed + BytecodeHealNops uint64 // Number of bytecodes not requested +} + +// Syncer is an Ethereum account and storage trie syncer based on snapshots and +// the snap protocol. It's purpose is to download all the accounts and storage +// slots from remote peers and reassemble chunks of the state trie, on top of +// which a state sync can be run to fix any gaps / overlaps. +// +// Every network request has a variety of failure events: +// - The peer disconnects after task assignment, failing to send the request +// - The peer disconnects after sending the request, before delivering on it +// - The peer remains connected, but does not deliver a response in time +// - The peer delivers a stale response after a previous timeout +// - The peer delivers a refusal to serve the requested state +type Syncer struct { + db ethdb.KeyValueStore // Database to store the trie nodes into (and dedup) + bloom *trie.SyncBloom // Bloom filter to deduplicate nodes for state fixup + + root common.Hash // Current state trie root being synced + tasks []*accountTask // Current account task set being synced + healer *healTask // Current state healing task being executed + update chan struct{} // Notification channel for possible sync progression + + peers map[string]*Peer // Currently active peers to download from + peerJoin *event.Feed // Event feed to react to peers joining + peerDrop *event.Feed // Event feed to react to peers dropping + + // Request tracking during syncing phase + statelessPeers map[string]struct{} // Peers that failed to deliver state data + accountIdlers map[string]struct{} // Peers that aren't serving account requests + bytecodeIdlers map[string]struct{} // Peers that aren't serving bytecode requests + storageIdlers map[string]struct{} // Peers that aren't serving storage requests + + accountReqs map[uint64]*accountRequest // Account requests currently running + bytecodeReqs map[uint64]*bytecodeRequest // Bytecode requests currently running + storageReqs map[uint64]*storageRequest // Storage requests currently running + + accountReqFails chan *accountRequest // Failed account range requests to revert + bytecodeReqFails chan *bytecodeRequest // Failed bytecode requests to revert + storageReqFails chan *storageRequest // Failed storage requests to revert + + accountResps chan *accountResponse // Account sub-tries to integrate into the database + bytecodeResps chan *bytecodeResponse // Bytecodes to integrate into the database + storageResps chan *storageResponse // Storage sub-tries to integrate into the database + + accountSynced uint64 // Number of accounts downloaded + accountBytes common.StorageSize // Number of account trie bytes persisted to disk + bytecodeSynced uint64 // Number of bytecodes downloaded + bytecodeBytes common.StorageSize // Number of bytecode bytes downloaded + storageSynced uint64 // Number of storage slots downloaded + storageBytes common.StorageSize // Number of storage trie bytes persisted to disk + + // Request tracking during healing phase + trienodeHealIdlers map[string]struct{} // Peers that aren't serving trie node requests + bytecodeHealIdlers map[string]struct{} // Peers that aren't serving bytecode requests + + trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running + bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running + + trienodeHealReqFails chan *trienodeHealRequest // Failed trienode requests to revert + bytecodeHealReqFails chan *bytecodeHealRequest // Failed bytecode requests to revert + + trienodeHealResps chan *trienodeHealResponse // Trie nodes to integrate into the database + bytecodeHealResps chan *bytecodeHealResponse // Bytecodes to integrate into the database + + trienodeHealSynced uint64 // Number of state trie nodes downloaded + trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk + trienodeHealDups uint64 // Number of state trie nodes already processed + trienodeHealNops uint64 // Number of state trie nodes not requested + bytecodeHealSynced uint64 // Number of bytecodes downloaded + bytecodeHealBytes common.StorageSize // Number of bytecodes persisted to disk + bytecodeHealDups uint64 // Number of bytecodes already processed + bytecodeHealNops uint64 // Number of bytecodes not requested + + startTime time.Time // Time instance when snapshot sync started + startAcc common.Hash // Account hash where sync started from + logTime time.Time // Time instance when status was last reported + + pend sync.WaitGroup // Tracks network request goroutines for graceful shutdown + lock sync.RWMutex // Protects fields that can change outside of sync (peers, reqs, root) +} + +func NewSyncer(db ethdb.KeyValueStore, bloom *trie.SyncBloom) *Syncer { + return &Syncer{ + db: db, + bloom: bloom, + + peers: make(map[string]*Peer), + peerJoin: new(event.Feed), + peerDrop: new(event.Feed), + update: make(chan struct{}, 1), + + accountIdlers: make(map[string]struct{}), + storageIdlers: make(map[string]struct{}), + bytecodeIdlers: make(map[string]struct{}), + + accountReqs: make(map[uint64]*accountRequest), + storageReqs: make(map[uint64]*storageRequest), + bytecodeReqs: make(map[uint64]*bytecodeRequest), + accountReqFails: make(chan *accountRequest), + storageReqFails: make(chan *storageRequest), + bytecodeReqFails: make(chan *bytecodeRequest), + accountResps: make(chan *accountResponse), + storageResps: make(chan *storageResponse), + bytecodeResps: make(chan *bytecodeResponse), + + trienodeHealIdlers: make(map[string]struct{}), + bytecodeHealIdlers: make(map[string]struct{}), + + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + trienodeHealReqFails: make(chan *trienodeHealRequest), + bytecodeHealReqFails: make(chan *bytecodeHealRequest), + trienodeHealResps: make(chan *trienodeHealResponse), + bytecodeHealResps: make(chan *bytecodeHealResponse), + } +} + +// Register injects a new data source into the syncer's peerset. +func (s *Syncer) Register(peer *Peer) error { + // Make sure the peer is not registered yet + s.lock.Lock() + if _, ok := s.peers[peer.id]; ok { + log.Error("Snap peer already registered", "id", peer.id) + + s.lock.Unlock() + return errors.New("already registered") + } + s.peers[peer.id] = peer + + // Mark the peer as idle, even if no sync is running + s.accountIdlers[peer.id] = struct{}{} + s.storageIdlers[peer.id] = struct{}{} + s.bytecodeIdlers[peer.id] = struct{}{} + s.trienodeHealIdlers[peer.id] = struct{}{} + s.bytecodeHealIdlers[peer.id] = struct{}{} + s.lock.Unlock() + + // Notify any active syncs that a new peer can be assigned data + s.peerJoin.Send(peer.id) + return nil +} + +// Unregister injects a new data source into the syncer's peerset. +func (s *Syncer) Unregister(id string) error { + // Remove all traces of the peer from the registry + s.lock.Lock() + if _, ok := s.peers[id]; !ok { + log.Error("Snap peer not registered", "id", id) + + s.lock.Unlock() + return errors.New("not registered") + } + delete(s.peers, id) + + // Remove status markers, even if no sync is running + delete(s.statelessPeers, id) + + delete(s.accountIdlers, id) + delete(s.storageIdlers, id) + delete(s.bytecodeIdlers, id) + delete(s.trienodeHealIdlers, id) + delete(s.bytecodeHealIdlers, id) + s.lock.Unlock() + + // Notify any active syncs that pending requests need to be reverted + s.peerDrop.Send(id) + return nil +} + +// Sync starts (or resumes a previous) sync cycle to iterate over an state trie +// with the given root and reconstruct the nodes based on the snapshot leaves. +// Previously downloaded segments will not be redownloaded of fixed, rather any +// errors will be healed after the leaves are fully accumulated. +func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { + // Move the trie root from any previous value, revert stateless markers for + // any peers and initialize the syncer if it was not yet run + s.lock.Lock() + s.root = root + s.healer = &healTask{ + scheduler: state.NewStateSync(root, s.db, s.bloom), + trieTasks: make(map[common.Hash]trie.SyncPath), + codeTasks: make(map[common.Hash]struct{}), + } + s.statelessPeers = make(map[string]struct{}) + s.lock.Unlock() + + if s.startTime == (time.Time{}) { + s.startTime = time.Now() + } + // Retrieve the previous sync status from LevelDB and abort if already synced + s.loadSyncStatus() + if len(s.tasks) == 0 && s.healer.scheduler.Pending() == 0 { + log.Debug("Snapshot sync already completed") + return nil + } + defer func() { // Persist any progress, independent of failure + for _, task := range s.tasks { + s.forwardAccountTask(task) + } + s.cleanAccountTasks() + s.saveSyncStatus() + }() + + log.Debug("Starting snapshot sync cycle", "root", root) + defer s.report(true) + + // Whether sync completed or not, disregard any future packets + defer func() { + log.Debug("Terminating snapshot sync cycle", "root", root) + s.lock.Lock() + s.accountReqs = make(map[uint64]*accountRequest) + s.storageReqs = make(map[uint64]*storageRequest) + s.bytecodeReqs = make(map[uint64]*bytecodeRequest) + s.trienodeHealReqs = make(map[uint64]*trienodeHealRequest) + s.bytecodeHealReqs = make(map[uint64]*bytecodeHealRequest) + s.lock.Unlock() + }() + // Keep scheduling sync tasks + peerJoin := make(chan string, 16) + peerJoinSub := s.peerJoin.Subscribe(peerJoin) + defer peerJoinSub.Unsubscribe() + + peerDrop := make(chan string, 16) + peerDropSub := s.peerDrop.Subscribe(peerDrop) + defer peerDropSub.Unsubscribe() + + for { + // Remove all completed tasks and terminate sync if everything's done + s.cleanStorageTasks() + s.cleanAccountTasks() + if len(s.tasks) == 0 && s.healer.scheduler.Pending() == 0 { + return nil + } + // Assign all the data retrieval tasks to any free peers + s.assignAccountTasks(cancel) + s.assignBytecodeTasks(cancel) + s.assignStorageTasks(cancel) + if len(s.tasks) == 0 { + // Sync phase done, run heal phase + s.assignTrienodeHealTasks(cancel) + s.assignBytecodeHealTasks(cancel) + } + // Wait for something to happen + select { + case <-s.update: + // Something happened (new peer, delivery, timeout), recheck tasks + case <-peerJoin: + // A new peer joined, try to schedule it new tasks + case id := <-peerDrop: + s.revertRequests(id) + case <-cancel: + return nil + + case req := <-s.accountReqFails: + s.revertAccountRequest(req) + case req := <-s.bytecodeReqFails: + s.revertBytecodeRequest(req) + case req := <-s.storageReqFails: + s.revertStorageRequest(req) + case req := <-s.trienodeHealReqFails: + s.revertTrienodeHealRequest(req) + case req := <-s.bytecodeHealReqFails: + s.revertBytecodeHealRequest(req) + + case res := <-s.accountResps: + s.processAccountResponse(res) + case res := <-s.bytecodeResps: + s.processBytecodeResponse(res) + case res := <-s.storageResps: + s.processStorageResponse(res) + case res := <-s.trienodeHealResps: + s.processTrienodeHealResponse(res) + case res := <-s.bytecodeHealResps: + s.processBytecodeHealResponse(res) + } + // Report stats if something meaningful happened + s.report(false) + } +} + +// loadSyncStatus retrieves a previously aborted sync status from the database, +// or generates a fresh one if none is available. +func (s *Syncer) loadSyncStatus() { + var progress syncProgress + + if status := rawdb.ReadSanpshotSyncStatus(s.db); status != nil { + if err := json.Unmarshal(status, &progress); err != nil { + log.Error("Failed to decode snap sync status", "err", err) + } else { + for _, task := range progress.Tasks { + log.Debug("Scheduled account sync task", "from", task.Next, "last", task.Last) + } + s.tasks = progress.Tasks + + s.accountSynced = progress.AccountSynced + s.accountBytes = progress.AccountBytes + s.bytecodeSynced = progress.BytecodeSynced + s.bytecodeBytes = progress.BytecodeBytes + s.storageSynced = progress.StorageSynced + s.storageBytes = progress.StorageBytes + + s.trienodeHealSynced = progress.TrienodeHealSynced + s.trienodeHealBytes = progress.TrienodeHealBytes + s.bytecodeHealSynced = progress.BytecodeHealSynced + s.bytecodeHealBytes = progress.BytecodeHealBytes + return + } + } + // Either we've failed to decode the previus state, or there was none. + // Start a fresh sync by chunking up the account range and scheduling + // them for retrieval. + s.tasks = nil + s.accountSynced, s.accountBytes = 0, 0 + s.bytecodeSynced, s.bytecodeBytes = 0, 0 + s.storageSynced, s.storageBytes = 0, 0 + s.trienodeHealSynced, s.trienodeHealBytes = 0, 0 + s.bytecodeHealSynced, s.bytecodeHealBytes = 0, 0 + + var next common.Hash + step := new(big.Int).Sub( + new(big.Int).Div( + new(big.Int).Exp(common.Big2, common.Big256, nil), + big.NewInt(accountConcurrency), + ), common.Big1, + ) + for i := 0; i < accountConcurrency; i++ { + last := common.BigToHash(new(big.Int).Add(next.Big(), step)) + if i == accountConcurrency-1 { + // Make sure we don't overflow if the step is not a proper divisor + last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + } + s.tasks = append(s.tasks, &accountTask{ + Next: next, + Last: last, + SubTasks: make(map[common.Hash][]*storageTask), + }) + log.Debug("Created account sync task", "from", next, "last", last) + next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) + } +} + +// saveSyncStatus marshals the remaining sync tasks into leveldb. +func (s *Syncer) saveSyncStatus() { + progress := &syncProgress{ + Tasks: s.tasks, + AccountSynced: s.accountSynced, + AccountBytes: s.accountBytes, + BytecodeSynced: s.bytecodeSynced, + BytecodeBytes: s.bytecodeBytes, + StorageSynced: s.storageSynced, + StorageBytes: s.storageBytes, + TrienodeHealSynced: s.trienodeHealSynced, + TrienodeHealBytes: s.trienodeHealBytes, + BytecodeHealSynced: s.bytecodeHealSynced, + BytecodeHealBytes: s.bytecodeHealBytes, + } + status, err := json.Marshal(progress) + if err != nil { + panic(err) // This can only fail during implementation + } + rawdb.WriteSnapshotSyncStatus(s.db, status) +} + +// cleanAccountTasks removes account range retrieval tasks that have already been +// completed. +func (s *Syncer) cleanAccountTasks() { + for i := 0; i < len(s.tasks); i++ { + if s.tasks[i].done { + s.tasks = append(s.tasks[:i], s.tasks[i+1:]...) + i-- + } + } +} + +// cleanStorageTasks iterates over all the account tasks and storage sub-tasks +// within, cleaning any that have been completed. +func (s *Syncer) cleanStorageTasks() { + for _, task := range s.tasks { + for account, subtasks := range task.SubTasks { + // Remove storage range retrieval tasks that completed + for j := 0; j < len(subtasks); j++ { + if subtasks[j].done { + subtasks = append(subtasks[:j], subtasks[j+1:]...) + j-- + } + } + if len(subtasks) > 0 { + task.SubTasks[account] = subtasks + continue + } + // If all storage chunks are done, mark the account as done too + for j, hash := range task.res.hashes { + if hash == account { + task.needState[j] = false + } + } + delete(task.SubTasks, account) + task.pend-- + + // If this was the last pending task, forward the account task + if task.pend == 0 { + s.forwardAccountTask(task) + } + } + } +} + +// assignAccountTasks attempts to match idle peers to pending account range +// retrievals. +func (s *Syncer) assignAccountTasks(cancel chan struct{}) { + s.lock.Lock() + defer s.lock.Unlock() + + // If there are no idle peers, short circuit assignment + if len(s.accountIdlers) == 0 { + return + } + // Iterate over all the tasks and try to find a pending one + for _, task := range s.tasks { + // Skip any tasks already filling + if task.req != nil || task.res != nil { + continue + } + // Task pending retrieval, try to find an idle peer. If no such peer + // exists, we probably assigned tasks for all (or they are stateless). + // Abort the entire assignment mechanism. + var idle string + for id := range s.accountIdlers { + // If the peer rejected a query in this sync cycle, don't bother asking + // again for anything, it's either out of sync or already pruned + if _, ok := s.statelessPeers[id]; ok { + continue + } + idle = id + break + } + if idle == "" { + return + } + // Matched a pending task to an idle peer, allocate a unique request id + var reqid uint64 + for { + reqid = uint64(rand.Int63()) + if reqid == 0 { + continue + } + if _, ok := s.accountReqs[reqid]; ok { + continue + } + break + } + // Generate the network query and send it to the peer + req := &accountRequest{ + peer: idle, + id: reqid, + cancel: cancel, + stale: make(chan struct{}), + origin: task.Next, + limit: task.Last, + task: task, + } + req.timeout = time.AfterFunc(requestTimeout, func() { + log.Debug("Account range request timed out") + select { + case s.accountReqFails <- req: + default: + } + }) + s.accountReqs[reqid] = req + delete(s.accountIdlers, idle) + + s.pend.Add(1) + go func(peer *Peer, root common.Hash) { + defer s.pend.Done() + + // Attempt to send the remote request and revert if it fails + if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, maxRequestSize); err != nil { + peer.Log().Debug("Failed to request account range", "err", err) + select { + case s.accountReqFails <- req: + default: + } + } + // Request successfully sent, start a + }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists + + // Inject the request into the task to block further assignments + task.req = req + } +} + +// assignBytecodeTasks attempts to match idle peers to pending code retrievals. +func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { + s.lock.Lock() + defer s.lock.Unlock() + + // If there are no idle peers, short circuit assignment + if len(s.bytecodeIdlers) == 0 { + return + } + // Iterate over all the tasks and try to find a pending one + for _, task := range s.tasks { + // Skip any tasks not in the bytecode retrieval phase + if task.res == nil { + continue + } + // Skip tasks that are already retrieving (or done with) all codes + if len(task.codeTasks) == 0 { + continue + } + // Task pending retrieval, try to find an idle peer. If no such peer + // exists, we probably assigned tasks for all (or they are stateless). + // Abort the entire assignment mechanism. + var idle string + for id := range s.bytecodeIdlers { + // If the peer rejected a query in this sync cycle, don't bother asking + // again for anything, it's either out of sync or already pruned + if _, ok := s.statelessPeers[id]; ok { + continue + } + idle = id + break + } + if idle == "" { + return + } + // Matched a pending task to an idle peer, allocate a unique request id + var reqid uint64 + for { + reqid = uint64(rand.Int63()) + if reqid == 0 { + continue + } + if _, ok := s.bytecodeReqs[reqid]; ok { + continue + } + break + } + // Generate the network query and send it to the peer + hashes := make([]common.Hash, 0, maxCodeRequestCount) + for hash := range task.codeTasks { + delete(task.codeTasks, hash) + hashes = append(hashes, hash) + if len(hashes) >= maxCodeRequestCount { + break + } + } + req := &bytecodeRequest{ + peer: idle, + id: reqid, + cancel: cancel, + stale: make(chan struct{}), + hashes: hashes, + task: task, + } + req.timeout = time.AfterFunc(requestTimeout, func() { + log.Debug("Bytecode request timed out") + select { + case s.bytecodeReqFails <- req: + default: + } + }) + s.bytecodeReqs[reqid] = req + delete(s.bytecodeIdlers, idle) + + s.pend.Add(1) + go func(peer *Peer) { + defer s.pend.Done() + + // Attempt to send the remote request and revert if it fails + if err := peer.RequestByteCodes(reqid, hashes, maxRequestSize); err != nil { + log.Debug("Failed to request bytecodes", "err", err) + select { + case s.bytecodeReqFails <- req: + default: + } + } + // Request successfully sent, start a + }(s.peers[idle]) // We're in the lock, peers[id] surely exists + } +} + +// assignStorageTasks attempts to match idle peers to pending storage range +// retrievals. +func (s *Syncer) assignStorageTasks(cancel chan struct{}) { + s.lock.Lock() + defer s.lock.Unlock() + + // If there are no idle peers, short circuit assignment + if len(s.storageIdlers) == 0 { + return + } + // Iterate over all the tasks and try to find a pending one + for _, task := range s.tasks { + // Skip any tasks not in the storage retrieval phase + if task.res == nil { + continue + } + // Skip tasks that are already retrieving (or done with) all small states + if len(task.SubTasks) == 0 && len(task.stateTasks) == 0 { + continue + } + // Task pending retrieval, try to find an idle peer. If no such peer + // exists, we probably assigned tasks for all (or they are stateless). + // Abort the entire assignment mechanism. + var idle string + for id := range s.storageIdlers { + // If the peer rejected a query in this sync cycle, don't bother asking + // again for anything, it's either out of sync or already pruned + if _, ok := s.statelessPeers[id]; ok { + continue + } + idle = id + break + } + if idle == "" { + return + } + // Matched a pending task to an idle peer, allocate a unique request id + var reqid uint64 + for { + reqid = uint64(rand.Int63()) + if reqid == 0 { + continue + } + if _, ok := s.storageReqs[reqid]; ok { + continue + } + break + } + // Generate the network query and send it to the peer. If there are + // large contract tasks pending, complete those before diving into + // even more new contracts. + var ( + accounts = make([]common.Hash, 0, maxStorageSetRequestCount) + roots = make([]common.Hash, 0, maxStorageSetRequestCount) + subtask *storageTask + ) + for account, subtasks := range task.SubTasks { + for _, st := range subtasks { + // Skip any subtasks already filling + if st.req != nil { + continue + } + // Found an incomplete storage chunk, schedule it + accounts = append(accounts, account) + roots = append(roots, st.root) + + subtask = st + break // Large contract chunks are downloaded individually + } + if subtask != nil { + break // Large contract chunks are downloaded individually + } + } + if subtask == nil { + // No large contract required retrieval, but small ones available + for acccount, root := range task.stateTasks { + delete(task.stateTasks, acccount) + + accounts = append(accounts, acccount) + roots = append(roots, root) + + if len(accounts) >= maxStorageSetRequestCount { + break + } + } + } + // If nothing was found, it means this task is actually already fully + // retrieving, but large contracts are hard to detect. Skip to the next. + if len(accounts) == 0 { + continue + } + req := &storageRequest{ + peer: idle, + id: reqid, + cancel: cancel, + stale: make(chan struct{}), + accounts: accounts, + roots: roots, + mainTask: task, + subTask: subtask, + } + if subtask != nil { + req.origin = subtask.Next + req.limit = subtask.Last + } + req.timeout = time.AfterFunc(requestTimeout, func() { + log.Debug("Storage request timed out") + select { + case s.storageReqFails <- req: + default: + } + }) + s.storageReqs[reqid] = req + delete(s.storageIdlers, idle) + + s.pend.Add(1) + go func(peer *Peer, root common.Hash) { + defer s.pend.Done() + + // Attempt to send the remote request and revert if it fails + var origin, limit []byte + if subtask != nil { + origin, limit = req.origin[:], req.limit[:] + } + if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, maxRequestSize); err != nil { + log.Debug("Failed to request storage", "err", err) + select { + case s.storageReqFails <- req: + default: + } + } + // Request successfully sent, start a + }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists + + // Inject the request into the subtask to block further assignments + if subtask != nil { + subtask.req = req + } + } +} + +// assignTrienodeHealTasks attempts to match idle peers to trie node requests to +// heal any trie errors caused by the snap sync's chunked retrieval model. +func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { + s.lock.Lock() + defer s.lock.Unlock() + + // If there are no idle peers, short circuit assignment + if len(s.trienodeHealIdlers) == 0 { + return + } + // Iterate over pending tasks and try to find a peer to retrieve with + for len(s.healer.trieTasks) > 0 || s.healer.scheduler.Pending() > 0 { + // If there are not enough trie tasks queued to fully assign, fill the + // queue from the state sync scheduler. The trie synced schedules these + // together with bytecodes, so we need to queue them combined. + var ( + have = len(s.healer.trieTasks) + len(s.healer.codeTasks) + want = maxTrieRequestCount + maxCodeRequestCount + ) + if have < want { + nodes, paths, codes := s.healer.scheduler.Missing(want - have) + for i, hash := range nodes { + s.healer.trieTasks[hash] = paths[i] + } + for _, hash := range codes { + s.healer.codeTasks[hash] = struct{}{} + } + } + // If all the heal tasks are bytecodes or already downloading, bail + if len(s.healer.trieTasks) == 0 { + return + } + // Task pending retrieval, try to find an idle peer. If no such peer + // exists, we probably assigned tasks for all (or they are stateless). + // Abort the entire assignment mechanism. + var idle string + for id := range s.trienodeHealIdlers { + // If the peer rejected a query in this sync cycle, don't bother asking + // again for anything, it's either out of sync or already pruned + if _, ok := s.statelessPeers[id]; ok { + continue + } + idle = id + break + } + if idle == "" { + return + } + // Matched a pending task to an idle peer, allocate a unique request id + var reqid uint64 + for { + reqid = uint64(rand.Int63()) + if reqid == 0 { + continue + } + if _, ok := s.trienodeHealReqs[reqid]; ok { + continue + } + break + } + // Generate the network query and send it to the peer + var ( + hashes = make([]common.Hash, 0, maxTrieRequestCount) + paths = make([]trie.SyncPath, 0, maxTrieRequestCount) + pathsets = make([]TrieNodePathSet, 0, maxTrieRequestCount) + ) + for hash, pathset := range s.healer.trieTasks { + delete(s.healer.trieTasks, hash) + + hashes = append(hashes, hash) + paths = append(paths, pathset) + pathsets = append(pathsets, [][]byte(pathset)) // TODO(karalabe): group requests by account hash + + if len(hashes) >= maxTrieRequestCount { + break + } + } + req := &trienodeHealRequest{ + peer: idle, + id: reqid, + cancel: cancel, + stale: make(chan struct{}), + hashes: hashes, + paths: paths, + task: s.healer, + } + req.timeout = time.AfterFunc(requestTimeout, func() { + log.Debug("Trienode heal request timed out") + select { + case s.trienodeHealReqFails <- req: + default: + } + }) + s.trienodeHealReqs[reqid] = req + delete(s.trienodeHealIdlers, idle) + + s.pend.Add(1) + go func(peer *Peer, root common.Hash) { + defer s.pend.Done() + + // Attempt to send the remote request and revert if it fails + if err := peer.RequestTrieNodes(reqid, root, pathsets, maxRequestSize); err != nil { + log.Debug("Failed to request trienode healers", "err", err) + select { + case s.trienodeHealReqFails <- req: + default: + } + } + // Request successfully sent, start a + }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists + } +} + +// assignBytecodeHealTasks attempts to match idle peers to bytecode requests to +// heal any trie errors caused by the snap sync's chunked retrieval model. +func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { + s.lock.Lock() + defer s.lock.Unlock() + + // If there are no idle peers, short circuit assignment + if len(s.bytecodeHealIdlers) == 0 { + return + } + // Iterate over pending tasks and try to find a peer to retrieve with + for len(s.healer.codeTasks) > 0 || s.healer.scheduler.Pending() > 0 { + // If there are not enough trie tasks queued to fully assign, fill the + // queue from the state sync scheduler. The trie synced schedules these + // together with trie nodes, so we need to queue them combined. + var ( + have = len(s.healer.trieTasks) + len(s.healer.codeTasks) + want = maxTrieRequestCount + maxCodeRequestCount + ) + if have < want { + nodes, paths, codes := s.healer.scheduler.Missing(want - have) + for i, hash := range nodes { + s.healer.trieTasks[hash] = paths[i] + } + for _, hash := range codes { + s.healer.codeTasks[hash] = struct{}{} + } + } + // If all the heal tasks are trienodes or already downloading, bail + if len(s.healer.codeTasks) == 0 { + return + } + // Task pending retrieval, try to find an idle peer. If no such peer + // exists, we probably assigned tasks for all (or they are stateless). + // Abort the entire assignment mechanism. + var idle string + for id := range s.bytecodeHealIdlers { + // If the peer rejected a query in this sync cycle, don't bother asking + // again for anything, it's either out of sync or already pruned + if _, ok := s.statelessPeers[id]; ok { + continue + } + idle = id + break + } + if idle == "" { + return + } + // Matched a pending task to an idle peer, allocate a unique request id + var reqid uint64 + for { + reqid = uint64(rand.Int63()) + if reqid == 0 { + continue + } + if _, ok := s.bytecodeHealReqs[reqid]; ok { + continue + } + break + } + // Generate the network query and send it to the peer + hashes := make([]common.Hash, 0, maxCodeRequestCount) + for hash := range s.healer.codeTasks { + delete(s.healer.codeTasks, hash) + + hashes = append(hashes, hash) + if len(hashes) >= maxCodeRequestCount { + break + } + } + req := &bytecodeHealRequest{ + peer: idle, + id: reqid, + cancel: cancel, + stale: make(chan struct{}), + hashes: hashes, + task: s.healer, + } + req.timeout = time.AfterFunc(requestTimeout, func() { + log.Debug("Bytecode heal request timed out") + select { + case s.bytecodeHealReqFails <- req: + default: + } + }) + s.bytecodeHealReqs[reqid] = req + delete(s.bytecodeHealIdlers, idle) + + s.pend.Add(1) + go func(peer *Peer) { + defer s.pend.Done() + + // Attempt to send the remote request and revert if it fails + if err := peer.RequestByteCodes(reqid, hashes, maxRequestSize); err != nil { + log.Debug("Failed to request bytecode healers", "err", err) + select { + case s.bytecodeHealReqFails <- req: + default: + } + } + // Request successfully sent, start a + }(s.peers[idle]) // We're in the lock, peers[id] surely exists + } +} + +// revertRequests locates all the currently pending reuqests from a particular +// peer and reverts them, rescheduling for others to fulfill. +func (s *Syncer) revertRequests(peer string) { + // Gather the requests first, revertals need the lock too + s.lock.Lock() + var accountReqs []*accountRequest + for _, req := range s.accountReqs { + if req.peer == peer { + accountReqs = append(accountReqs, req) + } + } + var bytecodeReqs []*bytecodeRequest + for _, req := range s.bytecodeReqs { + if req.peer == peer { + bytecodeReqs = append(bytecodeReqs, req) + } + } + var storageReqs []*storageRequest + for _, req := range s.storageReqs { + if req.peer == peer { + storageReqs = append(storageReqs, req) + } + } + var trienodeHealReqs []*trienodeHealRequest + for _, req := range s.trienodeHealReqs { + if req.peer == peer { + trienodeHealReqs = append(trienodeHealReqs, req) + } + } + var bytecodeHealReqs []*bytecodeHealRequest + for _, req := range s.bytecodeHealReqs { + if req.peer == peer { + bytecodeHealReqs = append(bytecodeHealReqs, req) + } + } + s.lock.Unlock() + + // Revert all the requests matching the peer + for _, req := range accountReqs { + s.revertAccountRequest(req) + } + for _, req := range bytecodeReqs { + s.revertBytecodeRequest(req) + } + for _, req := range storageReqs { + s.revertStorageRequest(req) + } + for _, req := range trienodeHealReqs { + s.revertTrienodeHealRequest(req) + } + for _, req := range bytecodeHealReqs { + s.revertBytecodeHealRequest(req) + } +} + +// revertAccountRequest cleans up an account range request and returns all failed +// retrieval tasks to the scheduler for reassignment. +func (s *Syncer) revertAccountRequest(req *accountRequest) { + log.Trace("Reverting account request", "peer", req.peer, "reqid", req.id) + select { + case <-req.stale: + log.Trace("Account request already reverted", "peer", req.peer, "reqid", req.id) + return + default: + } + close(req.stale) + + // Remove the request from the tracked set + s.lock.Lock() + delete(s.accountReqs, req.id) + s.lock.Unlock() + + // If there's a timeout timer still running, abort it and mark the account + // task as not-pending, ready for resheduling + req.timeout.Stop() + if req.task.req == req { + req.task.req = nil + } +} + +// revertBytecodeRequest cleans up an bytecode request and returns all failed +// retrieval tasks to the scheduler for reassignment. +func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { + log.Trace("Reverting bytecode request", "peer", req.peer) + select { + case <-req.stale: + log.Trace("Bytecode request already reverted", "peer", req.peer, "reqid", req.id) + return + default: + } + close(req.stale) + + // Remove the request from the tracked set + s.lock.Lock() + delete(s.bytecodeReqs, req.id) + s.lock.Unlock() + + // If there's a timeout timer still running, abort it and mark the code + // retrievals as not-pending, ready for resheduling + req.timeout.Stop() + for _, hash := range req.hashes { + req.task.codeTasks[hash] = struct{}{} + } +} + +// revertStorageRequest cleans up a storage range request and returns all failed +// retrieval tasks to the scheduler for reassignment. +func (s *Syncer) revertStorageRequest(req *storageRequest) { + log.Trace("Reverting storage request", "peer", req.peer) + select { + case <-req.stale: + log.Trace("Storage request already reverted", "peer", req.peer, "reqid", req.id) + return + default: + } + close(req.stale) + + // Remove the request from the tracked set + s.lock.Lock() + delete(s.storageReqs, req.id) + s.lock.Unlock() + + // If there's a timeout timer still running, abort it and mark the storage + // task as not-pending, ready for resheduling + req.timeout.Stop() + if req.subTask != nil { + req.subTask.req = nil + } else { + for i, account := range req.accounts { + req.mainTask.stateTasks[account] = req.roots[i] + } + } +} + +// revertTrienodeHealRequest cleans up an trienode heal request and returns all +// failed retrieval tasks to the scheduler for reassignment. +func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { + log.Trace("Reverting trienode heal request", "peer", req.peer) + select { + case <-req.stale: + log.Trace("Trienode heal request already reverted", "peer", req.peer, "reqid", req.id) + return + default: + } + close(req.stale) + + // Remove the request from the tracked set + s.lock.Lock() + delete(s.trienodeHealReqs, req.id) + s.lock.Unlock() + + // If there's a timeout timer still running, abort it and mark the trie node + // retrievals as not-pending, ready for resheduling + req.timeout.Stop() + for i, hash := range req.hashes { + req.task.trieTasks[hash] = [][]byte(req.paths[i]) + } +} + +// revertBytecodeHealRequest cleans up an bytecode request and returns all failed +// retrieval tasks to the scheduler for reassignment. +func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { + log.Trace("Reverting bytecode heal request", "peer", req.peer) + select { + case <-req.stale: + log.Trace("Bytecode heal request already reverted", "peer", req.peer, "reqid", req.id) + return + default: + } + close(req.stale) + + // Remove the request from the tracked set + s.lock.Lock() + delete(s.bytecodeHealReqs, req.id) + s.lock.Unlock() + + // If there's a timeout timer still running, abort it and mark the code + // retrievals as not-pending, ready for resheduling + req.timeout.Stop() + for _, hash := range req.hashes { + req.task.codeTasks[hash] = struct{}{} + } +} + +// processAccountResponse integrates an already validated account range response +// into the account tasks. +func (s *Syncer) processAccountResponse(res *accountResponse) { + // Switch the task from pending to filling + res.task.req = nil + res.task.res = res + + // Ensure that the response doesn't overflow into the subsequent task + last := res.task.Last.Big() + for i, hash := range res.hashes { + if hash.Big().Cmp(last) > 0 { + // Chunk overflown, cut off excess, but also update the boundary nodes + for j := i; j < len(res.hashes); j++ { + if err := res.trie.Prove(res.hashes[j][:], 0, res.overflow); err != nil { + panic(err) // Account range was already proven, what happened + } + } + res.hashes = res.hashes[:i] + res.accounts = res.accounts[:i] + res.cont = false // Mark range completed + break + } + } + // Itereate over all the accounts and assemble which ones need further sub- + // filling before the entire account range can be persisted. + res.task.needCode = make([]bool, len(res.accounts)) + res.task.needState = make([]bool, len(res.accounts)) + res.task.needHeal = make([]bool, len(res.accounts)) + + res.task.codeTasks = make(map[common.Hash]struct{}) + res.task.stateTasks = make(map[common.Hash]common.Hash) + + resumed := make(map[common.Hash]struct{}) + + res.task.pend = 0 + for i, account := range res.accounts { + // Check if the account is a contract with an unknown code + if !bytes.Equal(account.CodeHash, emptyCode[:]) { + if code := rawdb.ReadCodeWithPrefix(s.db, common.BytesToHash(account.CodeHash)); code == nil { + res.task.codeTasks[common.BytesToHash(account.CodeHash)] = struct{}{} + res.task.needCode[i] = true + res.task.pend++ + } + } + // Check if the account is a contract with an unknown storage trie + if account.Root != emptyRoot { + if node, err := s.db.Get(account.Root[:]); err != nil || node == nil { + // If there was a previous large state retrieval in progress, + // don't restart it from scratch. This happens if a sync cycle + // is interrupted and resumed later. However, *do* update the + // previous root hash. + if subtasks, ok := res.task.SubTasks[res.hashes[i]]; ok { + log.Error("Resuming large storage retrieval", "account", res.hashes[i], "root", account.Root) + for _, subtask := range subtasks { + subtask.root = account.Root + } + res.task.needHeal[i] = true + resumed[res.hashes[i]] = struct{}{} + } else { + res.task.stateTasks[res.hashes[i]] = account.Root + } + res.task.needState[i] = true + res.task.pend++ + } + } + } + // Delete any subtasks that have been aborted but not resumed. This may undo + // some progress if a newpeer gives us less accounts than an old one, but for + // now we have to live with that. + for hash := range res.task.SubTasks { + if _, ok := resumed[hash]; !ok { + log.Error("Aborting suspended storage retrieval", "account", hash) + delete(res.task.SubTasks, hash) + } + } + // If the account range contained no contracts, or all have been fully filled + // beforehand, short circuit storage filling and forward to the next task + if res.task.pend == 0 { + s.forwardAccountTask(res.task) + return + } + // Some accounts are incomplete, leave as is for the storage and contract + // task assigners to pick up and fill. +} + +// processBytecodeResponse integrates an already validated bytecode response +// into the account tasks. +func (s *Syncer) processBytecodeResponse(res *bytecodeResponse) { + batch := s.db.NewBatch() + + var ( + codes uint64 + bytes common.StorageSize + ) + for i, hash := range res.hashes { + code := res.codes[i] + + // If the bytecode was not delivered, reschedule it + if code == nil { + res.task.codeTasks[hash] = struct{}{} + continue + } + // Code was delivered, mark it not needed any more + for j, account := range res.task.res.accounts { + if res.task.needCode[j] && hash == common.BytesToHash(account.CodeHash) { + res.task.needCode[j] = false + res.task.pend-- + } + } + // Push the bytecode into a database batch + s.bytecodeSynced++ + s.bytecodeBytes += common.StorageSize(len(code)) + + codes++ + bytes += common.StorageSize(len(code)) + + rawdb.WriteCode(batch, hash, code) + s.bloom.Add(hash[:]) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to persist bytecodes", "err", err) + } + log.Debug("Persisted set of bytecodes", "count", codes, "bytes", bytes) + + // If this delivery completed the last pending task, forward the account task + // to the next chunk + if res.task.pend == 0 { + s.forwardAccountTask(res.task) + return + } + // Some accounts are still incomplete, leave as is for the storage and contract + // task assigners to pick up and fill. +} + +// processStorageResponse integrates an already validated storage response +// into the account tasks. +func (s *Syncer) processStorageResponse(res *storageResponse) { + // Switch the suntask from pending to idle + if res.subTask != nil { + res.subTask.req = nil + } + batch := s.db.NewBatch() + + var ( + slots int + nodes int + skipped int + bytes common.StorageSize + ) + // Iterate over all the accounts and reconstruct their storage tries from the + // delivered slots + delivered := make(map[common.Hash]bool) + for i := 0; i < len(res.hashes); i++ { + delivered[res.roots[i]] = true + } + for i, account := range res.accounts { + // If the account was not delivered, reschedule it + if i >= len(res.hashes) { + if !delivered[res.roots[i]] { + res.mainTask.stateTasks[account] = res.roots[i] + } + continue + } + // State was delivered, if complete mark as not needed any more, otherwise + // mark the account as needing healing + for j, acc := range res.mainTask.res.accounts { + if res.roots[i] == acc.Root { + // If the packet contains multiple contract storage slots, all + // but the last are surely complete. The last contract may be + // chunked, so check it's continuation flag. + if res.subTask == nil && res.mainTask.needState[j] && (i < len(res.hashes)-1 || !res.cont) { + res.mainTask.needState[j] = false + res.mainTask.pend-- + } + // If the last contract was chunked, mark it as needing healing + // to avoid writing it out to disk prematurely. + if res.subTask == nil && !res.mainTask.needHeal[j] && i == len(res.hashes)-1 && res.cont { + res.mainTask.needHeal[j] = true + } + // If the last contract was chunked, we need to switch to large + // contract handling mode + if res.subTask == nil && i == len(res.hashes)-1 && res.cont { + // If we haven't yet started a large-contract retrieval, create + // the subtasks for it within the main account task + if tasks, ok := res.mainTask.SubTasks[account]; !ok { + var ( + next common.Hash + ) + step := new(big.Int).Sub( + new(big.Int).Div( + new(big.Int).Exp(common.Big2, common.Big256, nil), + big.NewInt(storageConcurrency), + ), common.Big1, + ) + for k := 0; k < storageConcurrency; k++ { + last := common.BigToHash(new(big.Int).Add(next.Big(), step)) + if k == storageConcurrency-1 { + // Make sure we don't overflow if the step is not a proper divisor + last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + } + tasks = append(tasks, &storageTask{ + Next: next, + Last: last, + root: acc.Root, + }) + log.Debug("Created storage sync task", "account", account, "root", acc.Root, "from", next, "last", last) + next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) + } + res.mainTask.SubTasks[account] = tasks + + // Since we've just created the sub-tasks, this response + // is surely for the first one (zero origin) + res.subTask = tasks[0] + } + } + // If we're in large contract delivery mode, forward the subtask + if res.subTask != nil { + // Ensure the response doesn't overflow into the subsequent task + last := res.subTask.Last.Big() + for k, hash := range res.hashes[i] { + if hash.Big().Cmp(last) > 0 { + // Chunk overflown, cut off excess, but also update the boundary + for l := k; l < len(res.hashes[i]); l++ { + if err := res.tries[i].Prove(res.hashes[i][l][:], 0, res.overflow); err != nil { + panic(err) // Account range was already proven, what happened + } + } + res.hashes[i] = res.hashes[i][:k] + res.slots[i] = res.slots[i][:k] + res.cont = false // Mark range completed + break + } + } + // Forward the relevant storage chunk (even if created just now) + if res.cont { + res.subTask.Next = common.BigToHash(new(big.Int).Add(res.hashes[i][len(res.hashes[i])-1].Big(), big.NewInt(1))) + } else { + res.subTask.done = true + } + } + } + } + // Iterate over all the reconstructed trie nodes and push them to disk + slots += len(res.hashes[i]) + + it := res.nodes[i].NewIterator(nil, nil) + for it.Next() { + // Boundary nodes are not written for the last result, since they are incomplete + if i == len(res.hashes)-1 { + if _, ok := res.bounds[common.BytesToHash(it.Key())]; ok { + skipped++ + continue + } + } + // Node is not a boundary, persist to disk + batch.Put(it.Key(), it.Value()) + s.bloom.Add(it.Key()) + + bytes += common.StorageSize(common.HashLength + len(it.Value())) + nodes++ + } + it.Release() + } + if err := batch.Write(); err != nil { + log.Crit("Failed to persist storage slots", "err", err) + } + s.storageSynced += uint64(slots) + s.storageBytes += bytes + + log.Debug("Persisted set of storage slots", "accounts", len(res.hashes), "slots", slots, "nodes", nodes, "skipped", skipped, "bytes", bytes) + + // If this delivery completed the last pending task, forward the account task + // to the next chunk + if res.mainTask.pend == 0 { + s.forwardAccountTask(res.mainTask) + return + } + // Some accounts are still incomplete, leave as is for the storage and contract + // task assigners to pick up and fill. +} + +// processTrienodeHealResponse integrates an already validated trienode response +// into the healer tasks. +func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { + for i, hash := range res.hashes { + node := res.nodes[i] + + // If the trie node was not delivered, reschedule it + if node == nil { + res.task.trieTasks[hash] = res.paths[i] + continue + } + // Push the trie node into the state syncer + s.trienodeHealSynced++ + s.trienodeHealBytes += common.StorageSize(len(node)) + + err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + switch err { + case nil: + case trie.ErrAlreadyProcessed: + s.trienodeHealDups++ + case trie.ErrNotRequested: + s.trienodeHealNops++ + default: + log.Error("Invalid trienode processed", "hash", hash, "err", err) + } + } + batch := s.db.NewBatch() + if err := s.healer.scheduler.Commit(batch); err != nil { + log.Error("Failed to commit healing data", "err", err) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to persist healing data", "err", err) + } + log.Debug("Persisted set of healing data", "bytes", common.StorageSize(batch.ValueSize())) +} + +// processBytecodeHealResponse integrates an already validated bytecode response +// into the healer tasks. +func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) { + for i, hash := range res.hashes { + node := res.codes[i] + + // If the trie node was not delivered, reschedule it + if node == nil { + res.task.codeTasks[hash] = struct{}{} + continue + } + // Push the trie node into the state syncer + s.bytecodeHealSynced++ + s.bytecodeHealBytes += common.StorageSize(len(node)) + + err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + switch err { + case nil: + case trie.ErrAlreadyProcessed: + s.bytecodeHealDups++ + case trie.ErrNotRequested: + s.bytecodeHealNops++ + default: + log.Error("Invalid bytecode processed", "hash", hash, "err", err) + } + } + batch := s.db.NewBatch() + if err := s.healer.scheduler.Commit(batch); err != nil { + log.Error("Failed to commit healing data", "err", err) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to persist healing data", "err", err) + } + log.Debug("Persisted set of healing data", "bytes", common.StorageSize(batch.ValueSize())) +} + +// forwardAccountTask takes a filled account task and persists anything available +// into the database, after which it forwards the next account marker so that the +// task's next chunk may be filled. +func (s *Syncer) forwardAccountTask(task *accountTask) { + // Remove any pending delivery + res := task.res + if res == nil { + return // nothing to forward + } + task.res = nil + + // Iterate over all the accounts and gather all the incomplete trie nodes. A + // node is incomplete if we haven't yet filled it (sync was interrupted), or + // if we filled it in multiple chunks (storage trie), in which case the few + // nodes on the chunk boundaries are missing. + incompletes := light.NewNodeSet() + for i := range res.accounts { + // If the filling was interrupted, mark everything after as incomplete + if task.needCode[i] || task.needState[i] { + for j := i; j < len(res.accounts); j++ { + if err := res.trie.Prove(res.hashes[j][:], 0, incompletes); err != nil { + panic(err) // Account range was already proven, what happened + } + } + break + } + // Filling not interrupted until this point, mark incomplete if needs healing + if task.needHeal[i] { + if err := res.trie.Prove(res.hashes[i][:], 0, incompletes); err != nil { + panic(err) // Account range was already proven, what happened + } + } + } + // Persist every finalized trie node that's not on the boundary + batch := s.db.NewBatch() + + var ( + nodes int + skipped int + bytes common.StorageSize + ) + it := res.nodes.NewIterator(nil, nil) + for it.Next() { + // Boundary nodes are not written, since they are incomplete + if _, ok := res.bounds[common.BytesToHash(it.Key())]; ok { + skipped++ + continue + } + // Overflow nodes are not written, since they mess with another task + if _, err := res.overflow.Get(it.Key()); err == nil { + skipped++ + continue + } + // Accounts with split storage requests are incomplete + if _, err := incompletes.Get(it.Key()); err == nil { + skipped++ + continue + } + // Node is neither a boundary, not an incomplete account, persist to disk + batch.Put(it.Key(), it.Value()) + s.bloom.Add(it.Key()) + + bytes += common.StorageSize(common.HashLength + len(it.Value())) + nodes++ + } + it.Release() + + if err := batch.Write(); err != nil { + log.Crit("Failed to persist accounts", "err", err) + } + s.accountBytes += bytes + s.accountSynced += uint64(len(res.accounts)) + + log.Debug("Persisted range of accounts", "accounts", len(res.accounts), "nodes", nodes, "skipped", skipped, "bytes", bytes) + + // Task filling persisted, push it the chunk marker forward to the first + // account still missing data. + for i, hash := range res.hashes { + if task.needCode[i] || task.needState[i] { + return + } + task.Next = common.BigToHash(new(big.Int).Add(hash.Big(), big.NewInt(1))) + } + // All accounts marked as complete, track if the entire task is done + task.done = !res.cont +} + +// OnAccounts is a callback method to invoke when a range of accounts are +// received from a remote peer. +func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, accounts [][]byte, proof [][]byte) error { + size := common.StorageSize(len(hashes) * common.HashLength) + for _, account := range accounts { + size += common.StorageSize(len(account)) + } + for _, node := range proof { + size += common.StorageSize(len(node)) + } + logger := peer.logger.New("reqid", id) + logger.Trace("Delivering range of accounts", "hashes", len(hashes), "accounts", len(accounts), "proofs", len(proof), "bytes", size) + + // Whether or not the response is valid, we can mark the peer as idle and + // notify the scheduler to assign a new task. If the response is invalid, + // we'll drop the peer in a bit. + s.lock.Lock() + if _, ok := s.peers[peer.id]; ok { + s.accountIdlers[peer.id] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + // Ensure the response is for a valid request + req, ok := s.accountReqs[id] + if !ok { + // Request stale, perhaps the peer timed out but came through in the end + logger.Warn("Unexpected account range packet") + s.lock.Unlock() + return nil + } + delete(s.accountReqs, id) + + // Clean up the request timeout timer, we'll see how to proceed further based + // on the actual delivered content + req.timeout.Stop() + + // Response is valid, but check if peer is signalling that it does not have + // the requested data. For account range queries that means the state being + // retrieved was either already pruned remotely, or the peer is not yet + // synced to our head. + if len(hashes) == 0 && len(accounts) == 0 && len(proof) == 0 { + logger.Debug("Peer rejected account range request", "root", s.root) + s.statelessPeers[peer.id] = struct{}{} + s.lock.Unlock() + return nil + } + root := s.root + s.lock.Unlock() + + // Reconstruct a partial trie from the response and verify it + keys := make([][]byte, len(hashes)) + for i, key := range hashes { + keys[i] = common.CopyBytes(key[:]) + } + nodes := make(light.NodeList, len(proof)) + for i, node := range proof { + nodes[i] = node + } + proofdb := nodes.NodeSet() + + var end []byte + if len(keys) > 0 { + end = keys[len(keys)-1] + } + db, tr, notary, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) + if err != nil { + logger.Warn("Account range failed proof", "err", err) + return err + } + // Partial trie reconstructed, send it to the scheduler for storage filling + bounds := make(map[common.Hash]struct{}) + + it := notary.Accessed().NewIterator(nil, nil) + for it.Next() { + bounds[common.BytesToHash(it.Key())] = struct{}{} + } + it.Release() + + accs := make([]*state.Account, len(accounts)) + for i, account := range accounts { + acc := new(state.Account) + if err := rlp.DecodeBytes(account, acc); err != nil { + panic(err) // We created these blobs, we must be able to decode them + } + accs[i] = acc + } + response := &accountResponse{ + task: req.task, + hashes: hashes, + accounts: accs, + nodes: db, + trie: tr, + bounds: bounds, + overflow: light.NewNodeSet(), + cont: cont, + } + select { + case s.accountResps <- response: + case <-req.cancel: + case <-req.stale: + } + return nil +} + +// OnByteCodes is a callback method to invoke when a batch of contract +// bytes codes are received from a remote peer. +func (s *Syncer) OnByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { + s.lock.RLock() + syncing := len(s.tasks) > 0 + s.lock.RUnlock() + + if syncing { + return s.onByteCodes(peer, id, bytecodes) + } + return s.onHealByteCodes(peer, id, bytecodes) +} + +// onByteCodes is a callback method to invoke when a batch of contract +// bytes codes are received from a remote peer in the syncing phase. +func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { + var size common.StorageSize + for _, code := range bytecodes { + size += common.StorageSize(len(code)) + } + logger := peer.logger.New("reqid", id) + logger.Trace("Delivering set of bytecodes", "bytecodes", len(bytecodes), "bytes", size) + + // Whether or not the response is valid, we can mark the peer as idle and + // notify the scheduler to assign a new task. If the response is invalid, + // we'll drop the peer in a bit. + s.lock.Lock() + if _, ok := s.peers[peer.id]; ok { + s.bytecodeIdlers[peer.id] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + // Ensure the response is for a valid request + req, ok := s.bytecodeReqs[id] + if !ok { + // Request stale, perhaps the peer timed out but came through in the end + logger.Warn("Unexpected bytecode packet") + s.lock.Unlock() + return nil + } + delete(s.bytecodeReqs, id) + + // Clean up the request timeout timer, we'll see how to proceed further based + // on the actual delivered content + req.timeout.Stop() + + // Response is valid, but check if peer is signalling that it does not have + // the requested data. For bytecode range queries that means the peer is not + // yet synced. + if len(bytecodes) == 0 { + logger.Debug("Peer rejected bytecode request") + s.statelessPeers[peer.id] = struct{}{} + s.lock.Unlock() + return nil + } + s.lock.Unlock() + + // Cross reference the requested bytecodes with the response to find gaps + // that the serving node is missing + hasher := sha3.NewLegacyKeccak256() + + codes := make([][]byte, len(req.hashes)) + for i, j := 0, 0; i < len(bytecodes); i++ { + // Find the next hash that we've been served, leaving misses with nils + hasher.Reset() + hasher.Write(bytecodes[i]) + hash := hasher.Sum(nil) + + for j < len(req.hashes) && !bytes.Equal(hash, req.hashes[j][:]) { + j++ + } + if j < len(req.hashes) { + codes[j] = bytecodes[i] + j++ + continue + } + // We've either ran out of hashes, or got unrequested data + logger.Warn("Unexpected bytecodes", "count", len(bytecodes)-i) + return errors.New("unexpected bytecode") + } + // Response validated, send it to the scheduler for filling + response := &bytecodeResponse{ + task: req.task, + hashes: req.hashes, + codes: codes, + } + select { + case s.bytecodeResps <- response: + case <-req.cancel: + case <-req.stale: + } + return nil +} + +// OnStorage is a callback method to invoke when ranges of storage slots +// are received from a remote peer. +func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots [][][]byte, proof [][]byte) error { + // Gather some trace stats to aid in debugging issues + var ( + hashCount int + slotCount int + size common.StorageSize + ) + for _, hashset := range hashes { + size += common.StorageSize(common.HashLength * len(hashset)) + hashCount += len(hashset) + } + for _, slotset := range slots { + for _, slot := range slotset { + size += common.StorageSize(len(slot)) + } + slotCount += len(slotset) + } + for _, node := range proof { + size += common.StorageSize(len(node)) + } + logger := peer.logger.New("reqid", id) + logger.Trace("Delivering ranges of storage slots", "accounts", len(hashes), "hashes", hashCount, "slots", slotCount, "proofs", len(proof), "size", size) + + // Whether or not the response is valid, we can mark the peer as idle and + // notify the scheduler to assign a new task. If the response is invalid, + // we'll drop the peer in a bit. + s.lock.Lock() + if _, ok := s.peers[peer.id]; ok { + s.storageIdlers[peer.id] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + // Ensure the response is for a valid request + req, ok := s.storageReqs[id] + if !ok { + // Request stale, perhaps the peer timed out but came through in the end + logger.Warn("Unexpected storage ranges packet") + s.lock.Unlock() + return nil + } + delete(s.storageReqs, id) + + // Clean up the request timeout timer, we'll see how to proceed further based + // on the actual delivered content + req.timeout.Stop() + + // Reject the response if the hash sets and slot sets don't match, or if the + // peer sent more data than requested. + if len(hashes) != len(slots) { + s.lock.Unlock() + logger.Warn("Hash and slot set size mismatch", "hashset", len(hashes), "slotset", len(slots)) + return errors.New("hash and slot set size mismatch") + } + if len(hashes) > len(req.accounts) { + s.lock.Unlock() + logger.Warn("Hash set larger than requested", "hashset", len(hashes), "requested", len(req.accounts)) + return errors.New("hash set larger than requested") + } + // Response is valid, but check if peer is signalling that it does not have + // the requested data. For storage range queries that means the state being + // retrieved was either already pruned remotely, or the peer is not yet + // synced to our head. + if len(hashes) == 0 { + logger.Debug("Peer rejected storage request") + s.statelessPeers[peer.id] = struct{}{} + s.lock.Unlock() + return nil + } + s.lock.Unlock() + + // Reconstruct the partial tries from the response and verify them + var ( + dbs = make([]ethdb.KeyValueStore, len(hashes)) + tries = make([]*trie.Trie, len(hashes)) + notary *trie.KeyValueNotary + cont bool + ) + for i := 0; i < len(hashes); i++ { + // Convert the keys and proofs into an internal format + keys := make([][]byte, len(hashes[i])) + for j, key := range hashes[i] { + keys[j] = common.CopyBytes(key[:]) + } + nodes := make(light.NodeList, 0, len(proof)) + if i == len(hashes)-1 { + for _, node := range proof { + nodes = append(nodes, node) + } + } + var err error + if len(nodes) == 0 { + // No proof has been attached, the response must cover the entire key + // space and hash to the origin root. + dbs[i], tries[i], _, _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) + if err != nil { + logger.Warn("Storage slots failed proof", "err", err) + return err + } + } else { + // A proof was attached, the response is only partial, check that the + // returned data is indeed part of the storage trie + proofdb := nodes.NodeSet() + + var end []byte + if len(keys) > 0 { + end = keys[len(keys)-1] + } + dbs[i], tries[i], notary, cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) + if err != nil { + logger.Warn("Storage range failed proof", "err", err) + return err + } + } + } + // Partial tries reconstructed, send them to the scheduler for storage filling + bounds := make(map[common.Hash]struct{}) + + if notary != nil { // if all contract storages are delivered in full, no notary will be created + it := notary.Accessed().NewIterator(nil, nil) + for it.Next() { + bounds[common.BytesToHash(it.Key())] = struct{}{} + } + it.Release() + } + response := &storageResponse{ + mainTask: req.mainTask, + subTask: req.subTask, + accounts: req.accounts, + roots: req.roots, + hashes: hashes, + slots: slots, + nodes: dbs, + tries: tries, + bounds: bounds, + overflow: light.NewNodeSet(), + cont: cont, + } + select { + case s.storageResps <- response: + case <-req.cancel: + case <-req.stale: + } + return nil +} + +// OnTrieNodes is a callback method to invoke when a batch of trie nodes +// are received from a remote peer. +func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { + var size common.StorageSize + for _, node := range trienodes { + size += common.StorageSize(len(node)) + } + logger := peer.logger.New("reqid", id) + logger.Trace("Delivering set of healing trienodes", "trienodes", len(trienodes), "bytes", size) + + // Whether or not the response is valid, we can mark the peer as idle and + // notify the scheduler to assign a new task. If the response is invalid, + // we'll drop the peer in a bit. + s.lock.Lock() + if _, ok := s.peers[peer.id]; ok { + s.trienodeHealIdlers[peer.id] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + // Ensure the response is for a valid request + req, ok := s.trienodeHealReqs[id] + if !ok { + // Request stale, perhaps the peer timed out but came through in the end + logger.Warn("Unexpected trienode heal packet") + s.lock.Unlock() + return nil + } + delete(s.trienodeHealReqs, id) + + // Clean up the request timeout timer, we'll see how to proceed further based + // on the actual delivered content + req.timeout.Stop() + + // Response is valid, but check if peer is signalling that it does not have + // the requested data. For bytecode range queries that means the peer is not + // yet synced. + if len(trienodes) == 0 { + logger.Debug("Peer rejected trienode heal request") + s.statelessPeers[peer.id] = struct{}{} + s.lock.Unlock() + return nil + } + s.lock.Unlock() + + // Cross reference the requested trienodes with the response to find gaps + // that the serving node is missing + hasher := sha3.NewLegacyKeccak256() + + nodes := make([][]byte, len(req.hashes)) + for i, j := 0, 0; i < len(trienodes); i++ { + // Find the next hash that we've been served, leaving misses with nils + hasher.Reset() + hasher.Write(trienodes[i]) + hash := hasher.Sum(nil) + + for j < len(req.hashes) && !bytes.Equal(hash, req.hashes[j][:]) { + j++ + } + if j < len(req.hashes) { + nodes[j] = trienodes[i] + j++ + continue + } + // We've either ran out of hashes, or got unrequested data + logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + return errors.New("unexpected healing trienode") + } + // Response validated, send it to the scheduler for filling + response := &trienodeHealResponse{ + task: req.task, + hashes: req.hashes, + paths: req.paths, + nodes: nodes, + } + select { + case s.trienodeHealResps <- response: + case <-req.cancel: + case <-req.stale: + } + return nil +} + +// onHealByteCodes is a callback method to invoke when a batch of contract +// bytes codes are received from a remote peer in the healing phase. +func (s *Syncer) onHealByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { + var size common.StorageSize + for _, code := range bytecodes { + size += common.StorageSize(len(code)) + } + logger := peer.logger.New("reqid", id) + logger.Trace("Delivering set of healing bytecodes", "bytecodes", len(bytecodes), "bytes", size) + + // Whether or not the response is valid, we can mark the peer as idle and + // notify the scheduler to assign a new task. If the response is invalid, + // we'll drop the peer in a bit. + s.lock.Lock() + if _, ok := s.peers[peer.id]; ok { + s.bytecodeHealIdlers[peer.id] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + // Ensure the response is for a valid request + req, ok := s.bytecodeHealReqs[id] + if !ok { + // Request stale, perhaps the peer timed out but came through in the end + logger.Warn("Unexpected bytecode heal packet") + s.lock.Unlock() + return nil + } + delete(s.bytecodeHealReqs, id) + + // Clean up the request timeout timer, we'll see how to proceed further based + // on the actual delivered content + req.timeout.Stop() + + // Response is valid, but check if peer is signalling that it does not have + // the requested data. For bytecode range queries that means the peer is not + // yet synced. + if len(bytecodes) == 0 { + logger.Debug("Peer rejected bytecode heal request") + s.statelessPeers[peer.id] = struct{}{} + s.lock.Unlock() + return nil + } + s.lock.Unlock() + + // Cross reference the requested bytecodes with the response to find gaps + // that the serving node is missing + hasher := sha3.NewLegacyKeccak256() + + codes := make([][]byte, len(req.hashes)) + for i, j := 0, 0; i < len(bytecodes); i++ { + // Find the next hash that we've been served, leaving misses with nils + hasher.Reset() + hasher.Write(bytecodes[i]) + hash := hasher.Sum(nil) + + for j < len(req.hashes) && !bytes.Equal(hash, req.hashes[j][:]) { + j++ + } + if j < len(req.hashes) { + codes[j] = bytecodes[i] + j++ + continue + } + // We've either ran out of hashes, or got unrequested data + logger.Warn("Unexpected healing bytecodes", "count", len(bytecodes)-i) + return errors.New("unexpected healing bytecode") + } + // Response validated, send it to the scheduler for filling + response := &bytecodeHealResponse{ + task: req.task, + hashes: req.hashes, + codes: codes, + } + select { + case s.bytecodeHealResps <- response: + case <-req.cancel: + case <-req.stale: + } + return nil +} + +// hashSpace is the total size of the 256 bit hash space for accounts. +var hashSpace = new(big.Int).Exp(common.Big2, common.Big256, nil) + +// report calculates various status reports and provides it to the user. +func (s *Syncer) report(force bool) { + if len(s.tasks) > 0 { + s.reportSyncProgress(force) + return + } + s.reportHealProgress(force) +} + +// reportSyncProgress calculates various status reports and provides it to the user. +func (s *Syncer) reportSyncProgress(force bool) { + // Don't report all the events, just occasionally + if !force && time.Since(s.logTime) < 3*time.Second { + return + } + // Don't report anything until we have a meaningful progress + synced := s.accountBytes + s.bytecodeBytes + s.storageBytes + if synced == 0 { + return + } + accountGaps := new(big.Int) + for _, task := range s.tasks { + accountGaps.Add(accountGaps, new(big.Int).Sub(task.Last.Big(), task.Next.Big())) + } + accountFills := new(big.Int).Sub(hashSpace, accountGaps) + if accountFills.BitLen() == 0 { + return + } + s.logTime = time.Now() + estBytes := float64(new(big.Int).Div( + new(big.Int).Mul(new(big.Int).SetUint64(uint64(synced)), hashSpace), + accountFills, + ).Uint64()) + + elapsed := time.Since(s.startTime) + estTime := elapsed / time.Duration(synced) * time.Duration(estBytes) + + // Create a mega progress report + var ( + progress = fmt.Sprintf("%.2f%%", float64(synced)*100/estBytes) + accounts = fmt.Sprintf("%d@%v", s.accountSynced, s.accountBytes.TerminalString()) + storage = fmt.Sprintf("%d@%v", s.storageSynced, s.storageBytes.TerminalString()) + bytecode = fmt.Sprintf("%d@%v", s.bytecodeSynced, s.bytecodeBytes.TerminalString()) + ) + log.Info("State sync in progress", "synced", progress, "state", synced, + "accounts", accounts, "slots", storage, "codes", bytecode, "eta", common.PrettyDuration(estTime-elapsed)) +} + +// reportHealProgress calculates various status reports and provides it to the user. +func (s *Syncer) reportHealProgress(force bool) { + // Don't report all the events, just occasionally + if !force && time.Since(s.logTime) < 3*time.Second { + return + } + s.logTime = time.Now() + + // Create a mega progress report + var ( + trienode = fmt.Sprintf("%d@%v", s.trienodeHealSynced, s.trienodeHealBytes.TerminalString()) + bytecode = fmt.Sprintf("%d@%v", s.bytecodeHealSynced, s.bytecodeHealBytes.TerminalString()) + ) + log.Info("State heal in progress", "nodes", trienode, "codes", bytecode, + "pending", s.healer.scheduler.Pending()) +} diff --git a/eth/sync.go b/eth/sync.go index 26badd1e21..03a5165245 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" ) @@ -40,12 +41,12 @@ const ( ) type txsync struct { - p *peer + p *eth.Peer txs []*types.Transaction } // syncTransactions starts sending all currently pending transactions to the given peer. -func (pm *ProtocolManager) syncTransactions(p *peer) { +func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote // peer. Fun fact, this is quite an expensive operation as it needs to sort // the transactions if the sorting is not cached yet. However, with a random @@ -53,7 +54,7 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { // // TODO(karalabe): Figure out if we could get away with random order somehow var txs types.Transactions - pending, _ := pm.txpool.Pending() + pending, _ := h.txpool.Pending() for _, batch := range pending { txs = append(txs, batch...) } @@ -63,7 +64,7 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - if p.version >= eth65 { + if p.Version() >= eth.ETH65 { hashes := make([]common.Hash, len(txs)) for i, tx := range txs { hashes[i] = tx.Hash() @@ -73,8 +74,8 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { } // Out of luck, peer is running legacy protocols, drop the txs over select { - case pm.txsyncCh <- &txsync{p: p, txs: txs}: - case <-pm.quitSync: + case h.txsyncCh <- &txsync{p: p, txs: txs}: + case <-h.quitSync: } } @@ -82,8 +83,8 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { // connection. When a new peer appears, we relay all currently pending // transactions. In order to minimise egress bandwidth usage, we send // the transactions in small packs to one peer at a time. -func (pm *ProtocolManager) txsyncLoop64() { - defer pm.wg.Done() +func (h *handler) txsyncLoop64() { + defer h.wg.Done() var ( pending = make(map[enode.ID]*txsync) @@ -94,7 +95,7 @@ func (pm *ProtocolManager) txsyncLoop64() { // send starts a sending a pack of transactions from the sync. send := func(s *txsync) { - if s.p.version >= eth65 { + if s.p.Version() >= eth.ETH65 { panic("initial transaction syncer running on eth/65+") } // Fill pack with transactions up to the target size. @@ -108,14 +109,13 @@ func (pm *ProtocolManager) txsyncLoop64() { // Remove the transactions that will be sent. s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] if len(s.txs) == 0 { - delete(pending, s.p.ID()) + delete(pending, s.p.Peer.ID()) } // Send the pack in the background. s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) sending = true - go func() { done <- pack.p.SendTransactions64(pack.txs) }() + go func() { done <- pack.p.SendTransactions(pack.txs) }() } - // pick chooses the next pending sync. pick := func() *txsync { if len(pending) == 0 { @@ -132,8 +132,8 @@ func (pm *ProtocolManager) txsyncLoop64() { for { select { - case s := <-pm.txsyncCh: - pending[s.p.ID()] = s + case s := <-h.txsyncCh: + pending[s.p.Peer.ID()] = s if !sending { send(s) } @@ -142,13 +142,13 @@ func (pm *ProtocolManager) txsyncLoop64() { // Stop tracking peers that cause send failures. if err != nil { pack.p.Log().Debug("Transaction send failed", "err", err) - delete(pending, pack.p.ID()) + delete(pending, pack.p.Peer.ID()) } // Schedule the next send. if s := pick(); s != nil { send(s) } - case <-pm.quitSync: + case <-h.quitSync: return } } @@ -156,7 +156,7 @@ func (pm *ProtocolManager) txsyncLoop64() { // chainSyncer coordinates blockchain sync components. type chainSyncer struct { - pm *ProtocolManager + handler *handler force *time.Timer forced bool // true when force timer fired peerEventCh chan struct{} @@ -166,15 +166,15 @@ type chainSyncer struct { // chainSyncOp is a scheduled sync operation. type chainSyncOp struct { mode downloader.SyncMode - peer *peer + peer *eth.Peer td *big.Int head common.Hash } // newChainSyncer creates a chainSyncer. -func newChainSyncer(pm *ProtocolManager) *chainSyncer { +func newChainSyncer(handler *handler) *chainSyncer { return &chainSyncer{ - pm: pm, + handler: handler, peerEventCh: make(chan struct{}), } } @@ -182,23 +182,24 @@ func newChainSyncer(pm *ProtocolManager) *chainSyncer { // handlePeerEvent notifies the syncer about a change in the peer set. // This is called for new peers and every time a peer announces a new // chain head. -func (cs *chainSyncer) handlePeerEvent(p *peer) bool { +func (cs *chainSyncer) handlePeerEvent(peer *eth.Peer) bool { select { case cs.peerEventCh <- struct{}{}: return true - case <-cs.pm.quitSync: + case <-cs.handler.quitSync: return false } } // loop runs in its own goroutine and launches the sync when necessary. func (cs *chainSyncer) loop() { - defer cs.pm.wg.Done() + defer cs.handler.wg.Done() - cs.pm.blockFetcher.Start() - cs.pm.txFetcher.Start() - defer cs.pm.blockFetcher.Stop() - defer cs.pm.txFetcher.Stop() + cs.handler.blockFetcher.Start() + cs.handler.txFetcher.Start() + defer cs.handler.blockFetcher.Stop() + defer cs.handler.txFetcher.Stop() + defer cs.handler.downloader.Terminate() // The force timer lowers the peer count threshold down to one when it fires. // This ensures we'll always start sync even if there aren't enough peers. @@ -209,7 +210,6 @@ func (cs *chainSyncer) loop() { if op := cs.nextSyncOp(); op != nil { cs.startSync(op) } - select { case <-cs.peerEventCh: // Peer information changed, recheck. @@ -220,14 +220,13 @@ func (cs *chainSyncer) loop() { case <-cs.force.C: cs.forced = true - case <-cs.pm.quitSync: + case <-cs.handler.quitSync: // Disable all insertion on the blockchain. This needs to happen before // terminating the downloader because the downloader waits for blockchain // inserts, and these can take a long time to finish. - cs.pm.blockchain.StopInsert() - cs.pm.downloader.Terminate() + cs.handler.chain.StopInsert() + cs.handler.downloader.Terminate() if cs.doneCh != nil { - // Wait for the current sync to end. <-cs.doneCh } return @@ -245,19 +244,22 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { minPeers := defaultMinSyncPeers if cs.forced { minPeers = 1 - } else if minPeers > cs.pm.maxPeers { - minPeers = cs.pm.maxPeers + } else if minPeers > cs.handler.maxPeers { + minPeers = cs.handler.maxPeers } - if cs.pm.peers.Len() < minPeers { + if cs.handler.peers.Len() < minPeers { return nil } - - // We have enough peers, check TD. - peer := cs.pm.peers.BestPeer() + // We have enough peers, check TD + peer := cs.handler.peers.ethPeerWithHighestTD() if peer == nil { return nil } mode, ourTD := cs.modeAndLocalHead() + if mode == downloader.FastSync && atomic.LoadUint32(&cs.handler.snapSync) == 1 { + // Fast sync via the snap protocol + mode = downloader.SnapSync + } op := peerToSyncOp(mode, peer) if op.td.Cmp(ourTD) <= 0 { return nil // We're in sync. @@ -265,42 +267,42 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { return op } -func peerToSyncOp(mode downloader.SyncMode, p *peer) *chainSyncOp { +func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { peerHead, peerTD := p.Head() return &chainSyncOp{mode: mode, peer: p, td: peerTD, head: peerHead} } func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { // If we're in fast sync mode, return that directly - if atomic.LoadUint32(&cs.pm.fastSync) == 1 { - block := cs.pm.blockchain.CurrentFastBlock() - td := cs.pm.blockchain.GetTdByHash(block.Hash()) + if atomic.LoadUint32(&cs.handler.fastSync) == 1 { + block := cs.handler.chain.CurrentFastBlock() + td := cs.handler.chain.GetTdByHash(block.Hash()) return downloader.FastSync, td } // We are probably in full sync, but we might have rewound to before the // fast sync pivot, check if we should reenable - if pivot := rawdb.ReadLastPivotNumber(cs.pm.chaindb); pivot != nil { - if head := cs.pm.blockchain.CurrentBlock(); head.NumberU64() < *pivot { - block := cs.pm.blockchain.CurrentFastBlock() - td := cs.pm.blockchain.GetTdByHash(block.Hash()) + if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { + if head := cs.handler.chain.CurrentBlock(); head.NumberU64() < *pivot { + block := cs.handler.chain.CurrentFastBlock() + td := cs.handler.chain.GetTdByHash(block.Hash()) return downloader.FastSync, td } } // Nope, we're really full syncing - head := cs.pm.blockchain.CurrentHeader() - td := cs.pm.blockchain.GetTd(head.Hash(), head.Number.Uint64()) + head := cs.handler.chain.CurrentHeader() + td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64()) return downloader.FullSync, td } // startSync launches doSync in a new goroutine. func (cs *chainSyncer) startSync(op *chainSyncOp) { cs.doneCh = make(chan error, 1) - go func() { cs.doneCh <- cs.pm.doSync(op) }() + go func() { cs.doneCh <- cs.handler.doSync(op) }() } // doSync synchronizes the local blockchain with a remote peer. -func (pm *ProtocolManager) doSync(op *chainSyncOp) error { - if op.mode == downloader.FastSync { +func (h *handler) doSync(op *chainSyncOp) error { + if op.mode == downloader.FastSync || op.mode == downloader.SnapSync { // Before launch the fast sync, we have to ensure user uses the same // txlookup limit. // The main concern here is: during the fast sync Geth won't index the @@ -310,35 +312,33 @@ func (pm *ProtocolManager) doSync(op *chainSyncOp) error { // has been indexed. So here for the user-experience wise, it's non-optimal // that user can't change limit during the fast sync. If changed, Geth // will just blindly use the original one. - limit := pm.blockchain.TxLookupLimit() - if stored := rawdb.ReadFastTxLookupLimit(pm.chaindb); stored == nil { - rawdb.WriteFastTxLookupLimit(pm.chaindb, limit) + limit := h.chain.TxLookupLimit() + if stored := rawdb.ReadFastTxLookupLimit(h.database); stored == nil { + rawdb.WriteFastTxLookupLimit(h.database, limit) } else if *stored != limit { - pm.blockchain.SetTxLookupLimit(*stored) + h.chain.SetTxLookupLimit(*stored) log.Warn("Update txLookup limit", "provided", limit, "updated", *stored) } } // Run the sync cycle, and disable fast sync if we're past the pivot block - err := pm.downloader.Synchronise(op.peer.id, op.head, op.td, op.mode) + err := h.downloader.Synchronise(op.peer.ID(), op.head, op.td, op.mode) if err != nil { return err } - if atomic.LoadUint32(&pm.fastSync) == 1 { + if atomic.LoadUint32(&h.fastSync) == 1 { log.Info("Fast sync complete, auto disabling") - atomic.StoreUint32(&pm.fastSync, 0) + atomic.StoreUint32(&h.fastSync, 0) } - // If we've successfully finished a sync cycle and passed any required checkpoint, // enable accepting transactions from the network. - head := pm.blockchain.CurrentBlock() - if head.NumberU64() >= pm.checkpointNumber { + head := h.chain.CurrentBlock() + if head.NumberU64() >= h.checkpointNumber { // Checkpoint passed, sanity check the timestamp to have a fallback mechanism // for non-checkpointed (number = 0) private networks. if head.Time() >= uint64(time.Now().AddDate(0, -1, 0).Unix()) { - atomic.StoreUint32(&pm.acceptTxs, 1) + atomic.StoreUint32(&h.acceptTxs, 1) } } - if head.NumberU64() > 0 { // We've completed a sync cycle, notify all peers of new state. This path is // essential in star-topology networks where a gateway node needs to notify @@ -346,8 +346,7 @@ func (pm *ProtocolManager) doSync(op *chainSyncOp) error { // scenario will most often crop up in private and hackathon networks with // degenerate connectivity, but it should be healthy for the mainnet too to // more reliably update peers or the local TD state. - pm.BroadcastBlock(head, false) + h.BroadcastBlock(head, false) } - return nil } diff --git a/eth/sync_test.go b/eth/sync_test.go index ac1e5fad1b..473e19518f 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -22,43 +22,59 @@ import ( "time" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" ) -func TestFastSyncDisabling63(t *testing.T) { testFastSyncDisabling(t, 63) } +// Tests that fast sync is disabled after a successful sync cycle. func TestFastSyncDisabling64(t *testing.T) { testFastSyncDisabling(t, 64) } func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, 65) } // Tests that fast sync gets disabled as soon as a real block is successfully // imported into the blockchain. -func testFastSyncDisabling(t *testing.T, protocol int) { +func testFastSyncDisabling(t *testing.T, protocol uint) { t.Parallel() - // Create a pristine protocol manager, check that fast sync is left enabled - pmEmpty, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil) - if atomic.LoadUint32(&pmEmpty.fastSync) == 0 { + // Create an empty handler and ensure it's in fast sync mode + empty := newTestHandler() + if atomic.LoadUint32(&empty.handler.fastSync) == 0 { t.Fatalf("fast sync disabled on pristine blockchain") } - // Create a full protocol manager, check that fast sync gets disabled - pmFull, _ := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil) - if atomic.LoadUint32(&pmFull.fastSync) == 1 { + defer empty.close() + + // Create a full handler and ensure fast sync ends up disabled + full := newTestHandlerWithBlocks(1024) + if atomic.LoadUint32(&full.handler.fastSync) == 1 { t.Fatalf("fast sync not disabled on non-empty blockchain") } + defer full.close() + + // Sync up the two handlers + emptyPipe, fullPipe := p2p.MsgPipe() + defer emptyPipe.Close() + defer fullPipe.Close() - // Sync up the two peers - io1, io2 := p2p.MsgPipe() - go pmFull.handle(pmFull.newPeer(protocol, p2p.NewPeer(enode.ID{}, "empty", nil), io2, pmFull.txpool.Get)) - go pmEmpty.handle(pmEmpty.newPeer(protocol, p2p.NewPeer(enode.ID{}, "full", nil), io1, pmEmpty.txpool.Get)) + emptyPeer := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), emptyPipe, empty.txpool) + fullPeer := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), fullPipe, full.txpool) + defer emptyPeer.Close() + defer fullPeer.Close() + go empty.handler.runEthPeer(emptyPeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(empty.handler), peer) + }) + go full.handler.runEthPeer(fullPeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(full.handler), peer) + }) + // Wait a bit for the above handlers to start time.Sleep(250 * time.Millisecond) - op := peerToSyncOp(downloader.FastSync, pmEmpty.peers.BestPeer()) - if err := pmEmpty.doSync(op); err != nil { - t.Fatal("sync failed:", err) - } // Check that fast sync was disabled - if atomic.LoadUint32(&pmEmpty.fastSync) == 1 { + op := peerToSyncOp(downloader.FastSync, empty.handler.peers.ethPeerWithHighestTD()) + if err := empty.handler.doSync(op); err != nil { + t.Fatal("sync failed:", err) + } + if atomic.LoadUint32(&empty.handler.fastSync) == 1 { t.Fatalf("fast sync not disabled after successful synchronisation") } } diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 1828ad70fb..e0f4f95ff3 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -36,8 +36,8 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + ethproto "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/les" "github.com/ethereum/go-ethereum/log" @@ -444,13 +444,15 @@ func (s *Service) login(conn *connWrapper) error { // Construct and send the login authentication infos := s.server.NodeInfo() - var network, protocol string + var protocols []string + for _, proto := range s.server.Protocols { + protocols = append(protocols, fmt.Sprintf("%s/%d", proto.Name, proto.Version)) + } + var network string if info := infos.Protocols["eth"]; info != nil { - network = fmt.Sprintf("%d", info.(*eth.NodeInfo).Network) - protocol = fmt.Sprintf("eth/%d", eth.ProtocolVersions[0]) + network = fmt.Sprintf("%d", info.(*ethproto.NodeInfo).Network) } else { network = fmt.Sprintf("%d", infos.Protocols["les"].(*les.NodeInfo).Network) - protocol = fmt.Sprintf("les/%d", les.ClientProtocolVersions[0]) } auth := &authMsg{ ID: s.node, @@ -459,7 +461,7 @@ func (s *Service) login(conn *connWrapper) error { Node: infos.Name, Port: infos.Ports.Listener, Network: network, - Protocol: protocol, + Protocol: strings.Join(protocols, ", "), API: "No", Os: runtime.GOOS, OsVer: runtime.GOARCH, diff --git a/graphql/graphql.go b/graphql/graphql.go index 16a74b4037..22cfcf6637 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1040,10 +1040,6 @@ func (r *Resolver) GasPrice(ctx context.Context) (hexutil.Big, error) { return hexutil.Big(*price), err } -func (r *Resolver) ProtocolVersion(ctx context.Context) (int32, error) { - return int32(r.backend.ProtocolVersion()), nil -} - func (r *Resolver) ChainID(ctx context.Context) (hexutil.Big, error) { return hexutil.Big(*r.backend.ChainConfig().ChainID), nil } diff --git a/graphql/schema.go b/graphql/schema.go index d7b253f227..1fdc370040 100644 --- a/graphql/schema.go +++ b/graphql/schema.go @@ -310,8 +310,6 @@ const schema string = ` # GasPrice returns the node's estimate of a gas price sufficient to # ensure a transaction is mined in a timely fashion. gasPrice: BigInt! - # ProtocolVersion returns the current wire protocol version number. - protocolVersion: Int! # Syncing returns information on the current synchronisation state. syncing: SyncState # ChainID returns the current chain ID for transaction replay protection. diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 030bdb37a4..9ff1781e4a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -64,11 +64,6 @@ func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) return (*hexutil.Big)(price), err } -// ProtocolVersion returns the current Ethereum protocol version this node supports -func (s *PublicEthereumAPI) ProtocolVersion() hexutil.Uint { - return hexutil.Uint(s.b.ProtocolVersion()) -} - // Syncing returns false in case the node is currently not syncing with the network. It can be up to date or has not // yet received the latest block headers from its pears. In case it is synchronizing: // - startingBlock: block number this node started to synchronise from @@ -1905,13 +1900,12 @@ func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) { // PublicNetAPI offers network related RPC methods type PublicNetAPI struct { - net *p2p.Server - networkVersion uint64 + net *p2p.Server } // NewPublicNetAPI creates a new net API instance. -func NewPublicNetAPI(net *p2p.Server, networkVersion uint64) *PublicNetAPI { - return &PublicNetAPI{net, networkVersion} +func NewPublicNetAPI(net *p2p.Server) *PublicNetAPI { + return &PublicNetAPI{net} } // Listening returns an indication if the node is listening for network connections. @@ -1924,11 +1918,6 @@ func (s *PublicNetAPI) PeerCount() hexutil.Uint { return hexutil.Uint(s.net.PeerCount()) } -// Version returns the current ethereum protocol version. -func (s *PublicNetAPI) Version() string { - return fmt.Sprintf("%d", s.networkVersion) -} - // checkTxFee is an internal function used to check whether the fee of // the given transaction is _reasonable_(under the cap). func checkTxFee(gasPrice *big.Int, gas uint64, cap float64) error { diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 10e716bf20..f0a4c0493c 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -41,7 +41,6 @@ import ( type Backend interface { // General Ethereum API Downloader() *downloader.Downloader - ProtocolVersion() int SuggestPrice(ctx context.Context) (*big.Int, error) ChainDb() ethdb.Database AccountManager() *accounts.Manager diff --git a/les/client.go b/les/client.go index 47997a098b..198255dc54 100644 --- a/les/client.go +++ b/les/client.go @@ -171,7 +171,7 @@ func New(stack *node.Node, config *eth.Config) (*LightEthereum, error) { leth.blockchain.DisableCheckFreq() } - leth.netRPCService = ethapi.NewPublicNetAPI(leth.p2pServer, leth.config.NetworkId) + leth.netRPCService = ethapi.NewPublicNetAPI(leth.p2pServer) // Register the backend on the node stack.RegisterAPIs(leth.APIs()) diff --git a/les/enr_entry.go b/les/enr_entry.go index 8f0169bee1..a357f689df 100644 --- a/les/enr_entry.go +++ b/les/enr_entry.go @@ -35,9 +35,9 @@ func (e lesEntry) ENRKey() string { // setupDiscovery creates the node discovery source for the eth protocol. func (eth *LightEthereum) setupDiscovery() (enode.Iterator, error) { - if len(eth.config.DiscoveryURLs) == 0 { + if len(eth.config.EthDiscoveryURLs) == 0 { return nil, nil } client := dnsdisc.NewClient(dnsdisc.Config{}) - return client.NewIterator(eth.config.DiscoveryURLs...) + return client.NewIterator(eth.config.EthDiscoveryURLs...) } diff --git a/les/handler_test.go b/les/handler_test.go index 1612caf427..04277f661b 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -51,7 +51,7 @@ func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func TestGetBlockHeadersLes3(t *testing.T) { testGetBlockHeaders(t, 3) } func testGetBlockHeaders(t *testing.T, protocol int) { - server, tearDown := newServerEnv(t, downloader.MaxHashFetch+15, protocol, nil, false, true, 0) + server, tearDown := newServerEnv(t, downloader.MaxHeaderFetch+15, protocol, nil, false, true, 0) defer tearDown() bc := server.handler.blockchain diff --git a/les/peer.go b/les/peer.go index 31ee8f7f04..6004af03f5 100644 --- a/les/peer.go +++ b/les/peer.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/les/flowcontrol" lpc "github.com/ethereum/go-ethereum/les/lespay/client" lps "github.com/ethereum/go-ethereum/les/lespay/server" @@ -162,9 +161,17 @@ func (p *peerCommons) String() string { return fmt.Sprintf("Peer %s [%s]", p.id, fmt.Sprintf("les/%d", p.version)) } +// PeerInfo represents a short summary of the `eth` sub-protocol metadata known +// about a connected peer. +type PeerInfo struct { + Version int `json:"version"` // Ethereum protocol version negotiated + Difficulty *big.Int `json:"difficulty"` // Total difficulty of the peer's blockchain + Head string `json:"head"` // SHA3 hash of the peer's best owned block +} + // Info gathers and returns a collection of metadata known about a peer. -func (p *peerCommons) Info() *eth.PeerInfo { - return ð.PeerInfo{ +func (p *peerCommons) Info() *PeerInfo { + return &PeerInfo{ Version: p.version, Difficulty: p.Td(), Head: fmt.Sprintf("%x", p.Head()), diff --git a/les/server_handler.go b/les/server_handler.go index f965e3fc64..2316c9c5a4 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -47,7 +47,7 @@ import ( const ( softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header - ethVersion = 63 // equivalent eth version for the downloader + ethVersion = 64 // equivalent eth version for the downloader MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request MaxBodyFetch = 32 // Amount of block bodies to be fetched per retrieval request diff --git a/tests/block_test_util.go b/tests/block_test_util.go index be9cdb70cd..c043f0b3eb 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -147,7 +147,7 @@ func (t *BlockTest) Run(snapshotter bool) error { } // Cross-check the snapshot-to-hash against the trie hash if snapshotter { - if err := snapshot.VerifyState(chain.Snapshot(), chain.CurrentBlock().Root()); err != nil { + if err := snapshot.VerifyState(chain.Snapshots(), chain.CurrentBlock().Root()); err != nil { return err } } diff --git a/tests/fuzzers/rangeproof/corpus/1c14030f26872e57bf1481084f151d71eed8161c-1 b/tests/fuzzers/rangeproof/corpus/1c14030f26872e57bf1481084f151d71eed8161c-1 new file mode 100644 index 0000000000000000000000000000000000000000..31c08bafaff6f7268e7babe1eacae4239d397cf8 GIT binary patch literal 16005 zcmV;0K6=66;pKub3U;CmdmKwV>ns4?hdxdtiYg0XYkEI1tfyIU=vVuJ7grl*SUfuq z?f6g&1SQ-g78xigT0z@|i^hGY{NK1VH-4@U!L%s&^4QtoUBO}`vl+8%BX5;s@PV>o z_TY)faiJeZ#rCz6Q?lz-mhs>@WiM}C38le#3s=8QA7GmE`V2*$E5<%s`!w z{h(&U3fOb?*v%&h1=k3Qzg-FWZ_2iseeNiC0|rlS)?A>WwP1*S z>^+~04F@=PL-NSF)YSa@HAl& z>;MaSE-N3s2UR88A@`prkKW|*8QThA-=yI~)UK&{$vkBn7!YCWXfUPAn@~90koh^7 zTyYJvO2^HFE>6;>l%%nPMB(Iaq?uf?{v}gJ#|UPKNOgy4jDf6|p7WO)Ao`OL`c;Ev z@NGp!=uvCeBMYA5bJ#fvTs_F+kiWf=4;iOAq}rSHfdMVGYYaX#Bp^u4msUwMt)E?R zj-ya&SDNZb2>o&)FpNXz5h0Tflgx58!!^@3cQ=vEJO+FRjUITKQMy9kX`isg6J1)i zR63K{Ilk(7$@5dz2IB$&$0bw;U+IYMF{E!UBN5aUNdyABVWb_)^l3N>?o>C>$~koi zwi&eEY94n=;Jc&Ph5B2w7d&hvBVCdgHvY_)p6t7==9wHCOz8*8q&>p5mh+QtPsicf zJI~k;JRYP?DqMqIz#iQ7P?>-)&)!HEwh$u+3rta_sGebXNt7VGNrQu*g}l7Y+6PAEnc;&;7AUWP?Bse%ZrnDR4$a(-z9bT=@}Q!Gau$ zj>Jcsh0`;4L-NSF)Y+x0%H)JMeQRPi_gau{kan{X|OCx zFXA08d0@NFagrJmbSXEqBF`>i)$-8x;bVHy+G)`?2Gtf4+RddWyP#|0KB4A$H@r(%2K#5zS1IWy(bTL($g{owaB6pp^gxadgX@PZ`=L z0?fN%Ys(ZkQf)4jr-e}|v(=hb-P?vPP^dqi10D?%n`A(Yg-4W1o~W6RrT)UxuJcInWOKm2)OfY?<(S)}m{UD26S zmAQCtn{mI=0wI7F8mB@s%CQNpAzt@I=@nx|!`Mr$@mNd3HrQ}{c4i68N`>{X8Y;fi zi)M`nijHyxiI>E!_b6^*-qE7uGn)>%rx1^>(ghY)Dq`2BzX|iCM{IICh{Kfun}TXn z=b7ToJMO*k`P5T)_m$Q z;Rb3&Q=t8Bl-rl5J4mJ;1C2#Ji#ZTNvneJmzENSARj#ZXk(T~_&@vrjS}^)J%~`8X zS$4J}mjZz+J1I3f2~e{Cm_Z|0Y1v*W2SHI~jBg^!sJz*K(;`swOjbZ~A!CJAS|5__ z-b+ot6;$zE@VHO=t;`Vu$)7B?Q^c`N+V>-seI3h})2g1Kk*DnV>_r}%gx>bWJV>IHUnlo)_q3-r5t zEJX8O{AQrg&M#0wzfZ1L8xQF?U1|UTGNy)RyxAyQH2(~NB-9BM0l}-ZS`+y?9(cwl zS4N&<9>?t2!7$_1nU2i_At&W`XKJSXRO^6usUc_9XE%%*@|*7s3-RA@D$fn8o#}tx z%i20Hc`Ff^G|(v@DF&!B>qEfkL~Plu>WWpit(yXT+t>~`eyC>>s|HRmIAhd6JP_X* zsqM-5rzlR&R8{*RzI|?MhdyW5De9rXITnkHSmZGY#!{wZ zeUU0hcpfhV8pqqmsg2KPA}?QGNi^j?8jS&BGlKZ z^fzEbqTA)#LisOFOl6uMy-*@(?JvX7Y%nk+AU?8$cz!y`_`U-ay&bfgVLeKbx*32I zNk33fJ8y;7X`-@T7*Q6a@J>ST66Z(&1q!pNYnY`D=5}_`;%X(BIQU^CIcfy1Z`$*` z;E3L}!t*clNVuZm3WSp1iz@U|6&wL)+o+EYhj!MhigOGQ2N#o?}ZD%enh*d)3^6e z^bNq1A2pEFneTm_>;zX2iRpOy**4&%qWKfYnw$4aQc0g90Yc{vL&qQ;@o^ApIuW&G zc8xxu!hBwSVc2Pp6_BWZZZ>JNl2kN$fDyyyFY zi}oKFb<(YgKRl3z*9^Z4i}xU(jjANpffNOBSuUTuXh(%dl0i4wN5;m3{WrqIwOqsk zt$^+uiy2gnPkb&g9F-@Tu`euyzG~w5xJ5`$Q-e86x-)z$On5xDxEF-zxSGDSAK#}( zN%%`4^G`~udCq&sNL)}n21=O?cbvFARwdEpF$D|F$?(; z^4H1<(A(M#%*+dlWOo=z2W11XgjN?MwNx~}oB~BG5K4uQ%2S*IWUhX_*!Ae`bn_fe z;H*nly3Aecp;4!4MS}81?8Vp!r+|pFgeS`VdFzcD+YtwHrePV9%B@GoQ< zPqGNve&7mnn~9|1Tw1jcJ?C_8UCd)NG`!9D&r344@i&BCJ8h8~&YQD<-E*fR36E3V z53h(?rB7=dX^O&Fb3>leVo-n!R(h&mr%&+s^~Ctc)zUc${lUC+DkzqdwCI|D0#Rb# z)RSenRow+dFCSczL?+s4N-si#?h7}D(%)M^bDa-+=dY)^Tg*B)0i~FWXg0L6Vm*;7 z-}t(5Wk_*Gm;4yHl%}3lM?n-O9UohR7eWt2g&YB*lES$Bu*mIvd|KE_P4|;P()?ye zP?9-XapnuhR9Z1@tgJDV6Vz*K>RlU-w7-HQz(^>SfNQMtXvvb1-Xb7h(6InS8RnQW&!=1h~4 zP!q-?-J*(A09yN6#XKsjwK=`eNbUn@hg5%4svE^$rf}DoVo@%fwF>!C zCF*c+yum?krDYAxbtz($Z_&;o0I$2!XpKm{$ly$B>aj$l9vAw>#A?5?PlbOAvl`D= zgWvc3<|czbwtF<1)oLW8w}z(Z>5U5y5-M}DCS7jXy}HzZcRkY zY<>qU<_pX1G}yK_1r)96_pITChu2bJunMkCj0W8!aij6e9&Hk<9b$t~;$c!S{7G1| zm>HGtf~uQ~@5_9?i-y#PkHl)Y-DcJErgiqF_G9*Us)!JL8lvd!7D-zH7id3wTgSK) z;dB5EAOJ?3sLNJK`i$FGW1>vKXKHjiD?725y=%Q1(=frpGtuj z)2|AGy+51X#&RRp2OpE?&w>z8Glwy~k2CAX81d$5v2hwXRuJy_(<4K<rq%cg-YR-{e;uhz+HzaZdrCNDg6SWFuB8y6bwJ*IH9bDeOdA?FA%9PUd5#^vm zzS3tub`do*r*DS`xYet~?Kz}H+sBpxAm9ZlYn|ZzFExe8o$*O4oZ8tJ@JuyoHkA=v zcTem6D)1K$Da+o|yT9UJt#HUMGfLA7(S$13>#e2muep%kw?lYB;Gt*pLtl;b z&M6-uoAY(NRtH{%JC+pBm^*$prg`_ID(sAY>z#P2o;N>G0IkHZ%@F>1$VQg3_TlPi zOn5`tr?7Ag+)*c2cWMp5TXShp&j<|Rw$7O)lY zX$l8aVglC9-?f5ZgRa?@0UT!?JZBbGw<>18Sl)GCNd~K7XD=<$^~qD0IqvI}&R4Vz zj+`9pH0m{)lyNd%aq8Lr17!^e*=@rY2D=h?m$h>_6Mh?(E@T29c!0yUXv91OJXt2} zy5PLBbF$jWZUXgplJ9ChRw;X`z+S5QeKMQe6_xPl@+zwD#RedppY*0l85Ar#^~Yv& zi#&Z~wc`~n%-<5N*7E*B4|S36@bYu<;)7K+f2qNpGE{%^e^tV+s(PXMK=qz@s8B!(P*xl0@SHMeze>Va#=(-h=p zFXvrp+8CK`z{X@c1Dno|2!TUK%Mk99Lff8lzuFOT-2{;L$uDjdH_zX)b!!Q`qF03N zx4}N;Bp|7^mWtjbs)s?;s)DW#bXH69|ef@N?z(zdw1yG zkOrk)XJb6%CTLe-v+xPPm%7MJkURWFmv>l_e}x3R*O(UOiW%klR?=IP41XtbF#vpq zr_W1bmR)c4b=b>S+zlRs@lme)@)Tp%`YDvH3pqK4SA|o{F2RQ`uAG;L@N?`7o!zZUab$ z9$ULt^@e!%5;-#~a}nHkx2D9_%*zKLk1#q9kFE0(JBDeOz_jdqM08$x`KvagnvUx* z6uf}B7tquVLaaFpD-{59`}87leorpoQWKJTV5!aHJ{)R3K!!zM3!>5i-DOmjBz4`C zfl>?jm8pJYOd0WL%xeW*`X`YzltKzNiW!}je}(DqzuOsSyqO12c<`DccH$sq zlDGpZni0H_32c9OAEP8f0M~a+AHkHfh&WJMa`>*jWky2zStI0DtqeN2Adq_z*JZ<} zJUfQ!4k#$0MP>QV|Kc&XdK8(1XuR!2&x8e}Cc>Ldox6`f%T8)aG-RRbWlr+;2_&sY zd^@$H8bSC|X0dZD<_@3?D({Cd=O^`tOli}qm}c|4X<-55mzR5U4WaH-D<9O1)1P{O zy$3eEZFSqI9+%Z*Ki>=uIWp3)m^UD}f!(0QTw${RI;|L5X+~1MU{7x1k_aDB#d4 z%&_~wiqJT2pS`A9{0q&%07!BTo;#;x%OTN&Z^t-M`u7aHbxB51>}V zy^9KV?Y7#BA@&C^E@O-%LvC-tNEbyd^-$?NVbn^Ld%BC#{FL;)%pG@bG7>;Sqm{7w zt2i5Bw@-3hi}T06W(Wiw^w_bd0^^z^+9#=viRxWse&w)D4{ddHqItO)iXi1Iv~1jQVmqBJBR0j>lSY#8hi2;6RYbLWXTP%(~oVT+wYMPRocAt*l&4QYd0fovM(N)1K#V34RfGVm#8 zr0>3F@0t#hx{uu@i(l+SIvj*uZuLNK`AOoP34hZ! z_q0=->XiO}0rsScCxptd-V{m{i`_-#+fSqjwnpa&TGjTe+du7< zC8=HP-UecLStbbjyxbUVa}?yb^4%uJ#S4Y~2L@sWLaYA33|idARklwL41*a{oiEcJ)tdm5rcanKlcyeUCWA-Zn07Wbf@Z+iu&5>8Wh7yNc4Y{JukQgI>?LC3duLK+k3 zXX!JecPipHY&^Rl8S+Shbj2|7TMUC4M_VK8uc+Y-+*~ReeZ!ffBFpGoEJU*o*y-gU ztbAVbL)EB$Q3Kl6AzJ(<_{fRd(c^`9d(;xA5(rr}yMwe8Q1}>3Ev-<=3LI%)t|C(K zy#1ASgcK7Dm?%`8x+AVxwfn8FNFX^Hu?G$QbWhm#{?r^&IGLS+n{>F(Ks&^~lZ((^ z|752*9Th~QggJ$(BC_10SX%n2v6yIkoA0>V>0w zRtRSQ#()%c&W_R3YXs(%l`hv4-b?WasHe6YIh5nzQ8R{L0jt=>Vha4}KheJ*J4JcVr)z5Q~iW+c`9K2U1RHDkwQ7>cKT7ujzVGIKz22| z3Tv!pGBqfp>^8O6X@13l@`m2b?Ha($k`en&?M4a;i0}U%VsgxWI>-Q-^GaZZnU0N+ zS>|+zRdg?j6QI+})M^i_eQkn=>(9&wDtPTt-O=YDI4#~CebT~w$e{98XulI-%UYQC zZb5rZNMuRc7;@w*x$X>|_nXm$DR!Ic@D?owF6-Z^aB>ry|=~N!~ zo_1wj!MUr7JBZafgv09@uuCakN$i0}Ec71MUr?gxLy)0#d(R;zjiCOH_iW4RL`2r? zQ`inrb#ry(ym6IJ;}KJ{2Ss3}5{+qC(;a|^4;7~FOgNM!@^N)_&3QdzKePIpHdeOF z9qnhGq)ci!2PGnx>>Nahmnr7#+Vz-ScZ6(S%v69qH|HIAsRgsp-Av4DULJ@g&+GKc zGLP=;kn$KMAmoR{xAfHlCcE1d1q=8NPXn)go3xo<>rm_3P|xHeU9(uuy}sr5L*itX!9S8$PO z8E(bCGx@;;j{@B&;8E?|gfKe`CXs`|tl=_r4oYFsq6DCVAGEPntr84FW57>Q2Qj}K zu|Y~5P3>%GRUAerRDlhK~F%XnSi$a~6{Hd{k>}I*U#$UH5yp zc=}Kh6yWv{M<&R;{Nrd!G%)oOFzkF$WkiZ1*z?rkX~hJ%W~K&Q6`C9}ZQ{n&b^>P9 zOPT;Ru3ngDy81`NyCaUesj}r7H4ah8jx>po#*i0a3zF0IGHW*Tv}(7OweHYJa@*;n zdVS<`U0Z7?wT-wV1sk#(MO6%Ozd+wq;v|KlHYX_P~d?PfZV z453e=HVnXa_6}6UsYc1Q6K{WjV6g~CwEi7WBBdNNSGUmb$3{ltU+O*QXO6~N1L_OG z*@7Xblzz?vXqJp?Tf{aG|Odrp`km!!Pau7;0V+^PGaBSaV7!R8-=PA$hp$@MI1pYYo5i zPlBU1bBIZOk^F=KHT3<)l4bnI$~P=Rmf@07;K>GU3!&PZhldGR{4f(A(MS}0tZJa; z)-f@#mD(snS*^Lo#}DXeX6_3l;6OQXx!1LMIGqELB zvtiMJ2DRm_H?8};Z8$gc{JI%?c7H%_KLRK<5@k&p{HZZqIJ?w%n7S^^gpvw)2womr zFk3_C>xKdpe)sA|eq@4$JCYus`#nPv#}q(M>20y=LK9=Or4kGvbsP$y+En1P;{Q{| zo`54wl45{8RiR|NLJrX9GdIG?ffclmzzPs?$f+vs$BX9HhJ)^lGL{+KE70z1u!5zf z7T+%Wt0xtZa6l*H%(>{E{~Jz>%FJqjbc}L&qtA7usd+Z-;De?tZh3B?73|Qcj&xmE zF#v&%8us|&l_PP70^)*A6|t2P&5a|`%(TGVk{S_Lz4HebVCK_gjHKPx%}-IBa#U7~ zOO-wQV&dvg4mPk8;(OPj%agE-q50%vzw_gy}uOg=nqgW-GVPcS5bsgT=HBaq@na?;J zj{dsFfhCmRo*0Tio1Nu#O1yrC2Ve0aluOT0Fs{Hp4EGJ6`206@sH-pQAfY8E3HaV? z-A{(#7nU)Cxr*hSjdZ>U(ArgoCU^K<{G&~UK+nz{cdXm*8dSS7ALT8^WOOCB${WSq zsQ2$~1VRqx<08;nW&`YI9hj1QdAJqvLHCT!@10$Jp|FLE1+K6ZS2E{A-f;{d-sz*- znhe+fA7euSc$eEYaR2B#-$JAU4C|9aC;7w|O>z>bWz`h%mfBCHTdh??_)@HAu@>d) zyvaIP77|E}zo;5u^mRQYzPt_B@>|-LZchZJ$0s}`%#_Ew#u9T@k^Mq$F(>E0r>2=s z?C=b?M0XvA!qGYV2sRilmm~$!ggRV`8{)aMBFVp_ekl{B-;yZW31$b#2(4#_8|)yY z1;lH2r&%TM6eaVO5?0I;Zq`5dN!_jVG z9LmEi67+{E=IqJYD~B)_Cz9zHIVB|950W14@*zNmrkUvna9!-9;u4zZ$&Ki{DAuqm zTT1QkN8J+KRK*CLZez5yelJ4uIQ%CQkf(3>FR0r7*{fU3*DPr^9904X9;dsUsCzTY z$V=nc<)gIjz1p(N;kEKSuY4g!MQ8^APSZ>r9E|83A43dlE81C)>1z~4Bagu;)BI4l zdq~N&=~swXhIXcDZyy2Ga_G2Jw{3+2THNSphpQ8mdIU!zK1Sj?t8c_{lZzTxwGPdt2C|3IYV!HbAyJPH^k@F`Mn~V3dH)jYZBF z2tAbDE%Z-8ANhRuuW>V429LnYI@KsnQ_3w$Q%ZAx}n)kcZB%izp?SQVfG*XMe|6=@QLs(5JLRo_-iK5h5Wnf zpMPd7XP+3SKzHtn2{QTAn0!FQz<8W3{%W1GNu26xUe$BixXg~(mKrPfZN;s|s4@gK zWQlfqXct0Sb3%zb%_7^q0GzbOnBr~iW<<~{Ca*94P2>Ba4jcspm6~L74Dk>8u-sUi z>oBwrZWy}Qtt%%cByf05^mfKf7&aajG_giq7JchyxI@J*>j%Ar&)GIv2vW<}K`QJM* z^@ic%PSLS}<)yOTxQ$m;F?fmTfb%kqA8(>2wcXi(HQW5I-eBO43gHvnR3qg?sF5>1e|j}Z=I zP<_}@O6t_*oK#729=S|j9)F#M{*(&t!pSfjC75<0}bQ4 z|3J0(E7?vBA3+_t3OAtiNADYU6BQ`X9*_1YBcP zEn21E3tQ8bGWn5xOyJQQaKMD4YFx33p3WH(*I0(#ETJ|;R^6ukQ4s(OF7R6aMj7|z zAewt1lG#rY)JM_O-RMCPKuYgRRM09X9|2snrm+n#_-f)$_Jk|FyOfW6Erx*dkT0fk zCf+jJ>ZfNhR{7o#KTP>fAj5cf2uJ2bn?O!$j=uvVc~{a;3BgVe9Y%m;Az)O%Cvk{V zK(MyOk}knNe6#Kncp~kz3R56=4|z-q3sFoP_TY(PTFMhaIcDQ5&18I8L*z{~moCri zyB8fy;-$xVltCu8iV4D?Y$<24`&(6nk+tG=d$ve%!^z>Sw&AvmWX z9X*;zL2oH+butmyp(3&?Xq@%8@C%$-E=XCjhlVc7`3pmbgFt(fNL?|@ZPi-E)EPFy zyI>+dg#8(;VKK9zZp0(Cq@ATME1~{xBLcCDsorM->gzy@u$zDIu0Q^T8=`Ki8krc6 z<5w|Roru!<@?oa1izIXoPkCo>21}dOqf@A1i(z?7s3M*!KATcOlp(H^o17SN#K~Gf z;#ITmkDXs2Ye9;BocqiF_77i-S*RGH1+S1cKLn=FvRIYsTo*T*kVzftQ-Z2$%mT;4 zaZf=ig3fTqCNy-0hZ?ytO+%|aubVWiOK}V8QK9HznddrYX%P^V2j%T=lD5#H?J9*n z*__0Xe@m^*%uusYGP6bl69URk3Sk9uWrM16)rrS6uYaSjk+2ZHI-6(ZC$feTxiTH? zCy!K(Shz4O#Uik(j}!mMy4g{^>=(fZWM%xx1AZ=jbvTrdL{c9j`5TZ`$^z%+rNTrx z6ihKBA{~R1mA3GM{;IYb$XCOwr*IJZ*1>@$mug<~krzltlT z2d?BQM4d7BxbVVL=9;uM zI{nC=GmgUaafx{JPb7bDM1|-{v6EO{=z<@nV#wD)^7_H1r;Fuj$V{n!OQ*sX7N|al za3LkhdDA2O?D9IypgS!@KPjdNe95=INh1m)M&x&gx=6?)CIBp`0{A;Wz0ZR`{4C$o z|M|Blz_Cw+>#*e#cY%(aD>cX!fGR(@zvz+km#Vc`Be?85%w2L-J!>e2e~32jZ211JJM7Jz3}_~Np;1r0WXQ0~;#F zW{o=MtRi2vc|uT=ZLkHm!S{rfk>;ju*0cn;PwvG=o9O5gb-uL>gSa0diN8Q}bO73X zIO%tEir8Q;-Ic@$fFK+yAVFrSx+u%w5X3uyN81_{6mDH60-q%Kn2BMuBu*$tw1-a4 zdwRvTIUSZ-HGMi4oEp&|__>HJLPJgRioMsvce`vUrqc{@;U)rA;7znO`!&Q?!G8qG zK0^FCM4O)^sv|lyW&XA{*NAne3~H?b50Kfbh~Y)LsJha#c|AuQT(mHuvrxoNE?C44 zv3hfyA`MI=1D}H6N=&sbp<62f?PR@}8TwNe6R}?do{urflU7iK?l+!rO7~$YGOh=+ zA6Oim<@6?~YUfXshliSTcs3}cr2LWirt6h+8H{&|aGHH-5lC)?f zSK6ec)T-t2Mhy=dqP&oavX)IikZ28}JalSBEoDck@x~4=DeR8i?r& zEWr%W6}sE+j?~z~bV3kj56BhpD_G`*u;IG?Fl?EtoyFP*BP{AL!>lIq*Pyt&RSt7%y=? zSdJ@kuG`quc6JCa+e|s0@Q*Vn!f9R)_$8&vFFX05E6VxA4=jTDI+?F-B4E)*)$Qp7 zq2lrZ1?EFqYV?)eQ-bDsmRxkNsCPs_x)T`7u4UdSGXxehA_ zt(*UXR^-sh7ZiPL$eDsTHL{9+ZnZePE=U4ty}H6KepxN9F)e;@?AB!x5{Cp8Br_*4 z?*XOzb9P%Ap-P2gQE-jYVX`xt*}>wB$?L(_(XMIPprn%C{~<2QMEXkF@mJ?Dj21RB2r zzScFu!J?y9M9Owvpi@W8KMvacqYA@yF;4JW{_A^Mfh{5Hn6-qs4BPG3w4Mh>s;SUmt=*mn#DwCrOEuV$aYtSgcbNEI@I3Ged0EPiHy5%iD|t> z7@{o^6nXd9@_=~41h)!)xvginbX)H_2B&khQe>*Vk&+%(Qs9t3pGtkHmNJX& zQRiW!!*jZt7d@b6?3RYBfPEC6wuR?DysdaXXVC%`-*C{$+_98GY_o0?fq#n?F{<%H zYj+uDNLopJ*9_Y}z-O+9iC0B1+gQ>(1`Z1&B)lE84;xap=Im{MtBOf2CJ-KWI-O=; z&Qwwm&}rK<-w%INIwl%w+UL7RzEv(u?pEtppOXcV_nU-gwo$Lwz%F#q6IPq7XY6WW z2we?N`YeF{CceF<+v_d36P_Tdc6t%X8PkR76k-bsv1(s6zUFnFNOw9BDr#HZ9zi@B zmrxBz6uW5C^tvgRgFB7!O&Ql_4h+qGXEtPsSopqrYRIk0p86i3ez?xpY&@hUSPQy1n45UQaT!%rBun}0!nlH^0e!RTidgqNXu1p`S;UBMZU^<%)V3m9cD!{cw7*S zha{HGT7>FfY;fp}h8b772_CBa8jJ4bme6;E*2)5s_sB9Jz5`zz{s1>*#6?uxGS$_~ z_GwF(Uk~N10PZ`@7YbjRySKF5C&GApZR2F!V`C){{?*SpVWpuIjjQD^7a;&9xMDke zVq=5$-GmKOIBQ*Dh+CON__@aKuniY(8CfX|6KE*PtgWV6MsPVmh1VXopLNhu%te&e zrQnji_k{aAmn3s5N05;pm+;Vp>e608n(R*>vUe+-e%GNXg7*6#%b9COW>Ada$SS$A@7j}!xHrWA zLmxbMjNI!G<32&Ltat}#IFnlS64k>et&ar`Mas=!lcH4xccXMpOuV^BiOB7Gw!R@i zBDj61bU5lg?`m96SX}cpD2aJxe3EQP!(cKJrxAN<#I6zMZi9;(AIgnpfqx9=2oP$m zQ5_z?h=~$=CfhL&l{bld6_i%02Z^BXcnn8KEf3r@wd32@OrSp_D;uE9;+8!|cenjK z8JIi2D}Vmun|+nG>!iU-Egx6mbr2^=`HCVa#bi|4$2s3&50snZM!i9wOoDphk?Q;_ znu=ZB^cpKTHdIjFY=N%iDwiwm>wM*y=3g}4`$fs1V4ux>4^dGm7^kL5jiZjHf82|J z=l|J@7JN*C4aR=oz$$A=C4%^NW0^YZ)6_{WK}qas945md?86X?Wwzz~x5RVV-(Nrf zt%V$PWbJEi1mx7&_tHKN!SN+%tg4?#T$;ht`rfP>n>M1=1L}FQh!YHWvQOorK{MXW z>S-eZOy-PXju`qC1$U~lIGj#u;Z4lXl32-S)rZ9w*Qx)}H}e%s=+q*dZGolsAX}Yt z%I=vmX_E#`h<|3tjn(lOowY4R8$#fx5m&jozU4}d=7G6m(3QG3)C)5jMnf?II1#DO zhI}^fxQ>nyJ&w5Y^}o9(lBr6nz14=<4e(LCE@@=f0+n5^$oCRsEp@4H?35vzH>$-G(&t2And|3ZI?1}3_1jF8vM9Q8^wEAvcdw( zsn!=ZjH0wT-GkD1VjJ{DJBPU8{L=`)GVm-9_&JjE(g7wP+7jH;3jV8t7&#NX&-5=? zB-=yws>P|X#WN!(YwIa|dfy4-C_qt1Hz8-2U(|h`POO0k--8w4uJ~7br4yLTmG4}U z1>##Ly%(jEoImUwTUNd)<=!aZwLMkM@zb-g%!-#o8gjGfD)K-0?c|~L+38C;;M zIWxn*zF{^b*>q44pqplj?MCLWRsP;WTelF1#*~&g<`GZ0vjpAz}9E2z20aVZ=1<#10SXzD_Q6I5kC{6ZHpMYJe4`0w9h`|RW z7{1%Y4{dB5v>oGhPzq3}KN#%#P3$C8|DXB*`%a*vxf6BfG;+kq5;eie{W#xl8XR;S z%W^QB`{qyzIbp!h0>H_^^!|`eES@0|DOaw6`n|@SLf5LRFZ72H-E1+|p+KrhdUy>} zFRQ>2479G~PPa|tmbqc;iT{H`65sMF!qV3GDQqU8CrrNKn`h|c>!3Pn;;>IUEKRJw zd2sINRY%73ffdJFCSh~u7{QhHe>1nU0Pzy9U0+6I;VZXb4V1B2%YcR!7xHzlQUFMp zox=}2Pd2gyE$9IecXfi#+hL-IyZ56MX(VZkv2OM6dYfDRWhD3Xwh+kUGP;is_6mXc=K=EyP%;lH`R5sQIN&i z%2Xp>5{W`L}3A^^x<6YU!QWPc##GV}S(yDFeRz}LIB;hyh894repnu3J zDb_bt)Xua@h=O@n4&jOKn(8L$OJ6iRBDZav;!9PW?6x?pU9vI4N+-?TXGl z%mR5VT-6uSkNcJ$+KQ%o4+P5hh#D93h|pPRiyWrkX_hFt$@54-!u|Vekwb$y zVQa4*>8at^>xyeH#qa*b_uF2q3?_1`3XNAGbvF3A1ZMPIyaz|_`sBBL^$E#EZ#PF zDs-KcBYV)n>_hvFD~;d8IvD!$)}6$;{KvtggFu;++&Md<5h+oiI^Z4nDlgJW;vU)Z zz>T+~mvcvy>_EEONvN#`>0fE*w^RBV*a%sx{H!@qCsleL2H8iIbuWG@mU9h*?LndH zp7sfhm(oW94>LNHE6iVAcVM_@HLz0xWS|xJ(xc)642$fws}_aoMd6+SwsVp$Zqwf2n*x@_wv@^4NM?AT41zFlmhmycgmg z5N1&Y(+9$QZ&}4ie6JZP99?g1Au8zeNXL4isuwJlxX=RoZ;%lFXWc?3GDeoVW z04ZY>m;Rv%zoq84Ipk}S@agp^4)`u$!`bAK_R6{HA6DKwhk6iQNurw1vw$0>DbCU& literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/corpus/27e54254422543060a13ea8a4bc913d768e4adb6-2 b/tests/fuzzers/rangeproof/corpus/27e54254422543060a13ea8a4bc913d768e4adb6-2 new file mode 100644 index 0000000000000000000000000000000000000000..7bce13ef80e5a67258a89e3a0e4da1c7610e20bb GIT binary patch literal 15965 zcmV-jKBB?k;pKub3U;CmdmKwV>ns4?hdxdtiYg0XYkEI1tfyIU=vVuJ7grl*SUfuq z?f6g&1SQ-g78xigT0z@|i^hGY{NK1VH-4@U!L%s&^4QtoUBO}`vl+8%BX5;s@PV>o z_TY)faiJeZ#rCz6Q?lz-mhs>@WiM}C38le#3s=8QA7GmE`V2*$E5<%s`!w z{h(&U3fOb?*v%&h1=k3Qzg-FWZ_2iseeNiC0|rlS)?A>WwP1*S z>^+~04F@=PL-NSF)Y$v*=vWThFbta$09Ih zCU7!M(5?B^3Us_9+D}AEj|m;j*WsC$V=#b%$w;fvlIF^OqVR`jZj*RfA>l zZAC@sQES&D3!dV0*f|MYJ;>vbzrB$U8K*j=+MD%(0WGy_3_dg@AV|!YR!KCipIvZ{ zqflyBn(9ah{c<5Nj6>%UA(IZ1%yKouHPbeCH<8Ue27CvN9(bBjxUqiYQ`QFK0s+S*R0m(_i0(0@Z!RMd)D=kt0=r?P9n17-I127mH_*yCbqKZ@ zwBBkScS_}G;gZ;mr#u=f$LSUaQ$(L~r<7^~H3lSr0j%8+m$$TP`3}(7(udRZHv_TW z>%Gg|yQ%&LhUbtT+GwSB2~IMIstn@7aa)4MzGHEpckti|d9n0-OoNdA@Y#Mn;Jc&Ph5B2w7d&hvBVCdgHvY_)p6t7==9wHCOz8*8q&>p5mh+QO$Kl#L&)5$< z9;8hwT!US}9^Ca%nJ>@YNEfycBL@piQKqP#VR%YGAF`>qL(T7`w?utA4z3G=1ydlw z)}vtzn-Q?cS+N%l_5L5F)349{u}NfuKn{M{!)hsTMnuyV%Eesy5naK89Ey&_N1KJy zGj~Js$hy?o_7&IVG5-y6J`it7g08p8%a{UV5S>NsBN&U%#})m#6ESJ9EJ`op9WHrb zyUuZv8WMCVH?$<|d->hHyv)m+3rG8{H(%X4UDKpN2w<+F{(Ea+1z8pd$e9C?%k%_< zn80Lp1aIF`7u;HntKepnb}JFTr}Icsay;WRv3smjf{yp{%CLV=i>x)sK*llaf&1qT zU^6C3JgB}*+q!l@F@@W{_}6I(jdr@X+qf*PMWOC(Gw1OP9@C(fck{IkXtm3i!Bqka zwgxvI%y;{-t9Ww*A`^c3`z?2`(7M4;OGrQCkCSt+$1TKctaB4bph|&ZM zq*18#$m^rwu$+U}ikE-+Rw{}mrq>gYkB?G~ZcLv)Ey1n$hD>lIOcH|1{#iVsZYT=C z1$%mmx_|#9xX~eY)R;6mt1@J`J87i(|{%XVkLtn0D#Vkw5%-Vu09HKUt*l3|-NgQ`P5T)_m$Q;Rb3&Q=t8B zl-rl5J4mJ;1C2#Ji#ZTNvneJmzENSARj#ZXk(T~_&@vrjS}^)J%~`8XS$4J}mjZz+ zJ1I3f2~e{Cm_Z|0Y1v*W2SHI~jBg^!sJz*K(;`swOjbZ~A!CJAS|5__-b+ot6;$zE z@VHO=C?2%RB~9`jSVOI5;y-A zHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV1$K3m7=T<0^t*d3MDt$!W}wi{ zFHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*;!K<`d6ZtwGc*ZAJMxJ6G$L!fK z*GQ<#%Ulru|gwfOn}OXVzyoj2iNr?+pv_-*76=4Xd5$f8NX5Ixu-F5tua4 zDIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_OXA-LhPB1uQ)IdBC-x;ax$@r%zPR>+S z`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ipxex64`k1S2#aWJ!Zd2qrHqN`ZxC_%Q zx=y_KiFDAQZ<+iPs{e_8t;AE@;!>o^X5&SwO*9<7@JcdZ;CKCi0&)iuTeS)v_~5R# z*&0Ed*sNOEP2_fBNGJTLInlJE)@&7ccxIJGSbpHAx)?OJ^2So8V||e-M|d7D1RBTN z;AJIAOx;sT+jD*d6XAce{Tb{B=$BWCMous4Tr_pY>m>-as}4(4`t(c)?)m^k=hBsppXu5a4&yx@r5wZii+^GLX& z;R=M3--{~rQWYElXWOWc4u^KutBP|B5D4dKGE24;`fBItGCa|`#KEirg4AbhOHxUnBLPC^4nxNv9r1AxYB~|MWOj`{pu&7!eqq>Y zj}@1KYAK%k@6BWZZZ>JNl2kN$fDyyyFYi}oLN(yfURvFH^RiVT*LycfbJWM8B~o=d@e8? zl_#08FD!(pOGkhydcs#ba7li1zn!dCj-={}O_)8)4PfDtJ&U?q@ z21|WP^^b9hlkU!FqnBu^KCzhs`uppbTrQ|2+8iVyaZ!2kEJ+&MH_)3{h&fx~&(!q8 z)!O(8de5|d)q9%w6lDQKxA| zg7QY}#n=d^fQYk%C(8YN2;&;vD4xH+F*~5GLG?pU?1!E3FJu}|vIyCJ;0kh^iKO9N zTD1>7=X7pe%wse(yv_K}OER|cH-uh0ZIK$zo3nu3bEhH+k5kfOd6eb-XTZ0!u4@89=0iu$^xcsol?RS1@ zF_aV3YisIV8;-QUf+N64D3yR~tn+Bel9ApbAbn7(I*CfEmF=O@d}Z4Lp~otzy=*ad z1A_{lsSrEcNs?I=i)Ws5I=(UlkGDCqVL8v3&Y5hdJLXK2l2CS1#AatztTFvKdV`qH zc|;}<2as@mW}%e%kTP<()EP*CJu2L=^kbsT(HN!$N*nuTe=*Jq^OK&f38ZXa4TE)J zfIMj&R>nanYJv}sS7)7)cfi2p?~QS^-|{j6?N()FG%n$=XF8RqAD1dAh2rG8IB7r) zKIsir$_YSCO7|ijuw4^eUQC?-J*(A z09yN6#XKsjwK=`eNbUn@hg5%4svE^$rf}DoVo@%fwF>!CCF*c+yum?krDYAxbtz($ zZ_&;o0I$2!XpKm{$ly$B>aj$l9vAw>#A?5?PlbOAvl`D=gWvc3<|czbwtF<1)oLW8 zw}z(Z>5U5y5-M}DCS7jXy}HzZcRkYY<>qU<_pX1G}yK_1r)96 z_pITChu2bJunMkCj0W8!aij6e9&Hk<9b$t~;$c!S{7G1|m>HGtf~uQ~@5_9?i-y#P zkHl)Y-DcJErgiqF_G9*Us)!JL8lvd!7D-zH7id3wTgSK);dB5EAOJ?3sLNJK`i$FGW1>vKXKHjiD?725y=%Q1(=frpGtuj)2|AGy+51X#&RRp2OpE? z&w>z8Glwy~k2CAX81d$5v2hwXRuJy_(<4K<6GkO{_H*Ky^fFig;D&XHZ>7U#M* zByt3$T6tR&wF+e-i%NpEFTEKZT;9KVzEUd6l+yGO<)A~p(q}(*5j8WXZ-)oC)vLtq zIiy9~$Cd#g-~}mbo#6d1HHFBX@kuM3+SwQIOf_mYl@VNbPwV|E@D~?*^kr%(%ihzw zzv5r5aL6w+O4AF`geuqTt)=jCYO1#oI(#u?xBrMVA`X~5H(_99duv=6EE9jV*QY_A^YNM;NV2wgS$^`>B3V|0fKC z?8RF+XZVG>0AY>6T8zX?l&nPCIhY}_uX|v53^>;ouoduW3I|kT0@lsnwSr)SuGy9W z9A_OoXBJksDrUb}-gRF|2CHFbFD=pa$y1j(?(3A!SF{a|oE+;k>NT2_aWYe>DS zWeo_~ZNnD^yApVpwR1TWejAo9WC9;}fWx+E#5@E%Stjhd;JmVPvf9aR0`+&2?`l3) zDSN8GUaIDz@^kUxgH<(usllBxRDbe+Rl=^Sf9-mC-m6>rRZqarvzJFm-!)LZYyj#p0o@tj_|*pQt3!!Z@Zfn^}ZbI+xvL9wAu-khBx3_4r{%Tvf5@jv68>ujH+pl zvcBCu%VEvpfosRp6y#3jYYDreSA^}i!9L|AAgQ&MjMAZ}=7i`d z)H&-7?9F%BpJM!VAjI+F2HRW3V+hnA1&H-ZUg}hPcj(@b2BlqRV?5+0Xjfsg@Cm?| zy2wqCJN!nMcUY2tg#^6Um=@-W8Rhy`(p!`aec|0X${R*1d*A3 z`i63$7-pBT`9AbMV)JO8ikqub*;C};(x6TGFsHz714xD*Tf0~FhIsZ8IWsGB5!`pT zro`6F%LgEjFgg#9t@9E)hH00;wCsFDbY6M+t2U#Wj_WWKynwkE(9{h=tT_uS6##Sl z^dfP7PcGn66Owvhsm2quhUyL|D4|7V`Op92F}HdY znS*G&?L^On1*9gzn@yd&k3h>#YDzR@q3UH$^7aWNtw(%2wWAtA_)})Fb1dc#pbRSS zhcM?S^@mJp)2f(e^Sfzb0ppjKdvgt;?o=xu)Qr=gdVjqKHoa|i+o&Fw)nq^43=KIl z>Avlx7A|7-Wy>Y`IwY?6FfnQco75c7OA(PAXO;f&??NZ`@o9OIBuW4rds?9&Ao+(n(;i9Dgapi`xSBner0hR(``!SkdYrS#sG!VM|h}t z^V4#Prc^vOQHxT)<9C|M#c!zTirv52iy`(0FD_$@BSUU) zz(^NGE%i|8JYm#IlzY01()^V4z04hVZZZ-;LZg+i`l~n_VYg3mT#NI^zGesn9rW0- zrvl@eBibjajfv`AWPataO%H8#tV{b{Hk@D0TMq^{>7crjWqmT)1nuxp-|Ax`d3~W&TEBuHFrd8_s%UmJl#Cir-uvyy^e%T}>{G zc>~Lp@r?R%IwI`;oQ}s@al}+>DBwVy!jH%G)&$*j!Yom?4~$+=j2UFD_H*ZnI8ZT; zcVUa2LPcP=9w8_{4-IL9TY+pERV zU2gS2Z}~~$oe6)_H{?J_W2PSJ0$V#y`koRkYrF-|5BIcFoa&e_ARyW?yZN3JI$#?< zL;?1si6?~0u-+6(6pP(O<=aoB2)0J&2wK(ltJ^>Al_jZN?A``qcv&V0`n=p2ZF3al zxbod5#>EST{RakO212X;zzkaG5#@m%MG0z3c|*(j*i;TG<)XXrw($mqOd~T0*ic=* zBv$c71NaG%yH2e2G+Zq_d8Q}mh_ECc{k9LoBQE!81CFq`rJ&D%Ym5Tfo{q+HsF2>V zI0dIIt%-6U2^vXpOZY%X6)d)4Lw3+8lppS&=_{SDM3ylx^A@= z_n@9{dj+QwPE&Um{B|U4!qb3KaUu~x$G3Mv8WZSe=`*8uD&jY6Ji8zn@<@Sn#W3+( zgBeF#BkZrJ;SJneDjI#mnWG}h=vypAvkutlF(Ks&^~lZ((^|752*9Th~QggJ$(BC_10SX%n2v6yIk zoA0>V>0wRtRSQ#()%c&W_R3YXs(%l`hv4-b?Wa zsHe6YIh5nzQ8R{L0jt=>Vha4}KheJ*J4J z{e*CNDq#IxW9wj%LOJMm`ccY`LSf%Pb~U>SYpiB6H7KL(HnrDje#L?EhThEW8oXcS6Jg6*nD=f$dre4WN!sO4dLSKovPQMSsF32_TOY_? z$qAXT0&7-D1pX^Pfy@<1Wws>#Zx!iO9`~MhWnIC!tBN~_)jEX3>lv_1DPBqJfkrI! z9@bw_qUb}Ap>%uCAtsHW{*L!-%j!f#*6dT*4p4P-b>zHpl~3alQ?my}V5Sm{X;{-8 zfQJtirtVBQlqK?Ub#={oJ!3z!`kFRYw#yyuXPu->YB&caBA4tOM2MFu=Iq+_m|b^- zY+lS%fIT)KGyF(}V8zug^4T+hfvH&AmMMY(^|-ls*hcR7JqqMdK_H|K5w49V4V778%m z$L*;ifDeFZbu3O7@;`!=f1(OXZjJl=*n-jVN+>4CtQC;~O9o>`zT&C%IKqnU>rYp3k!cxj#lAE7!32*2-6-Hu?c9VgI}0X}gTbuf zGIS0~VbP)lpn@N?u~w}T3`1kUPf-UkzZ|haN?}w(O*_RnEhICnFstc|eOJoXqr`q_ zVLXP8{4Z#GYesVxlJtC3Yiv4;PApybd$)M{P!bg2_D3ejy!_*6N;EL_6EN(2QDsDm zBG~iP;c3MLxMrpXTosxeGHv3<)pi1A)JvKGHLhNmXS({kBaXVMvgH~z4pGRCG>MSL zkQZPJlGF7vYc}(=YPXiP?$Ahb+v%fvedKH|7vRJ-I_6S#?myLIJG`WAFfjSw3%Waz zb)Iq+4)7D(@tw{8;~#WsltYm1W;&1zp--YV48V5w4phXcM#;4kZ-0Pbu?R=B{vA&u zr5rO?x6tp$Mn>Xa>OJRYj>cL8>I=cyf+44re$E1ETF@f67li08mR(KUv{W{B1=R7> z4%w-aSc42^fj=dH+AyWvGV3mg5hklo;NI8Isl)b?OxeNbTED8aijj^YtldYR6HV6n zGJ&oF#tPgnpB*q<9Du<83(s^PQmxWUxVs3oH*=wR-U4u;suiZrLn6a3?g1ETUJ&z~ zf}&VTC{$G9jUjos?C@j`C2I}8^G|}KHgkwceUbcx0X6jf#*$_H$I3S>LYCo@QQ*l2 zZ405=n}>%9So|;(AJIq@e5`7q<<>DVu$9^WrnXlCvrPk2N5$8LiM3Lt^B zI;(u4a5+E&-lA(veK@1S=Vg2j8;yDBmTK*|BJZ}}eI`?jOGZu=ak`(An>x`gW*f_e zAZU&ZZHBSWQqMC`i*`;wGYb9?4CjOpE)l(o<#Y3Xu)PsVfR8oeusM2gb$<)Qc=^G# ziDcI&P>3l;6OQXx!1LMIGqELBvtiMJ2DRm_H?8};Z8$gc{JI%?c7H%_KLRK<5@k&p z{HZZqIJ?w%n7S^^gpvw)2womATSMpTh5{6R_v%J|WP*h|k{+M?Jwp=56hKeuZL#Y@ z6JxZc5)2@9915V?RN%AX|5L`EfFn(kVt_qWp=7&44$$W_H^Rw*6||4Q3J`I~sVeTr zi{{pbgYJtmmKoeD(C%xnf~BPv-!A*BCl!!zKquqOx#*t%8%~VM%xZvijBTv6T|djU&;_w7}ew8WC5$ z^9L7T=F?=1q}|rdPf?t5R91{jl|B1n;_6QhHn0=od)J}Mldz64(yC?{8IFl1BS*&|UOhPVDF>mS${E0eUlom%I;^)fl&55Z13ErwyZ6 zC7EGjkXv;f-q}DO9l6-l%74bp$jLq+zU45ajg^LBQ zuoPD^=R@9c3?Sa=quQDb*Z*Tf0eF|&HgNywJKsX20u1YuLnryf7fo^!sAbg@@s`?8 zrCY64L-%7T2SQZjUjlZZGVDxo8CBD23*YaE1mTpf3rpG5dCCrq^yT%f8 zR+0TeZZRk4zo({|PVDduw?uaxhQiS~`v^7|E|(+)(u6u(iW}m&BFVp_ekl{B-;yZW z31$b#2(4#_8|)yY1;lH2r&%TlIa*ZB_!Jqk{<5zAwY(vndt{`UF@Uc z5}N4Ajp(~5*03vEO6~7Q-4fhX#R#2lW3;t?FGBJ-{3jETr*HT#sM`M7t6R+1ENM0z zRRRMZr@Ne}do#+&OXJw(qqOe5+Oo^xwemc#d?7|fXa@jJ(@Y#3jOZL6Lkw#x+F6h3 zYZOHzkHIO^{7|@iNXfM6SBO`JcBW}>9|6^J=(tq3ZG{3_+~{bBs}q!Z2>c_VW~J{} z%qa;nmf^njkZrkLh?SimL##*?W~V^sMVn#}hmk%pgG9Q|^qSYx71!?cum%+0Vg)W4 z=|u33f#Yeo`q2xw{Fd^C9o#(&SW-&%=+gvO!Rd65LrZA?3(RrZ=V%YofsM6S)0qi| z^Vtxq$ty6+yrOfPt9X|u;L2sZn+T9dzfU5UEJtEYEu7uTiBus0tDDLK(;$haO?Lm zo9*sklz`5SMa~!qJ(S%o^iM$_`F!`UaQvkYD=vN3>iSR8D zLj2Ba4jcsp zm6~L74Dk>8u-sUi>oBwrZWy}Qtt%%cByf05^mfKf7&aajG_giq7JchyxI@J*>j%Ar& z)GIv2vW<}K`QJM*^@ic%PSLS}<)yOTxQ$m;F?fmTfb%kqA8(>2wcXi(HQW5I-eBO43gHvnR3 zqg?sF5>1e|j}Z=IP<_}@O6t_*oK#729=S|j9)F#M{*(& zt!pSfjC75<0}bQ4|3J0(E7?vBA3+_t3OAtiNADYU6BQ`X9*_1YBcPEn21E3tQ8bGWn5xOyJQQaKMD4YFx33p3WH(*I0(#ETJ|;R^6uk zQ4s(OF7R6aMj7|zAewt1lG#rY)JM_O-RMCPKuYgRRM09X9|2snrm+n#_-f)$_Jk|F zyOfW6Erx*dkT0fkCf+jJ>ZfNhR{7o#KTP>fAj5cf2uJ2bn?O!$j=uvVc~{a;3BgVe z9Y%m;Az)O%Cvk{VK(MyOk}knNe6#Kncp~kz3R56=4|z-q3sFoP_TY(PTFMhaIcDQ5 z&18I8L*z{~moCriyB8fy;-$xVltCu8iV4D?Y$<24`&(6nk+tG=d$ve%!^z>Sw&Avm zWX9X*;zL2oH+butmyp(3&?Xq@%8@C%$-E=XCjhlVc7`3pmbgFt(f zNL?|@ZPi-E)EPFyyI>+dg#8(;VKK9zZp0(Cq@ATME1~{xBLcCDsorM->gzy@u$zDI zu0Q^T8=`Ki8krc6<5w|Roru!<@?oa1izIXoPkCo>21}dOqf@A1i(z?7s3M*!KATcO zlp(H^o17SN#K~Gf;#ITmkDXs2Ye9;BocqiF_77i-S*RGH1+S1cKLn=FvRIYsTo*T* zkVzftQ-Z2$%mT;4aZf=ig3fTqCNy-0hZ?ytO+%|aubVWiOK}V8QK9HznddrYX%P^V z2j%T=lD5#H?J9*n*__0Xe@m^*%uusYGP6bl69URk3Sk9uWrM16)rrS6uYaSjk+2ZH zI-6(ZC$feTxiTH?Cy!K(Shz4O#Uik(j}!mMy4g{^>=(fZWM%xx1AZ=jbvTrdL{c9j z`5TZ`$^z%+rNTrx6ihKBA{~R1mA3GM{;IYb$XCOwr*IJZ*1>@$mug<~krzltlT2d?BQM4d7BxbVVL=9;uMI{nC=GmgUaafx{JPb7bDM1|-{v6EO{=z<@nV#wD)^7_H1r;Fuj z$V{n!OQ*sX7N|ala3LkhdDA2O?D9IypgS!@KPjdNe95=INh1m)M&x&gx=6?)CIBp` z0{A;Wz0ZR`{4C$o|M|Blz_Cw+>#*e#cY%(aD>cX!fGR(@zv zz+km#Vc`Be?85%w2L-J!>e2e~32jZ211JJM7Jz3}_~Np;1r0WXQ0~;#FW{o=MtRi2vc|uT=ZLkHm!S{rfk>;ju*0cn;PwvG=o9O5gb-uL> zgSa0diN8Q}bO73XIO%tEir8Q;-Ic@$fFK+yAVFrSx+u%w5X3uyN81_{6mDH60-q%K zn2BMuBu*$tw1-a4dwRvTIUSZ-HGMi4oEp&|__>HJLPJgRioMsvce`vUrqc{@;U)rA z;7znO`!&Q?!G8qGK0^FCM4O)^sv|lyW&XA{*NAne3~H?b50Kfbh~Y)LsJha#c|AuQ zT(mHuvrxoNE?C44v3hfyA`MI=1D}H6N=&sbp<62f?PR@}8TwNe6R}?do{urflU7iK z?l+!rO7~$YGOh=+A6Oim<@6?~YUfXshliSTcs3}cr2LWirt6h+8 zH{&|aGHH-5lC)?fSK6ec)T-t2Mhy=dqP&oavX)IikZ28}JalSBEoD zck@x~4=DeR8i?r&EWr%W6}sE+j?~z~bV3kj56BhpD_G`*u;IG?Fl?EtoyFP*BP{AL!>l zIq*Pyt&RSt7%y=?SdJ@kuG`quc6JCa+e|s0@Q*Vn!f9R)_$8&vFFX05E6VxA4=jTD zI+?F-B4E)*)$Qp7q2lrZ1?EFqYV?)eQ-bDsmRxkNsCPs_x) zT`7u4UdSGXxehA_t(*UXR^-sh7ZiPL$eDsTHL{9+ZnZePE=U4ty}H6KepxN9F)e;@ z?AB!x5{Cp8Br_*4?*XOzb9P%Ap-P2gQE-jYVX`xt*}>wB$?L(_(XMIPprn%C{~<2QME zXkF@mJ?Dj21RB2rzScFu!J?y9M9Owvpi@W8KMvacqYA@yF;4JW{_A^Mfh{5Hn6-qs z4BPG3w4Mh>s;SUmt=*mn#DwCrOEuV$aYtSgcbNEI@I3G zed0EPiHy5%iD|t>7@{o^6nXd9@_=~41h)!)xvginbX)H_2B&khQe>*Vk&+%( zQs9t3pGtkHmNJX&QRiW!!*jZt7d@b6?3RYBfPEC6wuR?DysdaXXVC%`-*C{$+_98G zY_o0?fq#n?F{<%HYj+uDNLopJ*9_Y}z-O+9iC0B1+gQ>(1`Z1&B)lE84;xap=Im{M ztBOf2CJ-KWI-O=;&Qwwm&}rK<-w%INIwl%w+UL7RzEv(u?pEtppOXcV_nU-gwo$Lw zz%F#q6IPq7XY6WW2we?N`YeF{CceF<+v_d36P_Tdc6t%X8PkR76k-bsv1(s6zUFnF zNOw9BDr#HZ9zi@BmrxBz6uW5C^tvgRgFB7!O&Ql_4h+qGXEtPsSopqrYRIk0p86i3 zez?xpY&@hUSPQy1n45UQaT!%rBun}0!nlH^0e!RTidgqNXu1p`S;UBMZU^< z%)V3m9cD!{cw7*Sha{HGT7>FfY;fp}h8b772_CBa8jJ4bme6;E*2)5s_sB9Jz5`zz z{s1>*#6?uxGS$_~_GwF(Uk~N10PZ`@7YbjRySKF5C&GApZR2F!V`C){{?*SpVWpuI zjjQD^7a;&9xMDkeVq=5$-GmKOIBQ*Dh+CON__@aKuniY(8CfX|6KE*PtgWV6MsPVm zh1VXopLNhu%te&erQnji_k{aAmn3s5N05;pm+;Vp>e608n(R*>vUe+-e%GNXg7*6#%b9COW>Ada$SS$A@7j}!xHrWALmxbMjNI!G<32&Ltat}#IFnlS64k>et&ar`Mas=!lcH4xccXMp zOuV^BiOB7Gw!R@iBDj61bU5lg?`m96SX}cpD2aJxe3EQP!(cKJrxAN<#I6zMZi9;( zAIgnpfqx9=2oP$mQ5_z?h=~$=CfhL&l{bld6_i%02Z^BXcnn8KEf3r@wd32@OrSp_ zD;uE9;+8!|cenjK8JIi2D}Vmun|+nG>!iU-Egx6mbr2^=`HCVa#bi|4$2s3&50snZ zM!i9wOoDphk?Q;_nu=ZB^cpKTHdIjFY=N%iDwiwm>wM*y=3g}4`$fs1V4ux>4^dGm z7^kL5jiZjHf82|J=l|J@7JN*C4aR=oz$$A=C4%^NW0^YZ)6_{WK}qas945md?86X? zWwzz~x5RVV-(Nrft%V$PWbJEi1mx7&_tHKN!SN+%tg4?#T$;ht`rfP>n>M1=1L}FQ zh!YHWvQOorK{MXW>S-eZOy-PXju`qC1$U~lIGj#u;Z4lXl32-S)rZ9w*Qx)}H}e%s z=+q*dZGolsAX}Yt%I=vmX_E#`h<|3tjn(lOowY4R8$#fx5m&jozU4}d=7G6m(3QG3 z)C)5jMnf?II1#DOhI}^fxQ>nyJ&w5Y^}o9(lBr6nz14=<4e(LCE@@=f0+n5^$oCRsEp@4H?35vzH>$-G(&t2And|3ZI?1}3_1jF z8vM9Q8^wEAvcdw(sn!=ZjH0wT-GkD1VjJ{DJBPU8{L=`)GVm-9_&JjE(g7wP+7jH; z3jV8t7&#NX&-5=?B-=yws>P|X#WN!(YwIa|dfy4-C_qt1Hz8-2U(|h`POO0k--8w4 zuJ~7br4yLTmG4}U1>##Ly%(jEoImUwTUNd)<=!aZwLMkM@zb-g%!-#o8gjGfD)K-0 z?c|~L+38C;;MIWxn*zF{^b*>q44pqplj?MCLWRsP;WTelF1#*~&g<`GZ0vjpAz}9E2z20aVZ=1<#10SXzD_Q6I5k zC{6ZHpMYJe4`0w9h`|RW7{1%Y4{dB5v>oGhPzq3}KN#%#P3$C8|DXB*`%a*vxf6BfG;+kq z5;eie{W#xl8XR;S%W^QB`{qyzIbp!h0>H_^^!|`eES@0|DOaw6`n|@SLf5LRFZ72H z-E1+|p+KrhdUy>}FRQ>2479G~PPa|tmbqc;iT{H`65sMF!qV3GDQqU8CrrNKn`h|c z>!3Pn;;>IUEKRJwd2sINRY%73ffdJFCSh~u7{QhHe>1nU0Pzy9U0+6I;VZXb4V1B2 z%YcR!7xHzlQUFMpox=}2Pd2gyE$9IecXfi#+hL-IyZ56MX(VZkv2OM6dYfDRWhD3Xwh+kUGP;is_6mXc=K=E zyP%;lH`R5sQIN&i%2Xp>5{W`L}3A^^x<6YU!QWPc##GV}S(yDFeRz}LI zB;hyh894repnu3JDb_bt)Xua@h=O@n4&jOKn(8L$OJ6iRBDZav;!9PW?6x z?pU9vI4N+-?TXGl%mR5VT-6uSkNcJ$+KQ%o4+P5hh#D93h|pPRiyWrkX_hFt z$@54-!u|Vekwb$yVQa4*>8at^>xyeH#qa*b_uF2q3?_1`3XNAGbvF3A1ZMPIyaz| z_`sBBL^$E#EZ#PFDs-KcBYV)n>_hvFD~;d8IvD!$)}6$;{KvtggFu;++&Md<5h+oi zI^Z4nDlgJW;vU)Zz>T+~mvcvy>_EEONvN#`>0fE*w^RBV*a%sx{H!@qCsleL2H8iI zbuWG@mU9h*?LndHp7sfhm(oW94>LNHE6iVAcVM_@HLz0xWS|xJ(xc)642$fws}_ao zMd6+SwsVp$Zqwf2n*x@_wv@^4NM? zAT41zFlmhmycgmg5N1&Y(+9$QZ&}4ie6JZP99?g1Au8zeNXL4isuwJlxX=RoZ z;%lFXWc?3GDeoVW04ZY>m;Rv%zoq84Ipk}S@agp^4)`u$!`bAK_R6{HA6DKwhk6iQ LNurw1vw$0>u6V$y literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/corpus/6bfc2cbe2d7a43361e240118439785445a0fdfb7-5 b/tests/fuzzers/rangeproof/corpus/6bfc2cbe2d7a43361e240118439785445a0fdfb7-5 new file mode 100644 index 0000000000000000000000000000000000000000..613e76a020f5870f2e45687b8bdb97a1b570381a GIT binary patch literal 15976 zcmV-uK9|AZ;pKucH90pk3U;CmdmKwV>ns4?hdxdtiYg0XYkEI1tfyIU=vVuJ7grl* zSUfuq?f6g&1SQ-g78xigT0z@|i^hGY{NK1VH-4@U!L%s&^4QtoUBO}`vl+8%BX5;s z@PV>o_TY)faiJeZ#rCz6Q?lz-mhs>@WiM}C38le#3s=8QA7GmE`V2*$E5< z%s`!w{h(&U3fOb?*v%&h1=k3Qzg-FWZ_2iseeNiC0|rlS)?A>W zwP1*S>^+~04F@=PL-NSF)Y$v*=vWThFbta z$09IhCU7!M(5?B^3Us_9+D}AEj|m;j*WsC$V=#b%$w;fvlIF^OqVR`jZj* zRfA>lZAC@sQES&D3!dV0*f|MYJ;>vbzrB$U8K*j=+MD%(0WGy_3_dg@AV|!YR!KCi zpIvZ{qflyBn(9ah{c<5Nj6>%UA(IZ1%yKouHPbeCH<8Ue27CvN9(bBjxUqiYQ`QFK0s+S*R0m(_i0(0@Z!RMd)D=kt0=r?P9n17-I127mH_*yC zbqKZ@wBBkScS_}G;gZ;mr#u=f$LSUaQ$(L~r<7^~H3lSr0j%8+m$$TP`3}(7(udRZ zHv_TW>%Gg|yQ%&LhUbtT+GwSB2~IMIstn@7aa)4MzGHEpckti|d9n0-OoNdA@Y#Mn z;Jc&Ph5B2w7d&hvBVCdgHvY_)p6t7==9wHCOz8*8q&>p5mh+QO$Kl#L z&)5$<9;8hwT!US}9^Ca%nJ>@YNEfycBL@piQKqP#VR%YGAF`>qL(T7`w?utA4z3G= z1ydlw)}vtzn-Q?cS+N%l_5L5F)349{u}NfuKn{M{!)hsTMnuyV%Eesy5naK89Ey&_ zN1KJyGj~Js$hy?o_7&IVG5-y6J`it7g08p8%a{UV5S>NsBN&U%#})m#6ESJ9EJ`op z9WHrbyUuZv8WMCVH?$<|d->hHyv)m+3rG8{H(%X4UDKpN2w<+F{(Ea+1z8pd$e9C? z%k%_x)sK*lla zf&1qTU^6C3JgB}*+q!l@F@@W{_}6I(jdr@X+qf*PMWOC(Gw1OP9@C(fck{IkXtm3i z!BqkawgxvI%y;{-t9Ww*A`^c3`z?2`(7M4;OGrQCkCSt+$1TKctaB4bp zh|&ZMq*18#$m^rwu$+U}ikE-+Rw{}mrq>gYkB?G~ZcLv)Ey1n$hD>lIOcH|1{#iVs zZYT=C1$%mmx_|#9xX~eY)R;6mt1@J`J87i(|{%XVkLtn0D#Vkw5%-Vu09HKUt*l3|-NgQ`P5T)_m$Q;Rb3& zQ=t8Bl-rl5J4mJ;1C2#Ji#ZTNvneJmzENSARj#ZXk(T~_&@vrjS}^)J%~`8XS$4J} zmjZz+J1I3f2~e{Cm_Z|0Y1v*W2SHI~jBg^!sJz*K(;`swOjbZ~A!CJAS|5__-b+ot z6;$zE@VHO=C?2%RB~9`jSVOI z5;y-AHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV1$K3m7=T<0^t*d3MDt$! zW}wi{FHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*;!K<`d6ZtwGc*ZAJMxJ6G z$L!fK*GQ<#%Ulru|gwfOn}OXVzyoj2iNr?+pv_-*76=4Xd5$f8NX5Ixu-F z5tua4DIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_OXA-LhPB1uQ)IdBC-x;ax$@r%z zPR>+S`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ipxex64`k1S2#aWJ!Zd2qrHqN`Z zxC_%Qx=y_KiFDAQZ<+i6s{e_8t;AE@;!>o^X5&SwO*9<7@JcdZ;CKCi0&)iuTeS)v z_~5R#*&0Ed*sNOEP2_fBNGJTLInlJE)@&7ccxIJGSbpHAx)?OJ^2So8V||e-M|d7D z1RBTN;AJIAOx;sT+jD*d6XAce{Tb{B=$BWCMous4Tr_pY>m>-as}4(4`t(c)?)m^k=hBsppXu5a4&yx@r5wZii+ z^GLX&;R=M3--{~rQWYElXWOWc4u^KutBP|B5D4dKGE24;`fBItGCa|`#KEirg4AbhOHxUnBLPC^4nxNv9r1AxYB~|MWOj`{pu&7! zeqq>Yj}@1KYAK%k@6BWZZZ>JNl2kN$fDyyyFYi}oLN(yfU< zJdlRh48IGD_aL8*swCEd6a{cuE}y$-M}RvFH^RiVT*LycfbJWM8B~o= zd@e8?l_#08FD!(pOGkhydcs#ba7li1zn!dCj-={}O_)8)4PfDtJ z&U?q@21|WP^^b9hlkU!FqnBu^KCzhs`uppbTrQ|2+8iVyaZ!2kEJ+&MH_)3{h&fx~ z&(!q8)!O(8de5|d)q9%w6lD zQKxA|g7QY}#n=d^fQYk%C(8YN2;&;vD4xH+F*~5GLG?pU?1!E3FJu}|vIyCJ;0kh^ ziKO9NTD1>7=X7pe%wse(yv_K}OER|cH-uh0ZIK$zo3nu3bEhH+k5kpyqLRY6{IJOFe0*BiN=^5ZK+^nXM^KVET5;wJ$5dJ| zZLF*@loQlzYwBGajh(6InS8RnQW&!=1h~4P!q-%gd~1}^Ig`)!oxkr!ZtN{J^jb9Z1%hQtGwwU|wTMdSi!18(ynS*AWPJ5G!POR)Fd zqKZ@iTKihXJSwZTIla(G?gMCtRDV;d8^vFyaMze(Q7)Xd3i(nc>Tqzp!9i}NWev@B zDPojw(as_Oue;J{jYz%7;7n@ju|%UD7y8A-YQM5ig?|gP8qZgQ-}n6HCWAk=do-HW zY9yn#hNkH0jSCMFDs!?XU2fUE!2CtiPw)3KR*u_+#td2K-0U1oWe8W}WAxD{s?r4Q zM?91gAq(csTYuTQ&goOjYi|WLwuvt8u{(Gr)rQ54<0SoVO+?IWeg`b(3(M^^*tRwW z6s_s^tl@=+*HU4y3a(9z2HhiZqw&igZ4#>;VuMlQVNx*sNm#R(8I|vXs+)`N%Y43z zhSZ0T#A>+RX4Uhib@rzAWA=Bdh!A`lqUh}wNm~IIXg_;f$G8*WbN~$?07jhTs_gO% ztXcZH_8$}LS3W#JEcX90^j--+anz@YX#=(q$rUOEn3DvbN`V*CuL^>_KbzggawFCU zACu?Lf)G$MhcUg6Gwa6~@#bl-{S57Z-cS#=OL)oXWa17j0Cs%iB z4ZvG-X;9Ay4B_QZ7~8pIGt#pwPg6S~-CSqoS!Zd7-!f&g%ny(eL=GAkZqy*w{v<;4&9P2ddHJX%hGG1}& z+5Q7%4G7t7!xsj-5_p%jb2$@!8>o* zYCcved#b=*s``C0o7@$Z@aXa?s_(@HAe^7{rb!tTEIakbW^;=?ePp%c6)nu)60O$q z{z4CRk?-*GbMfMXRW*O9!JRTxfAW7-!mg@+?Rt6Mt6TY1Pr%Q!mq$q7HBh~51Mt_H zIu`OWM+Qbhwn_XaqV^ADr^ar9Q_Q&(EhOg-9sf+R6mo@VZQM7ZT)udfNGsr}pvb8P z1YLd?JzLlDoL;)vkevL(F%`psWgU9#XpY$1`*^st+6k40H{e?iYrT-N+GaVilE2=J zs%eh0zTG~{Vbdz9;~t?5Rk#=~pN>w4OLQ=G!!G{hb>t|)`lbgg`jw&fiW(N~q13Ci zLssfY&%5X>cB&atCEeKF(-`GW%o^rq15pmrFkoU_KgzdM%KIJ4JskN4j(xIp3 zgy<;LIqMDV&3D+JV*GU=#PQ+=+grtB2-F`1i1kWd>QsAo=-!Y9rCn!ZJme;5S7Ed8 z3BZ@S$W4$t{6?2|SdxE*1iaUn7Uqf><@#3ATa*lcCvq_We1@mbOJbH?Z}oNDxUTv_ z4~W&>11h31fc`QYu!K#jCE%IQf!>d52j*O4oj5l;D9u#=fsR+H4&V^FRj=x44b4^r zk(qw_hH{}8W|y(~KJ-3f^Jt!mo2ygVQ{>>%piTKOr@(FlNQNF;yI1vwc=i%GGb?iu z+;_L8#MaEq2Oy6yIuDPn^AbCTX_vsX?0iIYUU~VeHlv!3>o63&fVmgY)D1$cISVTl z0CW5FB5{6CF5prVl6qjN&Eq~CYCb@QMPCb|(gEFNRFx!k-IRe+3;30(eq>A;@o3Cz z1zq|lku;P-3O0%votA%v>F>Ybd8?!1`?Pk8W}B6i{+Ws&=xFC>w5!YqIr#w4`>JBISa#y_6a1dM|?ZAqZ&c@Q)aPqEancN z3@Y!3Fy|-rhfHbHs+eZ;yJ=wozs^NP&~(G)cu`u);6r0#i>#g43f~ZlArTTKo&m zzyL^c4W2uvWXmDZgXH~%J7A+2j0)+R@jR6(09gL}6>Z^t-M`u7aHbxB51>}Vy^9KV?Y7#BA@&C^E@O-% zLvC-tNEbyd^-$?NVbn^Ld%BC#{FL;)%pG@bG7>;Sqm{7wt2i5Bw@-3hi}T06W(Wiw z^w_bd0^^z^+9#=viRxWse&w)D4{ddHqIt zO)iXi1Iv~1jQVmqBJBR0j>lSY#8hi2;6RYbLWXT zP%(~oVT+wYMPRocAt*l&4QYd0fovM(N)1K#V34RfGVm#8r0>3F@0t#hx{uu@i(l+S zIvj*uZuLNK`AOoP34hZ!_q0=->XiO}0rsScCxptd-V{m{i`_-#+fSqjwnpa&TGjTe+du7dAR zklwL41*a{oiEkiL=aJ$HwyjIXUIOWbC_0pSKz6pBPiz=e>& zqByl?>zR55>bvcgivl3Ev-<=3LI%)t|IWf{grlv6cY@XC{&%gBd%Gs`>n4? zAUPVb2MzvoPuTbV)ErVcnVo@~bhytzJH)<|i_l*GWT!bD6-1+iIfbesvfQIsTKcK6 zm}q;O@3`9OV*Hk>4XEe(#h1DAFlUR)!eU=GcNctls^p8j#0&tnpF$yzw}2J|%hg8E^sSgJWFg%SZdbb|?22xkArfE0Dkj?vR=1m=~MF4q&@ zOYsP(r?wk8l;hx0GlpLQtJuY23jFCm(Z3%%MTLM~I*(S(=+FK_x*i zjM%$+!B4r+5n2^%H(~t!5MUhFa^mpRYg9?{-q8Hknbk~yNnrkuBV*E#dxKt!Ks6HP za!@zCLZitDl{rL{NGmnqNl1}u&$Q~do(L5)=q5}+PFTlUg=}!Rj>;=9_~{1;M$ckw zOX^eogm8H(VEtWV>tK;WIp}u!QOb@&Vc$S@HMO@4=>{HkdP<3;45 zSkoPVhYuB|?o2q8CGv4~b&&WhKP;(naxqs8%r$w}PIe}NAop1Cv=WYTF$<=cf z3NYWt?WrPw4}fTOEKV2lKZ2Eiq6$iGjr;uAg3<9xC??3P6_Ei=TDgQove=iQT*;<# z9}9ntUFGy*c`WGZw9<*b;;HpG!iw$dPgih}X&G+CzBBp31djsUDBw};+=MVY3nr0+ z!K~pjbPh^k(V_&Pf*-W8R;>~YLu0^CQ3o-<9I-)4VN^p+JHP9OPT;Ru3ngDy862#j=HI`7#mmq95Yw9(C^1aM&e)UJ?CeR###gF3&GifA*Ymn&H`y#&?2}Ogy=4oT}|AyR5o@6 z)bZ2~*{PCPgA8VYKP7n?~9CaX{2-q+8m!}gL)*}>;pzpAu~k&Ys)-AA4i zP1gA`fvy6^3fwNA9WY!RfWZF?&vYMBtGU3!&PZhldGR{4f(A(MS}0tZJa;)-f@#mD(snS*^Lo#}DXeX6_PCKKf`vPh9-sR?LlVanKu_sy zvFkz;W3;6b3?OwJ3ZU9l;IrcYQ^uZvBTbTGfIU^AWV=ER(C0HZ!pVUZw2#0F5OK(< zD(=UN=GKOT?u#;(8Qd$-?rX4urKJ|%F8iw|6_9X1C*#by=$`)@PK?UTYJhZ%a(ScA zb)>0zHtpbprYvrGZlD$H(5Q}dU05*yfsPvX_~Ml#afbrpf=v~%l@iU3Bhk#Xz}%7= z5m&wQ2Nz)G(`1aK-PX-dQJivAR*XxPJ^Nzf>Q4?fuoL2Y*P+Xku#Pd(s$~RC*$?3) z81iZ*-w;%;(?vWG;ea4Cg_+Md9FG3F#wC>Bo*0Tio1Nu#O1yrC2Ve0aluOT0Fs?of z_YI%;{5N%|t1s&yp(Q5?_}**XPln(ZmNA04ishV*biUBqRfZ;a_+9*?O@%^D&F`IEeW9?0 ziv_N*6jw6mL*8)=Al~Vt+L{d4|6@Y|c$eEYaR2B#-$JAU4C|9aC;7w|O>z>bWz`h% zmfBCHTdh??_)@HAu@>d)yvaIP77|E}zo;5u^mRQYzPt_B@>|-LZchZJ$0s}`%#_Ew z#u9T@k^Mq$F(>E0r>2=s?C=b?M0XvA!qGYV2sRilmm~$!ggRV`8{)Yl$-kq1DHEmN zk|^2>#8C#A|n_S`z6O;=q$mV?J`ukcVTlc&7oDH3G$J{bJ4Mrc?hbrdm$=NH1Fc&A1=@>aBB-;;?9`5oXK!&E7=?8FK z?4#lmn&`=m=({M^uq#_i?e9n365Le92%T`$kv=hlM7q!Pn%C16*Y5PN1{B|7 z1uhxsMDUJ*<7v41(F?cymhy!i+&v3eQcCvd(*#$+>2!}nOKATK%yHT0Xb;kXjkQ

Pa>BrM`Ck<4mCkN=Y`#_E!Cw7L5Ojm z%f09hga_kTG

tqh-JF4rtl^#Vf-l_B@W!l!o}pHSb(%QvZ8f*rEyo1lTq}wmVL6 z>-Q}-FfcVRIWaXf?e1WdfXS|uqbJ@7ej@gzPEB9^1t;VP_1T|!dc6w+RLRxb|i95|KBHO(HoV3Q6;%)6_M9?cH zuP^>hx7hIbmJ6_d;;!{kR# z>y#V0?Cf>yrq}*(`wn zQAks76(1dL1kjJ5``P#hy3Fer0V@HL!GI0y_n9 z|0ObLr!FuH!khzPQMNtP|H8wZ?b!8Uv>1ro^ocr-r{l~mj%Ys4ua*1vTX2{> zhQbRv)e$O=Wtm^pD?G)rjgam6-#ak%hT-B)(XoN$rLx|*jaODvxsyf<+LVzjDX4_g z%Zf zuK?l{526xfj?<-aRuT*U`)|ZB?$bXlK@@bb)TkGlelepY2M6ih#gTA`Ja5qF%sJ~M_;AEXh>@_u#GHPxJGP}DbL z!GwMz8w=n!0AgvQT=~BeO^~;b5e{QeeHUH2`7@Ldq zm66YkUoFr_av~9}YbZU8bd9+5fy%GNn__=+K-N`6R(>zY;B%NHZfjAGCeFPA0I8dev zL`kqORBoMol?Ky}u1-1nAITO3Tw_%&TBYC%Tho;?`H_80;L#g!z=We}T(OFt&KVNd zSccs!p*BQT-KPCf5daG=@LK;y8TaKNntLFU*-sJFN72;X=s^)cO7BZl&?+Y%0bI1E zu?;WyYT{4!ge$$fl#hEYhJf;rFQ#%P-ZISq zwn%Zq$>FTF;kJup&G=>V;-JjY^<&B%J(@^CZz*hbG7;FJBC;!Job|Wx3!GUlNLjLn zhAzta3qyy4Kzo!(T`|jT)mp{W88*VZU?M(*{TZxbF|(m=#3Qw&ouw`-q5f|p0p+XJn}6`GKmLUqqHd}hnHZ1bS20Dd1r72OPkfBQ>bB! zVR=iaBAzNfn^HlPA+D60oEUM$$yz|-RkQAoonIhpL5hBy`^*3K4_}N~s2HIIuaGuB z1g6ijSe5Er7dM)aNge7_f~soF0>{E}PeCez&Tz*jG<1fC8o4n|L#sWnn>4ITaSQ5E zq3B_m=Q?I-5fGFIQXeAu8<17X0_WzX!bCX~Ofe)P9fOmVw(x`gs|!GGRl z^pz~;9Apg7<|jmzd|t&SRd=PJb{7{u+{1){AF~{<6;xN3LPTk>Ic1pi)RILWb5MpR zE=$^&rPK_fM(4vpmH)P(?Ldnz$Ys4pns*GzJ>&IX_Yy>HlMPK;a(&+8#j|zuJbm1t zd&zz5GmDIcV==WY5t@mqR7>l>iYuoFuH-65B|e=s1Clu{vr+Xo8JS{rnI zlJ~rrI{4VWlb==PN)geR*+nzboro!8-0gFeC*Cd!fz5!VVpCQqc#KqWEnvWMHw?MS zXM5l@!<;%7it}7citPjs$`98VYptzfoLt%TxBI3gZ@Dm~4YZ@7xbqYAZ6p_w(^Ec; z#U9|5n|1aQMGXn*gsPml@WNB(nzS`K{m7m(j>7bDiFou+B!6#2h3HAKlUQEpf*+=0 z$k##g`oX5Bi{)v^OsRiMr@|H%s6K~qAtlLq(!NjYqMk13z@GU?$lr z#0K9gN^hpnEnGH^`!t<0!!`rNg3IU#KB3$Sq80;`g}JKMO56F~@oTzqYGz#W&UfGo z;5B=uTSgZ-fp51Ie?ggd?fW+w`%(m2IQ|F{KyYGwfwu>bdpdxT-?qaF~&1)IJ;!wHQC^OM*lVA`+8gYR@7hqgENFrE_e^jU2;`DYbb_) zh&Ku&q8s~KOnM`3>gi)*X9jzd&?!0NQ*w>34LB*kCW+mBa{uARHJJCCy&NK)Y7FmG`tV{YPiUeMo)%k@J*MsK*%lRn6F|WiViu;$$ zI}e>ySgZv6X0?>6g$s zqQeav@DROMhci%j^HJ;%DF2Qci0KO~!3@w9y4&xL)Y!swLJ)$5$!psWfoP|79d_}+%3=?>Trg0}tvvah|tqj|2DGA$y(UR7^4lV3&AXRV;F%ie(N2xHLl0f~R zZL6dUZis>%Le$^|J#jG%Wpq8oaUr>P-}NF+TIhFXPH2j$RZIQGL&3i{%9os!x~o1e zOQJyd8yAt%OzAFgOW7ZwKQCJUYf2YM*{r5bnL0QS793pUY zp98PCL_T^?%fz`|DT+v5$R38d4l4((oBx7VuN`+%nmp|Eyjbx<#c@)f}&c2K{ zZyivM+*(8xFA27{qcfqx@DdZ=WhXI82yTPOOZ1bCMRD#dICw<+7=LQ(d##K^(}sgZ z9_E#r*X{G;H+gwzUFv^5=Y>}U8ovU*)-}SxqN7$s%649$Q%B4{4%+>r3d40VPVid( z>w8*(Eg|cewS>3~+wIr1o(D#%sos&Ax7!f3dUIzwSq3C(WQDGp#YAVN$^5d& zc2|Xj75FAP)ZWZ};x>YbjJt1%X}v`lqAd{=dH2}zfOx_Lw+eo_t!KA%TkkYU`?RVC zr*pJYWU9T9k{(u4;E+F`N`0x8GK=j|=V7D6bGn)rJ)manmWHccNu0#T1kA@4BI}yXRe2dS4A+}SkgQO4hthB zydAU;8&bCB>}`Lmib*Xd5FU0qon~LoR8kPoY1=d34}ViSCK_tm=etL~RW3{JR_j-v zlLe9Yn}lb!QLop)E_BcnR-3G6>}p{MT@6qAEP(zdzP+Z~>n*qwo*=4rdJ)MP(}n32 zVhakfYF{^$qz#iL*|wg6e2_gZt(us%#X~6C`?HL9l)|uZ zA{Dr!Ev1M|@jn-8S`>nCFUL?st$8%UV|noe=pwmNIvSd#RLSH5N^|@2wCjgk+q0fX z%T;vw_tQm1zRG&czEk@hW<@i2To8X1;F7)fg!?>~By%fAkdYsk@X&p4|v z<~*UANOw$E2I-zXj*ve(fm+lEzC8x7c{GxxTp?QG$1-lRcPpHJ*P$tb_WK{pnQKR8 zP>kTnD!H=b)e&rqP3(X_&pbLMHCLZTsh!vHFVXkLG`kdBhKgu}R}nostsb2y*1<9e zmh$FVb9dC@xvQV(X+ca(R_@4bX#oYLkOeVfhp%XP_@t2lB+0@H~1Bb z^?TbeaE__uMCqF&wffBO+LMa7H^l!#A3S%A-0KkIK0&dpcn4@WlUnr>)x#&Pj|C1z z%FSStqE!WVqjXM8ytzn;$nAQzz9B#&xP7Q}IO;v`YFtlPT=O+3iFsvwl59xBU@{V? z5qoOHt`X*LgNqv<%8h1$e+=gc5NfSa9Ui}ki4uDz+c6K7H;H={lvb(-iJijF3ie26G8Y?(9R8Zb*fv)5#mn-e-eC3$tUo_tP zMaiIGpUr&_QBf%vr>04bqmHJ3+>3zc|JjQcd`yB3#(v+xDr-q4g7|i0nL6v!)JZNu zN$hDHCc`1@!w`#Qw&nb{#Bs(moEs@g-=is-H+)n!(fh z-mDs%Hlo!7>Upw=6AXB=PvxRPGv3VVX(Itl=8R#E82S|jcdD{DoK9-tP0Y`dSjlJA zhs77yssGV8^A$_z)FPa1fu;5!Tb*;t?wK-alLk$Qe`d&y)$tgewJk*(Lg1$nSGl^r z$CN*iXTzg?y7^+ap$U%km4X|gZjORQztyHSMb4DaILwpV( z?7}l`mojk-Is|VT{J2UR#d}z?!UD^w))zL6qO>{PgVJ|m8}vmxhq&SV(+I#a@GKAb zIg<0z0VW^X65P`Y{;Pr*ITO6k^e+e7xM#i_ByGb1N!>nVJC-wER=Kv73GA!nCg z)P0^#tbqsLgB9Se_*Z+S6PU}D?_7}u;#()Z7p0S&KkOV^R=z3a-YDR;Jyp%|)3dP5 zikCwga0fZ^(^~yGGsC~WVKyY$bWjkWo!Ds>%h76^X)#CGp_g&z zsX6t$O0X9lh~`KC#&2sTE$b5d#*c!OoYdEZ&^n##J_g1_<~6#g_B}v;H9aJZle5U( z`QaaPR#{pg3t&pmwW!%L1axG-?m81Q8T#G! zf3x$!GdqhItaWOn8YC|GFSU;W#G&e}2gvUNz;{~N%06cKgF+;Y;%g!tgeT+yRL~>^ z&xoQ}T7Di;AF*L5P4-WpfL*E&ByBpt9LDxaqGfEiWGsj_GdqKX@0npXJavwk%5S5< zwdge9#;QnCn2nA)m1fWENS!u_!3QK5zT3nPZEPI09piOS3Q(v&80`8@>?BnGpZWm% zPN1W?6Lsb^a>U3IHNnaKINxp>9CRGZaxk3x=1>YbVZhGAo*@w_SFVBj zy~dnE*Q%>8^oJ1LY%$iMK&nZ4cnwo8tH2Qqw65e%w@u@gxnb*x|ARvk-|{NL($@GX zY$l;6OupfpXXxbXpgL;euunTIO{~6oaPH_;N5=Gl6~|j9VRPpg!Ik!ZGqh02YgCLtWVqgz)^Kaa{prJ80)pfH`kj2@`R4R0|IcCR18)`fKI>L%$+Uo>hl)=^;U zSslT0Aka`w{WcEnSf8gjDR9c|iq1XE0(mT4)fdu_`<5Qsil%!H1j_e_8W;13&{=4U z9H!rCmOI+AHsP9fiv^bW8e`&2)9Ia)T=b4|3D6ZL4eEZW$Hg^iImbMZK-0Z7205h{ zogplrOl@R!n@@qs^GHF${rhW?LxVYCYp)*Zso~h`ifb>$@BYR2+g_{;CUUC^jaOPv z?It^P6VJqA->sR6N@icXlqBs=TgwKJiR;W*^IM)36M<9Mai+Z_SuhJM%7f0>Z5ux` zDNz6)Dsc%qH=vdHz?5f1IN{(d-ZpqDbe)tVd(gq`L;H;@jo-vN82a+ooy58P$HAk6 zK$(-=IXj{eDN&$0;2roXFVack9@+B1jklwhb4QfyK)TvVsI3O+UuoyJQ~DX$2wAKA ztT|C9ReBx<*+-RiFMcYPa}9&-L80oN_6dxa(nkUhGdh$j%wJu1V7O;Buu}qLpcVMi zqv8S#i|nE#x2f}=BSg$bbWhPn^VGG_K&3aC*j>Qyb za%1?gUa)&GdcM_%2|>+2oP- W%DL(vR^B^@dJtVnqMFaMfE%W*cEP3q literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/corpus/a67e63bc0c0004bd009944a6061297cb7d4ac238-1 b/tests/fuzzers/rangeproof/corpus/a67e63bc0c0004bd009944a6061297cb7d4ac238-1 new file mode 100644 index 0000000000000000000000000000000000000000..805ad8df77be7ed7802ef8666f0cb5e180b2caaf GIT binary patch literal 14980 zcmV-~I(x-^I}WZ3f(26`!PcW;44V+4dFJ3 zqW*hpUjjP~hlXfc+zo+v^QgS@wGqHQDQ-Y57 z^2)G(Pm8QI$Uw$1>w){{4PY}SNj#{&OxwD4Krw~ezWCQ^35|BTx7)ZZtwo{kZ8PWb z3?9>8M&zy*7Hin@RQB)HKbcH_Cy*b~(e%}kMH%0&G`(bo^1wP*F9 zl>h~CbjzDh8QLcT%)4Q0%M>_LZ7!6jg;6Q9)tXk_+lDSss6Ux5*M+X+DClls{CQ%4 z*i}DSr11=0(V0_~xp;4zalg_6A%GSdr$REyu?ejqUiU@m6=Oxi*h{VPSWCh-*l>Jy zW(mwnh4ru+D!$W;W{n4mj&cQwm&C32C~jfi(W2xtn-00B5Rb0X1r}B+V%Mg>3G<{! zY;rq@!<7M>f@)Lenc~em?!EBi=X`mQ(cGuOAZYeNGYj+qgoI4P#B~9I*brIKB{CyJ z*F#zNA|hw$HyN-oh;*P~#Z?=_l>+#NP^N4USxF(D$wd4VwqQH##r$>GcuP9GX$_c= zH4&-oOHD%7eCjdb25Ln{p#5%?+n1+1NTwbGjYU0+IS@j#DJCtxQDK->uB;o8mi~Rv zG96-CF#0&nS*uQ2cD5py0)Z<#DK$C?P_qA+K_ggc*8 zRzPtfV}(^(ACm3fOHIEORPkN#xKI1#D%#~nD;zFpp$Jxx6++@HMDNkDfPALj`JYcn zWyDcp>C?2%RB~9`jSVOI5;y-AHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV z1$K3m7=T<0^t*d3MDt$!W}wi{FHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*; z!K<`d6ZtwGc*ZAJMxJ6G$DfFL?AgIE*GQ<#%Ulru|gwfOn}OXVzyoj2iNr z?+pv_-*76=4Xd5$f8NX5Ixu-F5tua4DIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_O zXA-LhPB1uQ)IdBC-x;ax$@r%zPR>+S`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ip zxex64`k1S2#aWJ!Zd2qrHqN`ZxC_%Qx=y_KiFDAQZ<+iPs{e_8t;AE@;!>o^X5&Sw zO*9<7@JcdZ;CKCi0&)iuTeS)v_+sF$w%Hm%oY<^d*iGbiVn`?as5#NJqtRdE+#_J^rwW_uS-XrCs74@PTEkN@>?}uS6^unR83->0iFgyuKk`_vb*0xDL$UT@Q zdt_^~K!6!K&uv$98H@mgWW{Ng=n5!WIz!K7w%l9}MxiD`=cw9b6w-ILNkwxRM}w=M zK9K4+TLtXLTZbal*QxY3U_+wY<=R5|FHTHlnjgJTB53U|!_aInFeD&8vV?ekI?4FH z0~Ea-w3=Z(N|Cx5fD=hSP*6K>h1O}JvR)Wb7Nqb_LhusjNB{*2v#D#Cr4HtHcG2Q$ zC73w)VI(w|^St1Q-nGK>FY`#aqTvdJlHZFe^imZZ0cYE&j}C`+)~kwh3=jzC zo^M975;N|U2{C4jW*h@~z`kybtGkuogvl@B$Eco`$Z`$?r*Ee%bOvUR*COb3;ByQ2 zoh>nVEg*co9$R@sQhOimGcamWVd1}%fYab_n+^;}kkpy)eVyzCR}YElc>38k;H9GZ6UUmH_e)YqpCbW6 z=MF>1ARX~>5NbLRwPbdUKA^&UUVdTNX^$0`f@&$AM*Hv7IYKriDS3c`nV02^m_Q?G zc*yDxgf5T%djq`Z`+ zH`zzV#)JJg!o;;)!~(5=?i-64REC3p&G^qtGPdzIgkC#s zks8jMvw+=mry>cDQ{4}*h+3sjYaD5c!dP=dp3-7afD2Z7s$Zv1@c8w__{Y`KISKv2 zymTrkmXoyTntuXOV&2q~Ww=${1w=0&T#`g4+G$EJLWAxLH-^&RTR?N24}0gYr@33q zIyV8On2TsOw6bD7kt^T$x^ZPlaYmQ?7`c?Do>fOd6eb-XTZ0!u4@89=0iu$^xcsol z?RS1@F_aV3YisIV8;-QUf+N64D3yR~tn+Be zl9ApbAbn7(I*CfEmF=O@d}Z4Lp~otzy=*ad1A_{lsSrEcNimXH6^mz{b2`2<1dq2l zvtc>Un9iAOr#t3Mlaf$&Qp9FwRje`nIC_Ja&v`^95C@QOeP*GQ`H(VlxYQX)fITYQ zu=Hc1%+VO81xg$HW`8lx3iFemtqG)TUk!tGVt_np99G6bC~ATak5^}%jgoi3z~t|Z zakSs^G6C&YWo0xj;jm{qm8TzbA|9|^6I@WZItJc7hr=*i6=61cVJ6~ z!~>SKm`#C2yV7WlNWIA5Ols<}M57)T z`o+X*zp_t-e+#o3&sT%r_x$E2gFm)=G@8|FB%`;6rs(O73l9=1bFwB~ZrQ!S{6*7G z@AooRj@yOC3|Z&g>>NyG2v_4{^wB4((gf{CJd_e43+ByRf7!au=~K&VZv{2Bi7xK3 zJ9s74hQ*BIB>iqpM9ge{2Q20b%k4DSwl)P6t?Boy;f06SQem(Pu1$;v-6L_M@yi}< z6003zgHhsPQZW2UShJWJmG6S8n~U$ue7=i@)Q6A6YPj8I)$^ux_NMk@_IIj?5PTY< z=&F=J=4r8U8aY-F z?)lRrL%HP|K$X((s{3)<+<-{S57Z-cS#=OL)oXWa17j0Cs%iB4ZvG-X;9Ay4B_QZ7~8pIGt#pwPg6S~ z-CSqoS!Zd7-!f&g%ny(eL=GAkZqy*w{v<;4&9P2ddHJX%hGG1}&+5Q7%4G7t7!xsj-5_p%jb2$@!8>o*YCcved#b=*s``C0o7@$Z@aXa?s_(@H zAe^7{rb!tTEIakbW^;=?ePp%c6)nu)60O$q{z4CRk?-*GbMfMXRW*O9!JRTxfAW7- z!mg@+?Rt6Mt6TY1Pr%Q!mq$q7HBh~51Mt_HIu`OWM+Qbhwn_XaqV^ADr^ar9Q_Q&( zEhOg-9sf+R6mo@VZQM7ZT)udfNGsr}pvb8P1YLd?JzLlDoL;)vkevL(F%`psWgU9# zXpY$1`*^st+6k40H{e?iYrT-N+GaVilE2=Js%eh0zTG~{Vbdz9;~t?5Rk#=~pN>w4 zOLQ=G!!G{hb>t|)`lbgg`jw&fiW(N~q13CiLssfY&%5X>cB&atCEeKF(-`GW%o^rq15pmrFkoU_KgzdM%KIJ4JskN4j(xIp3gy<;LIqMDV&3D+JV*GU=#PQ+=+grtB z2-F`1i1kWd>QsAo=-!Y9rCn!ZJme;5S7Ed83BZ@S$W4$t{6?2|SdxE*1iaUn7Uqf> z<@#3ATa*lcCvq_We1@mbOJbH?Z}oNDxUTv_4~W&>11h31fc`QYu!K#jCE%IQf!>d5 z2j*O4oj5l;D9u#=fsR+H4&V^FRj=x44b4^rk(qw_hH{}8W|y(~KJ-3f^Jt!mo2ygV zQ{>>%piTKOr@(FlNQNF;yI1vwc=i%GGb?iu+;_L8#MaEq2Oy6yIuDPn^AbCTX_vsX z?0iIYUU~VeHlv!3>o63&fVmgY)D1$cISVTl0CW5FB5{6CF5prVl6qjN&Eq~CYCb@Q zMPCb|(gEFNRFx!k-IRe+3;30(eq>A;@o3Cz1zq|lku;P-3O0%votA%v>F>Ybd8 z?gzY?2TyqLnj&`MAZ3!c11g#kypaiPe|R6GBtih!cT69_l(UF9P+D^MuDxYOLit%E z2quhUyL|D4|7V`Op92F}HdYnS*G&?L^On1*9gzn@yd&k3h># zYDzR@q3UH$^7aWNtw(%2wWAtA_)})Fb1dc#pbRSShcM?S^@mJp)2f(e^Sfzb0ppjK zdvgt;?o=xu)Qr=gdVjqKHoa|i+o&Fw)nq^43=KIl>Avlx7A|7-Wy>Y`IwY?6FfnQ zco75c7OA(PAXO;f&??NZ`@o9OIBuW4rds?9&Ao+( zn(;i9Dgapi`xSBner0hR(``!SkdYrS#sG!VM|h}t^V4#Prc^vOQHxT)<9C|M#gA{O z>5ARI+2nAh9)J&^R>QrE3U=+b+KVCf2QMyTj3YyCZ@@?wMJ@GE={#Z7N|bxLi_-j* z^u5d-cWyEgKtiLHu==Yw8)3Ika$Jk^$G&C=1ReC)v8Mv#nj_jLsf~&1U1WacuuTtb zb*xMKT{fIw&07x!HtC?cl4X4|*#z&(y32!BdR|iE{}5?_AcJ*IJ0ve_<2w9`%ei=M z6}p6s-DUnpV6NT`j2q5+WR?&xHHzO{9lYuP?_EtUjCljgmGO-FaylaH{+y1-T5-fw zYbfAAox+dD_0|O4biyoAwhxS6P>dO5t@d;0i8xR(j(1^;okB%mw;mxVKMxIQgIj@Y z8s$n2MfYHks5&z6DQ2YazGm;54wAZ$-6e}(>_j>ogk5g+KyUd;;++Y9(>LTmNMoiR z>H=FkPWqk_Eo;05&ky&sQ=ICUFCZY=F}wMm6gprVK12cbq=_el%CO!PN)(ITMdjO1 zqzJY~=LlNW_N&`J?Uf~|UF_ZlVt8352>QI-7;SSD_C9z_XiN_j)e`PftrD&?ZP@V4;=g-jze2-r|vz9d%hMFaQ=k-JW;^)y^9Jb9)k z=!mc+9{sit!y_*DX#M-?o# zVdQfrx}-fM(dK9c@736^aFgkhaRJ ziL_ymzLD)ccZaHsudOIc+-AT5;RaL`ibP4kg^<9aIJIW$nR*54yX}>W0yrS07klF; zsa7dY;4#GIU!7C#PYu014+t_+wyMu5mowj63;htd*Gdt?kCS%H5A(+UyYi`$j>3mM zRnDf*%2*9&46=!4$6xBfCJ%cWqC;`e7sOS2{ zm$~vVXN$|iVqZ3Q7kqiD36m`yy(bH=L=9QH$*Aw1L@d&7=wi`K=ecPr1+$S`}+IVf_6N zU>w(S;_%dKR7vvQ(EQe!)l7g%VE&IIW73d&gIP&d3nqsa)BIYg64D>dLr zNRevKwCcB>2o*EvCVxyoPFTlUg=}!Rj>;=9_~{1;M$ckwOX^eogm8H(VEtWV>tK;W zIp}u!QOb@&Vc$S@HMXcS6Jg6*nD=f$dre4WN!sO4dLSKovPQMSsF32_TOY_?$qAXT0&7-D1pX^P zfy@<1Wws>#Zx!iO9`~MhWnIC!tBN~_)jEX3>lv_1DPBqJfkrI!9@bw_qUb}Ap>%uC zAtsHW{*L!-%j!f#*6dT*4p4P-b>zHpl~3alQ?my}V5Sm{X;{-8fQJtirtVBQlqK?U zb#={oJ!3z!`kFRYw#yyuXPu->YB&caBA4tOM2MFu=Iq+_m|b^-Y+lS%fITF(}V8 zzug^4T+hfvH&AmMMY(^|-ls*hcR7JqqMdK_H|K5w49V4V778%m$L*;ifDeFZbu3O7 z@;`!=f1(OXZjJl=*n-jVN+>4CtQC;~O9o>` zzT&C%IKqnU>rYp3k!cxj#lAE7!32*2-6-Hu?c9VgI}0X}gTbufGIS0~VbP)lpn@N? zu~w}T3`1kUPf-UkzZ|haN?}w(O*_RnEhICnFstc|eOJoXqr`q_VLXP8{4Z#GYesVx zlJtC3Yiv4;PApybd$)M{P!bg2_7F!V$h`dHXi79N^%F4cd{JdYiXzza)ZuBx1h{6V z23!@I95QX<#?^KLX4FfX05z^&m}k2BN5s1$j=HI`7#mmq95Yw9(C^1a zM&e)UJ?CeR###gF3&GifA*Ymn&H`y#&?2}Ogy=4oT}|AyR5o@6)bZ2~*{PCPgA8VY zKP7n?~9CaX{2-q+8m!}gL)*}>;pzpAu~k&Ys)-AA4iw@udhGJ&oF#tPgn zpB*q<9Du<83(s^PQmxWUxVs3oH*=wR-U4u;suiZrLn6a3?g1ETUJ&z~f}&V+N+?uR z;*BABxa{y`4kc?1zw=Lmqc(GhNqv$0gaI}5{l=1I{Kv{SEJBvyl2PEv25k$W+M9=m z30V9v6Ccq?6nv~|pyk#vF|d`|C_`DTxyHv2=xAo{B2Rcj`NwX92MQp8v^uMNp>R1s z1Ky%*Ono?`!slgt4jYYm=$2~jxFYYi-+d-ii%UjM6>++slbbrxEoK|bg&=5-3~h$7 z&r;7bP>Xg>KQjvc5De#p5H1nDisf_jez3g}N`Q|w;;=b-aCLtR#CT`sY|^+1N9&C04Ux(SZiF<*hfZ`@C&9H}m|u8GCkrKyE(*C^ZsgO&R>DFF3g0I3U~-!9$PS5L+9&;0u+At>PCKKf`vPh9-sR?LlVanKu_syvFkz;W3;6b z3?OwJ3ZU9l;IrcYQ^uZvBTbTGfIU^AWV=ER(C0HZ!pVUZw2#0F5OK(0zHtpbp zrYvrGZlD$H(5Q}dU05*yfsPvX_~Ml#afbrpf=v~%l@iU3Bhk#Xz}%7=5m&wQ2Nz)G z(`1aK-PX-dQJivAR*XxPJ^Nzf>Q4?fuoL2Y*P+Xku#Pd(s$~RC*$?3)81iZ*-w;%; z(?vWG;ea4CLF@m{@<(!Rlz6j9TRfZ;a_+9*?O@%}DO9l6-l%74bp$jLq+zU45ajg^LBQ zuoPD^=R@9c3?Sa=quQDb*Z&`5Ljib~+ct3j=sVv+qyh}6!DhY zPo-O}RYUkvtY@(nnNIBR47Wsg9frcuIr|7U7%rD21=55%T#6gwxw9h4zoULB6Q$ph zDB1~T2gnGmXNMc?AfyGvYj>ww66qJ>z>`j6K61{GhhwvNrva8V0>x|nV$J8LmhB5g z?;AZIP%AElrM$>q9~c?hbrdm$=NH1Fc&A1=@>aBB-;;?9`5oXK!&E7=?8FK z?4#lmn&`=m=({M^uq#_i?e9n365Le92%T`$kv=hlM7q!Pn%C16*Y5PN1{B|7 z1uhxsMDUJ*<7v41(F?cymhy!i+&v3eQcCvd(*#$+>2!}nOKATK%yHT0Xb;kXjkQ

Pa>BrM`Ck<4mCkN=Y`#_E!Cw7L5Ojm z%f09hga_kTG

tqh-JF4rtl^#Vf-l_B@W!l!o}pHSb(%QvZ8f*rEyo1lTq}wmVL6 z>-RC6?e1WdfXS|uqbJ@7e zj@gzPEB9^1t;VP_1T|!dc6w+RLRxb|i95|KBHO(HoV3Q6;%)6_M9?cHuP^>hx7hIbmJ6_d;;!{kR#>y#V0?Cf>y zrq}*(`wnQAks76(1dL1kjJ5``P#hy3Fer0V@HL!GI0y_n9|0ObLr!FuH z!khzPQM zNtP|H8wZ?b!8Uv>1ro^ocr-r{l~mj%Ys4ua*1vTX2{>hQbRv)e$O= zWtm^pD?G)rjgam6-#ak%hT-B)(XoN$rLx|*jaODvxsyf<+LVzjDX4_g%ZfuK?l{526xf zj?<-aRuT*U`)|ZB?$bXlK@@bb)TkGlelepY2M6ih#gTA`Ja5qF%sJ~M_;AEXh>@_u#GHPxJGP}DbL!GwMz8w=n! z0AgvQT=~BeO^~;b5e{QeeHUH2`7@Ldqm66YkUoFr_ zav~9}YbZU8bd9+5fy%GNn__=+K-N`6R(>zY;B%NHZfjAGCeFPA0I8devL`kqORBoMo zl?Ky}u1-1nAITO3Tw_%&TBYC%Tho;?`H_80;L#g!z=We}T(OFt&KVNdSccs!p*BQT z-KPCf5daG=@LK;y8TaKNntLFU*-sJFN72;X=s^)cO7BZl&?+Y%0bI1Eu?;WyYT{4! zge$$fl#hEYhJf;rFQ#%P-ZISqwn%Zq$>FTF z;kJup&G=>V;-JjY^<&B%J(@^CZz*hbG7;FJBC;!Job|Wx3!GUlNLjLnhAzta3qyy4 zKzo!(T`|jT)mp{W88*VZU?M(*{TZxbF|(m=#3Qw&ouw`-q5f|p0p+XJ zn}6`GKmLUqqHd}hnHZ1bS20Dd1r72OPkfBQ>bB!VR=iaBAzNf zn^HlPA+D60oEUM$$yz|-RkQAoonIhpL5hBy`^*3K4_}N~s2HIIuaGuB1g6ijSe5Er z7dM)aNge7_f~soF0>{E}PeCez&Tz*jG<1fC8o4n|L#sWnn>4ITaSQ5Eq3B_m=Q?I- z5fGFI zQXeAu8<17X0_WzX!bCX~Ofe)P9fOmVw(x`gs|!GGRl^pz~;9Apg7 z<|jmzd|t&SRd=PJb{7{u+{1){AF~{<6;xN3LPTk>Ic1pi)RILWb5MpRE=$^&rPK_f zM(4vpmH)P(?Ldnz$Ys4pns*GzJ>&IX_Yy>HlMPK;a(&+8#j|zuJbm1td&zz5GmDIc zV==WY5t@mqR7>l>iYuoFuH-65B|e=s1Clu{vr+Xo8JS{rnIlJ~rrI{4VW zlb==PN)geR*+nzboro!8-0gFeC*Cd!fz5!VVpCQqc#KqWEnvWMHw?MSXM5l@!<;%7 zit}7citPjs$`98VYptzfoLt%TxBI3gZ@Dm~4YZ@7xbqYAZ6p_w(^Ec;#U9|5n|1aQ zMGXn*gsPml@WNB(nzS`K{m7m(j>7bDiFou+B!6#2h3HAKlUQEpf*+=0$k##g`oX5B zi{)v^OsRiMr@|H%s6K~qAtlLq(!NjYqMk13z@GU?$lr#0K9gN^hpn zEnGH^`!t<0!!`rNg3IU#KB3$Sq80;`g}JKMO56F~@oTzqYGz#W&UfGo;5B=uTSgZ- zfp51Ie?ggd?fW+w`%(m2IQ|F{KyYGwfwu>bdpd zxT-?qaF~&1)IJ;!wHQC^OM*lVA`+8gYR@7hqgENFrE_e^jU2;`DYbb_)h&Ku&q8s~K zOnM`3>gi)*X9jzd&?!0NQ*w>34LB*kCW+mBa{uARHJJCCy&NK)Y7FmG`tV{YPiUeMo)%k@J*MsK*%lRn6F|WiViu;$$I}e>ySgZv6X0?>6g$sqQeav@DROM zhci%j^HJ;%DF2Qci0KO~!3@w9y4&xL)Y!swLJ)$5$!psWfoP|79d_}+% z3=?>Trg0}tvvah|tqj|2DGA$y(UR7^4lV3&AXRV;F%ie(N2xHLl0f~RZL6dUZis>% zLe$^|J#jG%Wpq8oaUr>P-}NF+TIhFXPH2j$RZIQGL&3i{%9os!x~o1eOQJyd8yAt% zOzAFgOW7ZwKQCJUYf2YM*{r5bnL0QS793pUYp98PCL_T^? z%fz`|DT+v5$R38d4l4((oBx7VuN`+%nmp|Eyjbx<#c@)f}&c2K{ZyivM+*(8x zFA27{qcfqx@DdZ=WhXI82yTPOOZ1bCMRD#dICw<+7=LQ(d##K^(}sgZ9_E#r*X{G; zH+gwzUFv^5=Y>}U8ovU*)-}SxqN7$s%649$Q%B4{4%+>r3d40VPVid(>w8*(Eg|ce zwS>3~+wIr1o(D#%sos&Ax7!f3dUIzwSq3C(WQDGp#YAVN$^5d&c2|Xj75FAP z)ZWZ};x>YbjJt1%X}v`lqAd{=dH2}zfOx_Lw+eo_t!KA%TkkYU`?RVCr*pJYWU9T9 zk{(u4;E+F`N`0x8GK=j|=V7D6bGn)rJ)manmWHccNu0#T1kA@4BI}yXRe2dS4A+}SkgQO4hthBydAU;8&bCB z>}`Lmib*Xd5FU0qon~LoR8kPoY1=d34}ViSCK_tm=etL~RW3{JR_j-vlLe9Yn}lb! zQLop)E_BcnR-3G6>}p{MT@6qAEP(zdzP+Z~>n*qwo*=4rdJ)MP(}n32VhakfYF{^$qz#iL*|wg6e2_gZt(us%#X~6C`?HL9l)|uZA{Dr!Ev1M| z@jn-8S`>nCFUL?st$8%UV|noe=pwmNIvSd#RLSH5N^|@2wCjgk+q0fX%T;vw_tQm1 zzRG&czEk@hW<@i2To8X1;F7)fg!?>~By%fAkdYsk@X&p4|v<~*UANOw$E z2I-zXj*ve(fm+lEzC8x7c{GxxTp?QG$1-lRcPpHJ*P$tb_WK{pnQKR8P>kTnD!H=b z)e&rqP3(X_&pbLMHCLZTsh!vHFVXkLG`kdBhKgu}R}nostsb2y*1<9emh$FVb9dC@xvQV(X+ca(R_@4bX#oYLkOeVfhp%XP_@t2lB+0@H~1Bb^?TbeaE__u zMCqF&wffBO+LMa7H^l!#A3S%A-0KkIK0&dpcn4@WlUnr>)x#&Pj|C1z%FSStqE!WV zqjXM8ytzn;$nAQzz9B#&xP7Q}IO;v`YFtlPT=O+3iFsvwl59xBU@{V?5qoOHt`X*L zgNqv<%8h1$e+=gc5NfSa9Ui}ki4uDz+c6K7H;H={lvb(-iJijF3ie26G8Y?(9R8Zb*fw4=$mn-e-eC3$tUo_tPMaiIGpUr&_ zQBf%vr>04bqmHJ3+>3zc|JjQcd`yB3#(v+xDr-q4g7|i0nL6v!)JZNuN$hDHCc`1@ z!w`#Qw&nb{#Bs(moEs@g-=is-H+)n!(fh-mDs%Hlo!7 z>Upw=6AXB=PvxRPGv3VVX(Itl=8R#E82S|jcdD{DoK9-tP0Y`dSjlJAhs77yssGV8 z^A$_z)FPa1fu;5!Tb*;t?wK-alLk$Qe`d&y)$tgewJk*(Lg1$nSGl^r$CN*iXTzg?y7^+ap$U%km4X|gZjORQztyHSMb4DaILwpV(?7}l`mojk- zIs|VT{J2UR#d}z?!UD^w))zL6qO>{PgVJ|m8}vmxhq&SV(+I#a@GKAbIg<0z0VW^X z65P`Y{;Pr*ITO6k^e+e7xM#i_ByGb1N!>nVJC-wER=Kv73GA!nCg)P0^#tbqsL zgB9Se_*Z+S6PU}D?_7}u;#()Z7p0S&KkOV^R=z3a-YDR;Jyp%|)3dP5ikCwga0fZ^(^~yGGsC~WVKyY$bWjkWo!Ds>%h76^X)#CGp_g&zsX6t$O0X9l zh~`KC#&2sTE$b5d#*c!OoYdEZ&^n##J_g1_<~6#g_B}v;H9aJZle5U(`QaaPR#{pg3t&pmwW!%L1axG-?m81Q8T#G!f3x$!GdqhI ztaWOn8YC|GFSU;W#G&e}2gvUNz;{~N%06cKgF+;Y;%g!tgeT+yRL~>^&xoQ}T7Di; zAF*L5P4-WpfL*E&ByBpt9LDxaqGfEiWGsj_GdqKX@0npXJavwk%5S5?BnGpZWm%PN1W?6Lsb^ za>U3IHNnaKINxp>9CRGZaxk3x=1>YbVZhGAo*@w_SFVBjy~dnE*Q%>8 z^oJ1LY%$iMK&nZ4cnwo8tH2Qqw65e%w@u@gxnb*x|ARvk-|{NL($@GXY$l;6Oupfp zXXxbXpgL;euunTIO{~6oaPH_;N5=Gl6~|j9VRPpg!Ik!ZGqh02YgCLtWVqgz) z^Kaa{prJ80)pfH`kj2@`R4R0|IcCR18)`fKI>L%$+Uo>hl)=^;USslT0Aka`w z{WcEnSf8gjDR9c|iq1XE0(mT4)fdu_`<5Qsil%!H1j_e_8W;13&{=4U9H!rCmOI+A zHsP9fiv^bW8e`&2)9Ia)T=b4|3D6ZL4eEZW$Hg^iImbMZK-0Z7205h{ogplrOl@R! zn@@qs^GHF${rhW?LxVYCYp)*Zso~h`ifb>$@BYR2+g_{;CUUC^jaOPv?It^P6VJqA z->sR6N@icXlqBs=TgwKJiR;W*^IM)36M<9Mai+Z_SuhJM%7f0>Z5ux`DNz6)Dsc%q zH=vdHz?5f1IN{(d-ZpqDbe)tVd(gq`L;H;@jo-vN82a+ooy58P$HAk6K$(-=IXj{e zDN&$0;2roXFVack9@+B1jklwhb4QfyK)TvVsI3O+UuoyJQ~DX$2wAKAtT|C9ReBx< z*+-RiFMcYPa}9&-L80oN_6dxa(nkUhGdh$j%wJu1V7O;Buu}qLpcVMiqv8S#i|nE#x2f}=BSg$bbWhPn^VGG_K&3aC*j>Qyba%1?gUa)&< zWtJ1-YoCZ@{SQeg?;nx?DPt6u{-FuKrRKLeGdcM_%2|>+2oP-%DL(vR^B^@ OdJtVnqMFaMfE%U+^5S^_ literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/corpus/ae892bbae0a843950bc8316496e595b1a194c009-4 b/tests/fuzzers/rangeproof/corpus/ae892bbae0a843950bc8316496e595b1a194c009-4 new file mode 100644 index 0000000000000000000000000000000000000000..605acf81c1a9814ac7c3950c4cc47081b4d29f4e GIT binary patch literal 15977 zcmV-vK9<4Y;pKucH90pk3U;CmdmKwV>ns4?hdxdtiYg0XYkEI1tfyIU=vVuJ7grl* zSUfuq?f6g&1SQ-g78xigT0z@|i^hGY{NK1VH-4@U!L%s&^4QtoUBO}`vl+8%BX5;s z@PV>o_TY)faiJeZ#rCz6Q?lz-mhs>@WiM}C38le#3s=8QA7GmE`V2*$E5< z%s`!w{h(&U3fOb?*v%&h1=k3Qzg-FWZ_2iseeNiC0|rlS)?A>W zwP1*S>^+~04F@=PL-NSF)Y$v*=vWThFbta z$09IhCU7!M(5?B^3Us_9+D}AEj|m;j*WsC$V=#b%$w;fvlIF^OqVR`jZj* zRfA>lZAC@sQES&D3!dV0*f|MYJ;>vbzrB$U8K*j=+MD%(0WGy_3_dg@AV|!YR!KCi zpIvZ{qflyBn(9ah{c<5Nj6>%UA(IZ1%yKouHPbeCH<8Ue27CvN9(bBjxUqiYQ`QFK0s+S*R0m(_i0(0@Z!RMd)D=kt0=r?P9n17-I127mH_*yC zbqKZ@wBBkScS_}G;gZ;mr#u=f$LSUaQ$(L~r<7^~H3lSr0j%8+m$$TP`3}(7(udRZ zHv_TW>%Gg|yQ%&LhUbtT+GwSB2~IMIstn@7aa)4MzGHEpckti|d9n0-OoNdA@Y#Mn z;Jc&Ph5B2w7d&hvBVCdgHvY_)p6t7==9wHCOz8*8q&>p5mh+QO$Kl#L z&)5$<9;8hwT!US}9^Ca%nJ>@YNEfycBL@piQKqP#VR%YGAF`>qL(T7`w?utA4z3G= z1ydlw)}vtzn-Q?cS+N%l_5L5F)349{u}NfuKn{M{!)hsTMnuyV%Eesy5naK89Ey&_ zN1KJyGj~Js$hy?o_7&IVG5-y6J`it7g08p8%a{UV5S>NsBN&U%#})m#6ESJ9EJ`op z9WHrbyUuZv8WMCVH?$<|d->hHyv)m+3rG8{H(%X4UDKpN2w<+F{(Ea+1z8pd$e9C? z%k%_x)sK*lla zf&1qTU^6C3JgB}*+q!l@F@@W{_}6I(jdr@X+qf*PMWOC(Gw1OP9@C(fck{IkXtm3i z!BqkawgxvI%y;{-t9Ww*A`^c3`z?2`(7M4;OGrQCkCSt+$1TKctaB4bp zh|&ZMq*18#$m^rwu$+U}ikE-+Rw{}mrq>gYkB?G~ZcLv)Ey1n$hD>lIOcH|1{#iVs zZYT=C1$%mmx_|#9xX~eY)R;6mt1@J`J87i(|{%XVkLtn0D#Vkw5%-Vu09HKUt*l3|-NgQ`P5T)_m$Q;Rb3& zQ=t8Bl-rl5J4mJ;1C2#Ji#ZTNvneJmzENSARj#ZXk(T~_&@vrjS}^)J%~`8XS$4J} zmjZz+J1I3f2~e{Cm_Z|0Y1v*W2SHI~jBg^!sJz*K(;`swOjbZ~A!CJAS|5__-b+ot z6;$zE@VHO=C?2%RB~9`jSVOI z5;y-AHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV1$K3m7=T<0^t*d3MDt$! zW}wi{FHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*;!K<`d6ZtwGc*ZAJMxJ6G z$L!fK*GQ<#%Ulru|gwfOn}OXVzyoj2iNr?+pv_-*76=4Xd5$f8NX5Ixu-F z5tua4DIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_OXA-LhPB1uQ)IdBC-x;ax$@r%z zPR>+S`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ipxex64`k1S2#aWJ!Zd2qrHqN`Z zxC_%Qx=y_KiFDAQZ<+i6s{e_8t;AE@;!>o^X5&SwO*9<7@JcdZ;CKCi0&)iuTeS)v z_~5R#*&0Ed*sNOEP2_fBNGJTLInlJE)@&7ccxIJGSbpHAx)?OJ^2So8V||e-M|d7D z1RBTN;AJIAOx;sT+jD*d6XAce{Tb{B=$BWCMous4Tr_pY>m>-as}4(4`t(c)?)m^k=hBsppXu5a4&yx@r5wZii+ z^GLX&;R=M3--{~rQWYElXWOWc4u^KutBP|B5D4dKGE24;`fBItGCa|`#KEirg4AbhOHxUnBLPC^4nxNv9r1AxYB~|MWOj`{pu&7! zeqq>Yj}@1KYAK%k@6BWZZZ>JNl2kN$fDyyyFYi}oLN(yfU< zJdlRh48IGD_aL8*swCEd6a{cuE}y$-M}RvFH^RiVT*LycfbJWM8B~o= zd@e8?l_#08FD!(pOGkhydcs#ba7li1zn!dCj-={}O_)8)4PfDtJ z&U?q@21|WP^^b9hlkU!FqnBu^KCzhs`uppbTrQ|2+8iVyaZ!2kEJ+&MH_)3{h&fx~ z&(!q8)!O(8de5|d)q9%w6lD zQKxA|g7QY}#n=d^fQYk%C(8YN2;&;vD4xH+F*~5GLG?pU?1!E3FJu}|vIyCJ;0kh^ ziKO9NTD1>7=X7pe%wse(yv_K}OER|cH-uh0ZIK$zo3nu3bEhH+k5kfOd6eb-XTZ0!u4@89=0iu$^xcsol?RS1@F_aV3YisIV8;-QUf+N64D3yR~tn+Bel9ApbAbn7(I*CfEmF=O@d}Z4Lp~otz zy=*ad1A_{lsSrEcNs?I=i)Ws5I=(UlkGDCqVL8v3&Y5hdJLXK2l2CS1#AatztTFvK zdV`qHc|;}<2as@mW}%e%kTP<()EP*CJu2L=^kbsT(HN!$N*nuTe=*Jq^OK&f38ZXa z4TE)JfIMj&R>nanYJv}sS7)7)cfi2p?~QS^-|{j6?N()FG%n$=XF8RqAD1dAh2rG8 zIB7r)KIsir$_YSCO7|ijuw4^eUQC? z-J*(A09yN6#XKsjwK=`eNbUn@hg5%4svE^$rf}DoVo@%fwF>!CCF*c+yum?krDYAx zbtz($Z_&;o0I$2!XpKm{$ly$B>aj$l9vAw>#A?5?PlbOAvl`D=gWvc3<|czbwtF<1 z)oLW8w}z(Z>5U5y5-M}DCS7jXy}HzZcRkYY<>qU<_pX1G}yK_ z1r)96_pITChu2bJunMkCj0W8!aij6e9&Hk<9b$t~;$c!S{7G1|m>HGtf~uQ~@5_9? zi-y#PkHl)Y-DcJErgiqF_G9*Us)!JL8lvd!7D-zH7id3wTgSK);dB5EAOJ?3sLNJK`i$FGW1>vKXKHjiD?725y=%Q1(=frpGtuj)2|AGy+51X#&RRp z2OpE?&w>z8Glwy~k2CAX81d$5v2hwXRuJy_(<4K<6GkO{_H*Ky^fFig;D&XHZ> z7U#M*Byt3$T6tR&wF+e-i%NpEFTEKZT;9KVzEUd6l+yGO<)A~p(q}(*5j8WXZ-)oC z)vLtqIiy9~$Cd#g-~}mbo#6d1HHFBX@kuM3+SwQIOf_mYl@VNbPwV|E@D~?*^kr%( z%ihzwzv5r5aL6w+O4AF`geuqTt)=jCYO1#oI(#u?xBrMVA`X~5H(_99duv=6EE9jV*QY_A^YNM;NV2wgS$^`>B3V|0fKC?8RF+XZVG>0AY>6T8zX?l&nPCIhY}_uX|v53^>;ouoduW3I|kT0@lsnwSr)S zuGy9W9A_OoXBJksDrUb}-gRF|2CHFbFD=pa$y1j(?(3A!SF{a|oE+;k>NT2_aWYe>DSWeo_~ZNnD^yApVpwR1TWejAo9WC9;}fWx+E#5@E%Stjhd;JmVPvf9aR0`+&2 z?`l3)DSN8GUaIDz@^kUxgH<(usllBxRDbe+Rl=^Sf9-mC-m6>rRZqarvzJFm-!)LZYyj#p0o@tj_|*pQt3!!Z@Zfn^}ZbI+xvL9wAu-khBx3_4r{%Tvf5@jv68>u zjH+plvcBCu%VEvpfosRp6y#3jYYDreSA^}i!9L|AAgQ&MjMAZ} z=7i`d)H&-7?9F%BpJM!VAjI+F2HRW3V+hnA1&H-ZUg}hPcj(@b2BlqRV?5+0Xjfsg z@Cm?|y2wqCJN!nMcUY2tg#^6Um=@-W8Rhy`(p!`aec|0X${R* z1d*A3`i63$7-pBT`9AbMV)JO8ikqub*;C};(x6TGFsHz714xD*Tf0~FhIsZ8IWsGB z5!`pTro`6F%LgEjFgg#9t@9E)hH00;wCsFDbY6M+t2U#Wj_WWKynwkE(9{h=tT_uS z6##Sl^dfP7PcGn66Owvhsm2quhUyL|D4|7V`Op92 zF}HdYnS*G&?L^On1*9gzn@yd&k3h>#YDzR@q3UH$^7aWNtw(%2wWAtA_)})Fb1dc# zpbRSShcM?S^@mJp)2f(e^Sfzb0ppjKdvgt;?o=xu)Qr=gdVjqKHoa|i+o&Fw)nq^4 z3=KIl>Avlx7A|7-Wy>Y`IwY?6FfnQco75c7OA(PAXO;f&??NZ`@o9OIBuW4rds?9 z&Ao+(n(;i9Dgapi`xSBner0hR(``!SkdYrS#sG!V zM|h}t^V4#Prc^vOQHxT)<9C|M#c!zTirv52iy`(0FD_$@ zBSUU)z(^NGE%i|8JYm#IlzY01()^V4z04hVZZZ-;LZg+i`l~n_VYg3mT#NI^zGesn z9rW0-rvl@eBibjajfv`AWPataO%H8#tV{b{Hk@D0TMq^{>7crjWqmT)1nuxp-|Ax`d3~W&TEBuHFrd8_s%UmJl#Cir-uvyy^e% zT}>{Gc>~Lp@r?R%IwI`;oQ}s@al}+>DBwVy!jH%G)&$*j!Yom?4~$+=j2UFD_H*Zn zI8ZT;cVUa2LPcP=9w8_{4-IL9TY+pERVU2gS2Z}~~$oe6)_H{?J_W2PSJ0$V#y`koRkYrF-|5BIcFoa&e_ARyW?yZN3J zI$#?Al_jZN?A``qcv&V0`n=p2 zZF3alxbod5#>EST{RakO212X;zzkaG5#@m%MG0z3c|*(j*i;TG<)XXrw($mqOd~T0 z*ic=*Bv$c71NaG%yH2e2G+Zq_d8Q}mh_ECc{k9LoBQE!81CFq`rJ&D%Ym5Tfo{q+H zsF2>VI0dIIt%-6U2^vXpOZY%X6)d)4Lw3+8lppS&=_{SDM3yl zx^A@=_n@9{dj+QwPE&Um{B|U4!qb3KaUu~x$G3Mv8WZSe=`*8uD&jY6Ji8zn@<@Sn z#W3+(gBeF#BkZrJ;SJneDjI#mnWG}h=vypAvkutlF(Ks&^~lZ((^|752*9Th~QggJ$(BC_10SX%n2 zv6yIkoA0>V>0wRtRSQ#()%c&W_R3YXs(%l`hv4 z-b?WasHe6YIh5nzQ8R{L0jt=>Vha4}KheJ*J4J{e*CNDq#IxW9wj%LOJMm`ccY`LSf%Pb~U>SYpiB6H7KL(HnrDje#L?EhThEW z8oXcS6Jg6*nD=f$dre4WN!sO4dLSKovPQMSsF32_ zTOY_?$qAXT0&7-D1pX^Pfy@<1Wws>#Zx!iO9`~MhWnIC!tBN~_)jEX3>lv_1DPBqJ zfkrI!9@bw_qUb}Ap>%uCAtsHW{*L!-%j!f#*6dT*4p4P-b>zHpl~3alQ?my}V5Sm{ zX;{-8fQJtirtVBQlqK?Ub#={oJ!3z!`kFRYw#yyuXPu->YB&caBA4tOM2MFu=Iq+_ zm|b^-Y+lS%fIT)KGyF(}V8zug^4T+hfvH&AmMMY(^|-ls*hcR7JqqMdK_H|K5w49V4V z778%m$L*;ifDeFZbu3O7@;`!=f1(OXZjJl=*n-jVN+>4CtQC;~O9o>`zT&C%IKqnU>rYp3k!cxj#lAE7!32*2-6-Hu?c9VgI}0X} zgTbufGIS0~VbP)lpn@N?u~w}T3`1kUPf-UkzZ|haN?}w(O*_RnEhICnFstc|eOJoX zqr`q_VLXP8{4Z#GYesVxlJtC3Yiv4;PApybd$)M{P!bg2_D3ejy!_*6N;EL_6EN(2 zQDsDmBG~iP;c3MLxMrpXTosxeGHv3<)pi1A)JvKGHLhNmXS({kBaXVMvgH~z4pGRC zG>MSLkQZPJlGF7vYc}(=YPXiP?$Ahb+v%fvedKH|7vRJ-I_6S#?myLIJG`WAFfjSw z3%Wazb)Iq+4)7D(@tw{8;~#WsltYm1W;&1zp--YV48V5w4phXcM#;4kZ-0Pbu?R=B z{vA&ur5rO?x6tp$Mn>Xa>OJRYj>cL8>I=cyf+44re$E1ETF@f67li08mR(KUv{W{B z1=R7>4%w-aSc42^fj=dH+AyWvGV3mg5hklo;NI8Isl)b?OxeNbTED8aijj^YtldYR z6HV6nGJ&oF#tPgnpB*q<9Du<83(s^PQmxWUxVs3oH*=wR-U4u;suiZrLn6a3?g1ET zUJ&z~f}&VTC{$G9jUjos?C@j`C2I}8^G|}KHgkwceUbcx0X6jf#*$_H$I3S>LYCo@ zQQ*l2Z405=n}>%9So|;(AJIq@e5`7q<<>DVu$9^WrnXlCvrPk2N5$8LiM z3Lt^BI;(u4a5+E&-lA(veK@1S=Vg2j8;yDBmTK*|BJZ}}eI`?jOGZu=ak`(An>x`g zW*f_eAZU&ZZHBSWQqMC`i*`;wGYb9?4CjOpE)l(o<#Y3Xu)PsVfR8oeusM2gb$<)Q zc=^G#iDcI&P>3l;6OQXx!1LMIGqELBvtiMJ2DRm_H?8};Z8$gc{JI%?c7H%_KLRK< z5@k&p{HZZqIJ?w%n7S^^gpvw)2womATSMpTh5{6R_v%J|WP*h|k{+M?Jwp=56hKeu zZL#Y@6JxZc5)2@9915V?RN%AX|5L`EfFn(kVt_qWp=7&44$$W_H^Rw*6||4Q3J`I~ zsVeTri{{pbgYJtmmKoeD(C%xnf~BPv-!A*BCl!!zKquqOx#*t%8%~VM%xZvijBTv6T|djU&;_w7}ew z8WC5$^9L7T=F?=1q}|rdPf?t5R91{jl|B1n;_6QhHn0=od)J}Mldz64(yC?{8IFl1BS*&|UOhPVDF>mS${E0eUlom%I;^)fl&55Z13E zrwyZ6C7EGjkXv;f-q}DO9l6-l%74bp$jLq+zU45aj zg^LBQuoPD^=R@9c3?Sa=quQDb*Z*Tf0eF|&HgNywJKsX20u1YuLnryf7fo^!sAbg@ z@s`?8rCY64L-%7T2SQZjUjlZZGVDxo8CBD23*YaE1mTpf3rpG5dCCrq^ zyT%f8R+0TeZZRk4zo({|PVDduw?uaxhQiS~`v^7|E|(+)(u6u(iW}m&BFVp_ekl{B z-;yZW31$b#2(4#_8|)yY1;lH2r&%TlIa*ZB_!Jqk{<5zAwY(vndt{` zUF@Uc5}N4Ajp(~5*03vEO6~7Q-4fhX#R#2lW3;t?FGBJ-{3jETr*HT#sM`M7t6R+1 zENM0zRRRMZr@Ne}do#+&OXJw(qqOe5+Oo^xwemc#d?7|fXa@jJ(@Y#3jOZL6Lkw#x z+F6h3YZOHzkHIO^{7|@iNXfM6SBO`JcBW}>9|6^J=(tq3ZG{3_+~{bBs}q!Z2>c_V zW~J{}%qa;nmf^njkZrkLh?SimL##*?W~V^sMVn#}hmk%pgG9Q|^qSYx71!?cum%+0 zVg)W4=|u33f#Yeo`q2xw{Fd^C9o#(&SW-&%=+gvO!Rd65LrZA?3(RrZ=V%YofsM6S z)0qi|^Vtxq$ty6+yrOfPt9X|u;L2sZn+T9dzfU5UEJtEYEu7uTiBus0tDDLK(;$h zaO?LiHZU+XFgYiSR8DLj2Ba4jcspm6~L74Dk>8u-sUi>oBwrZWy}Qtt%%cByf05^m zfKf7&aajG_giq7 zJchyxI@J*>j%Ar&)GIv2vW<}K`QJM*^@ic%PSLS}<)yOTxQ$m;F?fmTfb%kqA8(>2wcXi(HQ zW5I-eBO43gHvnR3qg?sF5>1e|j}Z=IP<_}@O6t_*oK#7 z29=S|j9)F#M{*(&t!pSfjC75<0}bQ4|3J0(E7?vBA3+_t3OAtiNADYU6BQ`X9*_1YBcPEn21E3tQ8bGWn5xOyJQQaKMD4YFx33p3WH( z*I0(#ETJ|;R^6ukQ4s(OF7R6aMj7|zAewt1lG#rY)JM_O-RMCPKuYgRRM09X9|2sn zrm+n#_-f)$_Jk|FyOfW6Erx*dkT0fkCf+jJ>ZfNhR{7o#KTP>fAj5cf2uJ2bn?O!$ zj=uvVc~{a;3BgVe9Y%m;Az)O%Cvk{VK(MyOk}knNe6#Kncp~kz3R56=4|z-q3sFoP z_TY(PTFMhaIcDQ5&18I8L*z{~moCriyB8fy;-$xVltCu8iV4D?Y$<24`&(6nk+tG= zd$ve%!^z>Sw&AvmWX9X*;zL2oH+butmyp(3&?Xq@%8@C%$-E=XCj zhlVc7`3pmbgFt(fNL?|@ZPi-E)EPFyyI>+dg#8(;VKK9zZp0(Cq@ATME1~{xBLcCD zsorM->gzy@u$zDIu0Q^T8=`Ki8krc6<5w|Roru!<@?oa1izIXoPkCo>21}dOqf@A1 zi(z?7s3M*!KATcOlp(H^o17SN#K~Gf;#ITmkDXs2Ye9;BocqiF_77i-S*RGH1+S1c zKLn=FvRIYsTo*T*kVzftQ-Z2$%mT;4aZf=ig3fTqCNy-0hZ?ytO+%|aubVWiOK}V8 zQK9HznddrYX%P^V2j%T=lD5#H?J9*n*__0Xe@m^*%uusYGP6bl69URk3Sk9uWrM16 z)rrS6uYaSjk+2ZHI-6(ZC$feTxiTH?Cy!K(Shz4O#Uik(j}!mMy4g{^>=(fZWM%xx z1AZ=jbvTrdL{c9j`5TZ`$^z%+rNTrx6ihKBA{~R1mA3GM{;IYb$XCOwr*IJZ*1>@$mug<~krzltlT2d?BQM4d7BxbVVL=9;uMI{nC=GmgUaafx{JPb7bDM1|-{v6EO{=z<@n zV#wD)^7_H1r;Fuj$V{n!OQ*sX7N|ala3LkhdDA2O?D9IypgS!@KPjdNe95=INh1m) zM&x&gx=6?)CIBp`0{A;Wz0ZR`{4C$o|M|Blz_Cw+>#*e#cY%(aD>cX!fGR(@zvz+km#Vc`Be?85%w2L-J!>e2 ze~32jZ211JJM7Jz3}_~Np;1r0WXQ0~;#FW{o=MtRi2vc|uT=ZLkHm!S{rfk>;ju*0cn; zPwvG=o9O5gb-uL>gSa0diN8Q}bO73XIO%tEir8Q;-Ic@$fFK+yAVFrSx+u%w5X3uy zN81_{6mDH60-q%Kn2BMuBu*$tw1-a4dwRvTIUSZ-HGMi4oEp&|__>HJLPJgRioMsv zce`vUrqc{@;U)rA;7znO`!&Q?!G8qGK0^FCM4O)^sv|lyW&XA{*NAne3~H?b50Kfb zh~Y)LsJha#c|AuQT(mHuvrxoNE?C44v3hfyA`MI=1D}H6N=&sbp<62f?PR@}8TwNe z6R}?do{urflU7iK?l+!rO7~$YGOh=+A6Oim<@6?~YUfXshliSTcs3}cr2LWirt6h+8H{&|aGHH-5lC)?fSK6ec)T-t2Mhy=dqP&oavX) zIikZ28}JalSBEoDck@x~4=DeR8i?r&EWr%W6}sE+j?~z~bV3kj56BhpD_G`*u;IG?Fl? zEtoyFP*BP{AL!>lIq*Pyt&RSt7%y=?SdJ@kuG`quc6JCa+e|s0@Q*Vn!f9R)_$8&v zFFX05E6VxA4=jTDI+?F-B4E)*)$Qp7q2lrZ1?EFqYV?)eQ- zbDsmRxkNsCPs_x)T`7u4UdSGXxehA_t(*UXR^-sh7ZiPL$eDsTHL{9+ZnZePE=U4t zy}H6KepxN9F)e;@?AB!x5{Cp8Br_*4?*XOzb9P%Ap-P2gQE-jYVX`xt*}>wB$?L(_(X zMIPprn%C{~<2QMEXkF@mJ?Dj21RB2rzScFu!J?y9M9Owvpi@W8KMvacqYA@yF;4JW z{_A^Mfh{5Hn6-qs4BPG3w4Mh>s;SUmt=*mn#DwCrOEuV z$aYtSgcbNEI@I3Ged0EPiHy5%iD|t>7@{o^6nXd9@_=~41h)!)xvginbX)H_ z2B&khQe>*Vk&+%(Qs9t3pGtkHmNJX&QRiW!!*jZt7d@b6?3RYBfPEC6wuR?DysdaX zXVC%`-*C{$+_98GY_o0?fq#n?F{<%HYj+uDNLopJ*9_Y}z-O+9iC0B1+gQ>(1`Z1& zB)lE84;xap=Im{MtBOf2CJ-KWI-O=;&Qwwm&}rK<-w%INIwl%w+UL7RzEv(u?pEtp zpOXcV_nU-gwo$Lwz%F#q6IPq7XY6WW2we?N`YeF{CceF<+v_d36P_Tdc6t%X8PkR7 z6k-bsv1(s6zUFnFNOw9BDr#HZ9zi@BmrxBz6uW5C^tvgRgFB7!O&Ql_4h+qGXEtPs zSopqrYRIk0p86i3ez?xpY&@hUSPQy1n45UQaT!%rBun}0!nlH^0e!RTidgq zNXu1p`S;UBMZU^<%)V3m9cD!{cw7*Sha{HGT7>FfY;fp}h8b772_CBa8jJ4bme6;E z*2)5s_sB9Jz5`zz{s1>*#6?uxGS$_~_GwF(Uk~N10PZ`@7YbjRySKF5C&GApZR2F! zV`C){{?*SpVWpuIjjQD^7a;&9xMDkeVq=5$-GmKOIBQ*Dh+CON__@aKuniY(8CfX| z6KE*PtgWV6MsPVmh1VXopLNhu%te&erQnji_k{aAmn3s5N05;pm+;Vp>e608n(R*>vUe+-e%GNXg7*6#%b9CO zW>Ada$SS$A@7j}!xHrWALmxbMjNI!G<32&Ltat}#IFnlS64k>et&ar` zMas=!lcH4xccXMpOuV^BiOB7Gw!R@iBDj61bU5lg?`m96SX}cpD2aJxe3EQP!(cKJ zrxAN<#I6zMZi9;(AIgnpfqx9=2oP$mQ5_z?h=~$=CfhL&l{bld6_i%02Z^BXcnn8K zEf3r@wd32@OrSp_D;uE9;+8!|cenjK8JIi2D}Vmun|+nG>!iU-Egx6mbr2^=`HCVa z#bi|4$2s3&50snZM!i9wOoDphk?Q;_nu=ZB^cpKTHdIjFY=N%iDwiwm>wM*y=3g}4 z`$fs1V4ux>4^dGm7^kL5jiZjHf82|J=l|J@7JN*C4aR=oz$$A=C4%^NW0^YZ)6_{W zK}qas945md?86X?Wwzz~x5RVV-(Nrft%V$PWbJEi1mx7&_tHKN!SN+%tg4?#T$;ht z`rfP>n>M1=1L}FQh!YHWvQOorK{MXW>S-eZOy-PXju`qC1$U~lIGj#u;Z4lXl32-S z)rZ9w*Qx)}H}e%s=+q*dZGolsAX}Yt%I=vmX_E#`h<|3tjn(lOowY4R8$#fx5m&jo zzU4}d=7G6m(3QG3)C)5jMnf?II1#DOhI}^fxQ>nyJ&w5Y^}o9(lBr6nz14=<4e(LC zE@@=f0+n5^$oCRsEp@4H?35vzH>$-G(&t2 zAnd|3ZI?1}3_1jF8vM9Q8^wEAvcdw(sn!=ZjH0wT-GkD1VjJ{DJBPU8{L=`)GVm-9 z_&JjE(g7wP+7jH;3jV8t7&#NX&-5=?B-=yws>P|X#WN!(YwIa|dfy4-C_qt1Hz8-2 zU(|h`POO0k--8w4uJ~7br4yLTmG4}U1>##Ly%(jEoImUwTUNd)<=!aZwLMkM@zb-g z%!-#o8gjGfD)K-0?c|~L+38C;;MIWxn*zF{^b*>q44pqplj?MCLWRsP;WTelF1#*~&g<`GZ0vjpAz}9E2z20aVZ= z1<#10SXzD_Q6I5kC{6ZHpMYJe4`0w9h`|RW7{1%Y4{dB5v>oGhPzq3}KN#%#P3$C8|DXB* z`%a*vxf6BfG;+kq5;eie{W#xl8XR;S%W^QB`{qyzIbp!h0>H_^^!|`eES@0|DOaw6 z`n|@SLf5LRFZ72H-E1+|p+KrhdUy>}FRQ>2479G~PPa|tmbqc;iT{H`65sMF!qV3G zDQqU8CrrNKn`h|c>!3Pn;;>IUEKRJwd2sINRY%73ffdJFCSh~u7{QhHe>1nU0Pzy9 zU0+6I;VZXb4V1B2%YcR!7xHzlQUFMpox=}2Pd2gyE$9IecXfi#+hL-IyZ56MX(VZk zv2OM6dYfDRWhD3Xwh+k zUGP;is_6mXc=K=EyP%;lH`R5sQIN&i%2Xp>5{W`L}3A^^x<6YU!QWPc# z#GV}S(yDFeRz}LIB;hyh894repnu3JDb_bt)Xua@h=O@n4&jOKn(8L$OJ6iRBDZav;!9PW?6x?pU9vI4N+-?TXGl%mR5VT-6uSkNcJ$+KQ%o4+P5hh#D93h|pPR ziyWrkX_hFt$@54-!u|Vekwb$yVQa4*>8at^>xyeH#qa*b_uF2q3?_1`3XNA< zPwgf zGbvF3A1ZMPIyaz|_`sBBL^$E#EZ#PFDs-KcBYV)n>_hvFD~;d8IvD!$)}6$;{Kvtg zgFu;++&Md<5h+oiI^Z4nDlgJW;vU)Zz>T+~mvcvy>_EEONvN#`>0fE*w^RBV*a%sx z{H!@qCsleL2H8iIbuWG@mU9h*?LndHp7sfhm(oW94>LNHE6iVAcVM_@HLz0xWS|xJ z(xc)642$fws}_aoMd6+SwsVp$Zqw zf2n*x@_wv@^4NM?AT41zFlmhmycgmg5N1&Y(+9$QZ&}4ie6JZP99?g1Au8zeN zXL4isuwJlxX=RoZ;%lFXWc?3GDeoVW04ZY>m;Rv%zoq84Ipk}S@agp^4)`u$!`bAK X_R6{HA6DKwhk6iQNurw1vw$0>5y--x literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/corpus/ee05d0d813f6261b3dba16506f9ea03d9c5e993d-2 b/tests/fuzzers/rangeproof/corpus/ee05d0d813f6261b3dba16506f9ea03d9c5e993d-2 new file mode 100644 index 0000000000000000000000000000000000000000..8f32dd775ada5351365dbe3035668944b112046e GIT binary patch literal 16000 zcmV-`K7YaB;pKub3U;CmdmKwV>ns4?hdxdtiYg0XYkEI1tfyIU=vVuJ7grl*SUfuq z?f6g&1SQ-g78xigT0z@|i^hGY{NK1VH-4@U!L%s&^4QtoUBO}`vl+8%BX5;s@PV>o z_TY)faiJeZ#rCz6Q?lz-mhs>@WiM}C38le#3s=8QKVX{k`V2*$E5<%s`!w z{h(&U3fOb?*v%&h1=k3Qzg-FWZ_2iseeNiC0|rlS)?A>WwP1*S z>^+~04F@>QaB;7Ku_Xq--ZKONJhE~?FMuhJSGtt}bJEV6DPVv^mTf+(DeA`_W-=QX z^Bm&k8$pFdRRh8Wo1oz$THK(QTc>Lr&P0jDD+_Fl{=U-$v*=vWThFbta$09IhCU7!M(5?B^ z3Us_9+D}AEj|m;j*WsC$V=#b%$w;fvlIF^OqVR`jZj*RfA>lZAC@sQES&D z3!dV0*f|MYJ;>vbzrB$U8K*j=+MD%(0WGy_3_dg@AV|!YR!KCipIvZ{qflyBn(9ah z{c<5Nj6>%UA(IZ1%yKouHPbeCH<8Ue27CvN9(bBjx4@$zq;D=G5!4k)1OmHZq#eujX*de*R5#GdIdurO8MNMN9(PLR zXyKCBji)>sE63>;2UA3!a;KDP12qODfB~%C5SO>KYWWV(*wTm7^fv>s-s`=~+`Fm% z28QR59@=Q7cL`21h^h?Y!f{)I#=c{5pLg)!33;*f<$O$okpA%5em&(Y(YkaRp--q9 z=(P#evU>@T&9rzjoxOnMl{``s3yJ|Ur=8mK4|pRPjj|%r*TVuY)pArezYv{G+E~iq zyQA2J`dhOXJZvN*U6L0z{>+!2?7OYznH(BS=?BWBJ;Jq?^OJ5*$Kl#L&)5$<9;8hw zT!US}9^Ca%nSd|P-bfd=5F-Z*Oi`w&o?&=OK_9ZIxI@kFq_;$UI}WZ3f(26`!PcW; z44V+4dFJ3qW*hpUjjP~hlXfc+zo+v^QgS@wGqHQDQ-Y57^2)G(Pm8QI$Uw$1>w){{4PY}S zNj#{&OxwD4Krw~ezWCQ^35|BTx7)ZZtwo{kZ8PWb3?9>8M&zy*7H zin@RQB)HKbcH_Cy*b~(e%}kMH%0&G`(bo^1wP*F9l>h~CbjzDh8QLcT%)4Q0%M>_L zZ7!6jg;6Q9)tXk_+lDSss6Ux5*M+X+DClls{CQ%4*i}DSr11=0(V0_~xp;4zalg_6 zA%GSdr$REyu?ejqUiU@m6=Oxi*h{VPSWCh-*l>JyW(mwnh4ru+D!$W;W{n4mj&cQw zm&C32C~jfi(W2xtn-00B5Rb0X1r}B+V%Mg>3G<{!Y;rq@!<7M>f@)Lenc~em?!EBi z=X`mQ(cGuOAZYeNGYj+qgoI4P#B~9I*brIKB{CyJ*F#zNA|hw$HyN-oh;*P~#Z?=_ zl>+#NP^N4USxF(D$wd4VwqQH##r$>GcuP9GX$_c=H4&-oOHD%7eCjdb25Ln{p#5%? z+n1+1NTwbGjYU0+IS@j#DJCtxQDK->uB;o8mi~RvG96-CF#0&nS*uQ2cD5py0)Z<# zDK$C?P_qA+K_ggc*8RzPtfV}(^(ACm3fOHIEORPkN# zxKI1#D%#~nD;zFpp$Jxx6++@HMDNkDfPALj`JYcnWyDcp>C?2%RB~9`jSVOI5;y-A zHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV1$K3m7=T<0^t*d3MDt$!W}wi{ zFHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*;!K<`d6ZtwGc*ZAJMxJ6G$DfFL z?AgIE*GQ<#%Ulru|gwfOn}OXVzyoj2iNr?+pv_-*76=4Xd5$f8NX5Ixu-F z5tua4DIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_OXA-LhPB1uQ)IdBC-x;ax$@r%z zPR>+S`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ipxex64`k1S2#aWJ!Zd2qrHqN`Z zxC_%Qx=y_KiFDAQZ<+iPs{e_8t;AE@;!>o^X5&SwO*9<7@JcdZ;CKCi0&)iuTeS)v z_+sF$w%Hm%oY<^d*iGbiVn`?as5#NJqtRdE+#_J^rwW_uS-XrCs74@PT zEkN@>?}uS6^unR83->0iFgyuKk`_vb*0xDL$UT@Qdt_^~K!6!K&uv$98H@mgWW{Ng z=n5!WIz!K7w%l9}MxiD`=cw9b6w-ILNkwxRM}w=MK9K4+TLtXLTZbal*QxY3U_+wY z<=R5|FHTHlnjgJTB53U|!_aInFeD&8vV?ekI?4FH0~Ea-w3=Z(N|Cx5fD=hSP*6K> zh1O}JvR)Wb7Nqb_LhusjNB{*2v#D#Cr4HtHcG2Q$C73w)VI(w|^St1Q-nGK> zFY`#aqTvdJlHZFe^imZZ0cYE&j}C`+)~kwh3=jzCo^M975;N|U2{C4jW*h@~z`kyb ztGkuogvl@B$Eco`$Z`$?r*Ee%bOvUR*COb3;ByQ2oh>nVEg*co9$R@sQhOimGcamW zVd1}%fYab_n+^;} zkkpy)eVyzCR}YElc>38k;H9GZ6UUmH_e)YqpCbW6=MF>1ARX~>5NbLRwPbdUKA^&U zUVdTNX^$0`f@&$AM*Hv7IYKriDS3c`nV02^m_Q?Gc*yDxgf5T%djq`Z`+H`zzV#)JJg!o;;)!~(5=?i-64 zREC3p&G^qtGPdzIgkC#sks8jMvw+=mry>cDQ{4}*h+3sj zYaD5c!dP=dp3-7afD2Z7s$Zv1@c8w__{Y`KISKv2ymTrkmXoyTntuXOV&2q~Ww=${ z1w=0&T#`g4+G$EJLWAxLH-^&RTR?N24}0gYr@33qIyV8On2TsOw6bD7kt^T$x^ZPl zaYmQ?7`c?Do>fOd6eb-XTZ0!u4@89=0iu$^xcsol?RS1@F_aV3YisIV8;-QUf+N64D3yR~tn+Bel9ApbAbn7(I*CfEmF=O@d}Z4L zp~otzy=*ad1A_{lsSrEcNimXH6^mz{b2`2<1dq2lvtc>Un9iAOr#t3Mlaf$&Qp9Fw zRje`nIC_Ja&v`^95C@QOeP*GQ`H(VlxYQX)fITYQu=Hc1%+VO81xg$HW`8lx3iFem ztqG)TUk!tGVt_np99G6bC~ATak5^}%jgoi3z~t|ZakSs^G6C&YWo0xj;jm{qm8Tz< zDk+8HbA|9|^6I@WZItJc7hr=*i6=61cVJ6~!~>SKm`#C2yV7WlNWIA5Ols<}M57)T`o+X*zp_t-e+#o3&sT%r_x$E2 zgFm)=G@8|FB%`;6rs(O73l9=1bFwB~ZrQ!S{6*7G@AooRj@yOC3|Z&g>>NyG2v_4{ z^wB4((gf{CJd_e43+ByRf7!au=~K&VZv{2Bi7xK3J9s74hQ*BIB>iqpM9ge{2Q20b z%k4DSwl)P6t?Boy;f06SQem(Pu1$;v-6L_M@yi}<6003zgHhsPQZW2UShJWJmG6S8 zn~U$ue7=i@)Q6A6YPj8I)$^ux_NMk@_IIj?5PTY<=&F=J=4r8U8aY-F?)lRrL%HP|K$X((s{3)<+<-{S5 z7Z-cS#=OL)oXW za17j0Cs%iB4ZvG-X;9Ay4B_QZ7~8pIGt#pwPg6S~-CSqoS!Zd7-!f&g%ny(eL=GAk zZqy*w{v<;4&9P2dd zHJX%hGG1}&+5Q7%4G7t7!xsj-5_p%jb2$@!8>o*YCcved#b=*s``C0o7@$Z@aXa?s_(@HAe^7{rb!tTEIakbW^;=?ePp%c z6)nu)60O$q{z4CRk?-*GbMfMXRW*O9!JRTxfAW7-!mg@+?Rt6Mt6TY1Pr%Q!mq$q7 zHBh~51Mt_HIu`OWM+Qbhwn_XaqV^ADr^ar9Q_Q&(EhOg-9sf+R6mo@VZQM7ZT)udf zNGsr}pvb8P1YLd?JzLlDoL;)vkevL(F%`psWgU9#XpY$1`*^st+6k40H{e?iYrT-N z+GaVilE2=Js%eh0zTG~{Vbdz9;~t?5Rk#=~pN>w4OLQ=G!!G{hb>t|)`lbgg`jw&f ziW(N~q13CiLssfY&%5X>cB&atCEeKF(-`GW%o^rq15pmrFkoU_KgzdM%KIJ4J zskN4j(xIp3gy<;LIqMDV&3D+JV*GU=#PQ+=+grtB2-F`1i1kWd>QsAo=-!Y9rCn!Z zJme;5S7Ed83BZ@S$W4$t{6?2|SdxE*1iaUn7Uqf><@#3ATa*lcCvq_We1@mbOJbH? zZ}oNDxUTv_4~W&>11h31fc`QYu!K#jCE%IQf!>d52j*O4oj5l;D9u#=fsR+H4&V^F zRj=x44b4^rk(qw_hH{}8W|y(~KJ-3f^Jt!mo2ygVQ{>>%piTKOr@(FlNQNF;yI1vw zc=i%GGb?iu+;_L8#MaEq2Oy6yIuDPn^AbCTX_vsX?0iIYUU~VeHlv!3>o63&fVmgY z)D1$cISVTl0CW5FB5{6CF5prVl6qjN&Eq~CYCb@QMPCb|(gEFNRFx!k-IRe+3;30( zeq>A;@o3Cz1zq|lku;P-3O0%votA%v>F>Ybd8?gzY?2TyqLnj&`MAZ3!c11g#k zypaiPe|R6GBtih!cT69_l(UF9P+D^MuDxYOLit%E2quhUyL| zD4|7V`Op92F}HdYnS*G&?L^On1*9gzn@yd&k3h>#YDzR@q3UH$^7aWNtw(%2wWAtA z_)})Fb1dc#pbRSShcM?S^@mJp)2f(e^Sfzb0ppjKdvgt;?o=xu)Qr=gdVjqKHoa|i z+o&Fw)nq^43=KIl>Avlx7A|7-Wy>Y`IwY?6FfnQco75c7OA(PAXO;f&??NZ`@o9O zIBuW4rds?9&Ao+(n(;i9Dgapi`xSBner0hR(``!S zkdYrS#sG!VM|h}t^V4#Prc^vOQHxT)<9C|M#gA{O>5ARI+2nAh9)J&^R>QrE3U=+b z+KVCf2QMyTj3YyCZ@@?wMJ@GE={#Z7N|bxLi_-j*^u5d-cWyEgKtiLHu==Yw8)3Ik za$Jk^$G&C=1ReC)v8Mv#nj_jLsf~&1U1WacuuTtbb*xMKT{fIw&07x!HtC?cl4X4| z*#z&(y32!BdR|iE{}5?_AcJ*IJ0ve_<2w9`%ei=M6}p6s-DUnpV6NT`j2q5+WR?&x zHHzO{9lYuP?_EtUjCljgmGO-FaylaH{+y1-T5-fwYbfAAox+dD_0|O4biyoAwhxS6 zP>dO5t@d;0i8xR(j(1^;okB%mw;mxVKMxIQgIj@Y8s$n2MfYHks5&z6DQ2YazGm;5 z4wAZ$-6e}(>_j>ogk5g+KyUd;;++Y9(>LTmNMoiR>H=FkPWqk_Eo;05&ky&sQ=ICU zFCZY=F}wMm6gprVK12cbq=_el%CO!PN)(ITMdjO1qzJY~=LlNW_N&`J?Uf~|UF_Zl zVt8352>QI-7;SSD_C9z_XiN_j)e`PftrD&?ZP z@V4;=g-jze2-r|vz9d%hMFaQ=k-JW;^)y^9Jb9)k=!mc+9{sit!y_*DX#M-?o#VdQfrx}-fM(dK9c@736^aFgkhaRJiL_ymzLD)ccZaHsudOIc+-AT5 z;RaL`ibP4kg^<9aIJIW$nR*54yX}>W0yrS07klF;sa7dY;4#GIU!7C#PYu014+t_+ zwyMu5mowj63;htd*Gdt?kCS%H5A(+UyYi`$j>3mMRnDf*%2*9&46=!4$6xBfCJ%cW zqC;`e7sOS2{m$~vVXN$|iVqZ3Q7kqiD3 z6m`yy(bH=L=9QH$*Aw1L@d&7=wi`K=ecPr1+$S`}+IVf_6NU>w(S;_%dKR7vvQ(EQe!)l7g% zVE&IIW73d&gIP&d3nqsa)BIYg64D>dLrNRevKwCcB>2o*EvCVxyoPFTlU zg=}!Rj>;=9_~{1;M$ckwOX^eogm8H(VEtWV>tK;WIp}u!QOb@&Vc$S@HMXcS6Jg6*nD=f$dre4W zN!sO4dLSKovPQMSsF32_TOY_?$qAXT0&7-D1pX^Pfy@<1Wws>#Zx!iO9`~MhWnIC! ztBN~_)jEX3>lv_1DPBqJfkrI!9@bw_qUb}Ap>%uCAtsHW{*L!-%j!f#*6dT*4p4P- zb>zHpl~3alQ?my}V5Sm{X;{-8fQJtirtVBQlqK?Ub#={oJ!3z!`kFRYw#yyuXPu-> zYB&caBA4tOM2MFu=Iq+_m|b^-Y+lS%fITF(}V8zug^4T+hfvH&AmMMY(^|-ls*h zcR7JqqMdK_H|K5w49V4V778%m$L*;ifDeFZbu3O7@;`!=f1(OXZjJl=*n-jVN+>4C ztQC;~O9o>`zT&C%IKqnU>rYp3k!cxj#lAE7 z!32*2-6-Hu?c9VgI}0X}gTbufGIS0~VbP)lpn@N?u~w}T3`1kUPf-UkzZ|haN?}w( zO*_RnEhICnFstc|eOJoXqr`q_VLXP8{4Z#GYesVxlJtC3Yiv4;PApybd$)M{P!bg2 z_7F!V$h`dHXi79N^%F4cd{JdYiXzza)ZuBx1h{6V23!@I95QX<#?^KLX4FfX05z^& zm}k2BN5s1$j=HI`7#mmq95Yw9(C^1aM&e)UJ?CeR###gF3&GifA*Ymn z&H`y#&?2}Ogy=4oT}|AyR5o@6)bZ2~*{PCPgA8VYKP7n?~9CaX{2-q+8m z!}gL)*}>;pzpAu~k&Ys)-AA4iw@udhGJ&oF#tPgnpB*q<9Du<83(s^PQmxWUxVs3o zH*=wR-U4u;suiZrLn6a3?g1ETUJ&z~f}&V+N+?uR;*BABxa{y`4kc?1zw=Lmqc(Gh zNqv$0gaI}5{l=1I{Kv{SEJBvyl2PEv25k$W+M9=m30V9v6Ccq?6nv~|pyk#vF|d`| zC_`DTxyHv2=xAo{B2Rcj`NwX92MQp8v^uMNp>R1s1Ky%*Ono?`!slgt4jYYm=$2~j zxFYYi-+d-ii%UjM6>++slbbrxEoK|bg&=5-3~h$7&r;7bP>Xg>KQjvc5De#p5H1nD zisf_jez3g}N`Q|w;;=b-aCLtR#CT`sY|^+1N9&C04Ux(SZiF z<*hfZ`@C&9H}m|u8GCkrKyE(*C^ZsgO&R>DFF3g0I3U~-!9$PS5L+9&; z0u+At>PCKKf`vPh9-sR?LlVanKu_syvFkz;W3;6b3?OwJ3ZU9l;IrcYQ^uZvBTbTG zfIU^AWV=ER(C0HZ!pVUZw2#0F5OK(0zHtpbprYvrGZlD$H(5Q}dU05*yfsPvX z_~Ml#afbrpf=v~%l@iU3Bhk#Xz}%7=5m&wQ2Nz)G(`1aK-PX-dQJivAR*XxPJ^Nzf z>Q4?fuoL2Y*P+Xku#Pd(s$~RC*$?3)81iZ*-w;%;(?vWG;ea4CL zF@m{@<(!Rlz6j9TRfZ;a_+9*?O@%}DO9l6-l%74bp$jLq+zU45ajg^LBQuoPD^=R@9c3?Sa=quQDb*Z&`5 zLjib~+ct3j=sVv+qyh}6!DhYPo-O}RYUkvtY@(nnNIBR47Wsg z9frcuIr|7U7%rD21=55%T#6gwxw9h4zoULB6Q$phDB1~T2gnGmXNMc?AfyGvYj>ww z66qJ>z>`j6K61{GhhwvNrva8V0>x|nV$J8LmhB5g?;AZIP%AElrM$>q9~c? zhbrdm$=NH1Fc&A1=@>aBB-;;?9`5oXK!&E7=?8FK?4#lmn&`=m=({M^uq#_i?e9n3 z65Le92%T`$kv=hlM7q!Pn%C16*Y5PN1{B|71uhxsMDUJ*<7v41(F?cymhy!i z+&v3eQcCvd(*#$+>2!}nOKATK%yHT0Xb;kXjkQPa>BrM`Ck<4mCkN=Y`#_E!Cw7L5Ojm%f09hga_kTG

tqh-JF4rtl^ z#Vf-l_B@W!l!o}pHSb(%QvZ8f*rEyo1lTq}wmVL6>-RC6?e1WdfXS|uqbJ@7ej@gzPEB9^1t;VP_1T|!dc6w+R zLRxb|i95|KBHO(HoV3Q6;%)6_M9?cHuP^>hx7hIbmJ6_d;;!{kR#>y#V0?Cf>yrq}*(`wnQAks76(1 zdL1kjJ5``P#hy3Fer0V@HL!GI0y_n9|0ObLr!FuH!khzPQMNtP|H8wZ?b!8Uv>1ro^ocr-r{ zl~mj%Ys4ua*1vTX2{>hQbRv)e$O=Wtm^pD?G)rjgam6-#ak%hT-B) z(XoN$rLx|*jaODvxsyf<+LVzjDX4_g%ZfuK?l{526xfj?<-aRuT*U`)|ZB?$bXlK@@b< zD(8)iEiJB|{fzlh5`dz#*NF!i1)^dAOfx!k51LhlV>b)TkGlelepY2M6ih#gTA`Ja z5qF%sJ~M_;AEXh>@_u#GHPxJGP}DbL!GwMz8w=n!0AgvQT=~BeO^~;b5e{QeeHUH2 z`7@Ldqm66YkUoFr_av~9}YbZU8bd9+5fy%GNn__=+K z-N`6R(>zY;B%NHZfjAGCeFPA0I8devL`kqORBoMol?Ky}u1-1nAITO3Tw_%&TBYC% zTho;?`H_80;L#g!z=We}T(OFt&KVNdSccs!p*BQT-KPCf5daG=@LK;y8TaKNntLFU z*-sJFN72;X=s^)cO7BZl&?+Y%0bI1Eu?;WyYT{4!ge$$fl#hEYhJf;rFQ#%P-ZISqwn%Zq$>FTF;kJup&G=>V;-JjY^<&B%J(@^C zZz*hbG7;FJBC;!Job|Wx3!GUlNLjLnhAzta3qyy4Kzo!(T`|jT)mp{W88*VZU?M(* z{TZxbF|(m=#3Qw&ouw`-q5f|p0p+XJn}6`GKmLUqqHd}hnHZ1bS20Dd1r72OPkfBQ>bB!VR=iaBAzNfn^HlPA+D60oEUM$$yz|-RkQAo zonIhpL5hBy`^*3K4_}N~s2HIIuaGuB1g6ijSe5Er7dM)aNge7_f~soF0>{E}PeCez z&Tz*jG<1fC8o4n|L#sWnn>4ITaSQ5Eq3B_m=Q?I-5fGFIQXeAu8<17X0_WzX!bCX~Ofe)P z9fOmVw(x`gs|!GGRl^pz~;9Apg7<|jmzd|t&SRd=PJb{7{u+{1){ zAF~{<6;xN3LPTk>Ic1pi)RILWb5MpRE=$^&rPK_fM(4vpmH)P(?Ldnz$Ys4pns*Gz zJ>&IX_Yy>HlMPK;a(&+8#j|zuJbm1td&zz5GmDIcV==WY5t@mqR7>l>iYuoFuH-65 zB|e=s1Clu{vr+Xo8JS{rnIlJ~rrI{4VWlb==PN)geR*+nzboro!8-0gFe zC*Cd!fz5!VVpCQqc#KqWEnvWMHw?MSXM5l@!<;%7it}7citPjs$`98VYptzfoLt%T zxBI3gZ@Dm~4YZ@7xbqYAZ6p_w(^Ec;#U9|5n|1aQMGXn*gsPml@WNB(nzS`K{m7m( zj>7bDiFou+B!6#2h3HAKlUQEpf*+=0$k##g`oX5Bi{)v^OsRiMr@|H%s6K~qAtlLq z(!NjYqMk13z@GU?$lr#0K9gN^hpnEnGH^`!t<0!!`rNg3IU#KB3$S zq80;`g}JKMO56F~@oTzqYGz#W&UfGo;5B=uTSgZ-fp51Ie?ggd?fW+w`%(m2IQ|F{ zKyYGwfwu>bdpdxT-?qaF~&1)IJ;!wHQC^OM*lVA z`+8gYR@7hqgENFrE_e^jU2;`DYbb_)h&Ku&q8s~KOnM`3>gi)*X9jzd&?!0NQ*w>34LB z*kCW+mBa{uARHJJCCy&NK)Y7FmG`tV{YP ziUeMo)%k@J*MsK*%lRn6F|WiViu;$$I}e>ySgZv6X0?>6g$sqQeav@DROMhci%j^HJ;%DF2Qci0KO~!3@w9 zy4&xL)Y!swLJ)$5$!psWfoP|79d_}+%3=?>Trg0}tvvah|tqj|2DGA$y z(UR7^4lV3&AXRV;F%ie(N2xHLl0f~RZL6dUZis>%Le$^|J#jG%Wpq8oaUr>P-}NF+ zTIhFXPH2j$RZIQGL&3i{%9os!x~o1eOQJyd8yAt%OzAFgOW7ZwKQCJUYf2YM*{r5b znL0QS793pUYp98PCL_T^?%fz`|DT+v5$R38d4l4((oBx7V zuN`+%nmp|Eyjbx<#c@)f}&c2K{ZyivM+*(8xFA27{qcfqx@DdZ=WhXI82yTPO zOZ1bCMRD#dICw<+7=LQ(d##K^(}sgZ9_E#r*X{G;H+gwzUFv^5=Y>}U8ovU*)-}Sx zqN7$s%649$Q%B4{4%+>r3d40VPVid(>w8*(Eg|cewS>3~+wIr1o(D#%sos&Ax7!f3 zdUIzwSq3C(WQDGp#YAVN$^5d&c2|Xj75FAP)ZWZ};x>YbjJt1%X}v`lqAd{= zdH2}zfOx_Lw+eo_t!KA%TkkYU`?RVCr*pJYWU9T9k{(u4;E+F`N`0x8GK=j|=V7D6 zbGn)rJ)manmWHccNu0# zT1kA@4BI}yXRe2dS4A+}SkgQO4hthBydAU;8&bCB>}`Lmib*Xd5FU0qon~LoR8kPo zY1=d34}ViSCK_tm=etL~RW3{JR_j-vlLe9Yn}lb!QLop)E_BcnR-3G6>}p{MT@6qA zEP(zdzP+Z~>n*qwo*=4rdJ)MP(}n32VhakfYF{^$qz#iL z*|wg6e2_gZt(us%#X~6C`?HL9l)|uZA{Dr!Ev1M|@jn-8S`>nCFUL?st$8%UV|noe z=pwmNIvSd#RLSH5N^|@2wCjgk+q0fX%T;vw_tQm1zRG&czEk@hW<@i2To8X1;F7)f zg!?>~By%fAkdYsk@X&p4|v<~*UANOw$E2I-zXj*ve(fm+lEzC8x7c{Gxx zTp?QG$1-lRcPpHJ*P$tb_WK{pnQKR8P>kTnD!H=b)e&rqP3(X_&pbLMHCLZTsh!vH zFVXkLG`kdBhKgu}R}nostsb2y*1<9emh$FVb9dC@xvQV(X+ca(R_@4 zbX#oYLkOeVfhp%XP_@t2lB+0@H~1Bb^?TbeaE__uMCqF&wffBO+LMa7H^l!#A3S%A z-0KkIK0&dpcn4@WlUnr>)x#&Pj|C1z%FSStqE!WVqjXM8ytzn;$nAQzz9B#&xP7Q} zIO;v`YFtlPT=O+3iFsvwl59xBU@{V?5qoOHt`X*LgNqv<%8h1$e+=gc5NfSa9Ui}k zi4uDz+c6K7H;H={lvb(-iJijF3ie26G z8Y?(9R8Zb*fw4=$mn-e-eC3$tUo_tPMaiIGpUr&_QBf%vr>04bqmHJ3+>3zc|JjQc zd`yB3#(v+xDr-q4g7|i0nL6v!)JZNuN$hDHCc`1@!w`#Qw&nb{#B@Nt@g-=is-H+)n!(fh-mDs%Hlo!7>Upw=6AXB=PvxRPGv3VVX(Itl z=8R#E82S|jcdD{DoK9-tP0Y`dSjlJAhs77yssGV8^A$_z)FPa1fu;5!Tb*;t?wK-a zlLk$Qe`d&y)$tgewJk*(Lg1$nSGl^r$CN*iXTzg?y7^+ap z$U%km4X|gZjORQztyHSMb4DaILwpV(?7}l`mojk-Is|VT{J2UR#d}z?!UD^w))zL6 zqO>{PgVJ|m8}vmxhq&SV(+I#a@GKAbIg<0z0VW^X65P`Y{;Pr*ITO6k^e+e7xM z#i_ByGb1N!>nVJC-wER=Kv73GA!nCg)P0^#tbqsLgB9Se_*Z+S6PU}D?_7}u;#()Z z7p0S&KkOV^R=z3a-YDR;Jyp%|)3dP5ikCwga0fZ^(^~yGGsC~W zVKyY$bWjkWo!Ds>%h76^X)#CGp_g&zsX6t$O0X9lh~`KC#&2sTE$b5d#*c!OoYdEZ z&^n##J_g1_<~6#g_B}v;H9aJZle5U(`QaaP zR#{pg3t&pmwW!%L1axG-?m81Q8T#G!f3x$!GdqhItaWOn8YC|GFSU;W#G&e}2gvUN zz;{~N%06cKgF+;Y;%g!tgeT+yRL~>^&xoQ}T7Di;AF*L5P4-WpfL*E&ByBpt9LDxa zqGfEiWGsj_GdqKX@0npXJavwk%5S5?BnGpZWm%PN1W?6Lsb^a>U3IHNnaKINxp>9CRGZaxk3x z=1>YbVZhGAo*@w_SFVBjy~dnE*Q%>8^oJ1LY%$iMK&nZ4cnwo8tH2Qq zw65e%w@u@gxnb*x|ARvk-|{NL($@GXY$l;6OupfpXXxbXpgL;euunTIO{~6oaPH_; zN5=Gl6~|j9VRPpg!Ik!ZGqh0 z2YgCLtWVqgz)^Kaa{prJ80)pfH`kj2@`R4R0| zIcCR18)`fKI>L%$+Uo>hl)=^;USslT0Aka`w{WcEnSf8gjDR9c|iq1XE0(mT4 z)fdu_`<5Qsil%!H1j_e_8W;13&{=4U9H!rCmOI+AHsP9fiv^bW8e`&2)9Ia)T=b4| z3D6ZL4eEZW$Hg^iImbMZK-0Z7205h{ogplrOl@R!n@@qs^GHF${rhW?LxVYCYp)*Z zso~h`ifb>$@BYR2+g_{;CUUC^jaOPv?It^P6VJqA->sR6N@icXlqBs=TgwKJiR;W* z^IM)36M<9Mai+Z_SuhJM%7f0>Z5ux`DNz6)Dsc%qH=vdHz?5f1IN{(d-ZpqDbe)tV zd(gq`L;H;@jo-vN82a+ooy58P$HAk6K$(-=IXj{eDN&$0;2roXFVack9@+B1jklwh zb4QfyK)TvVsI3O+UuoyJQ~DX$2wAKAtT|C9ReBx<*+-RiFMcYPa}9&-L80oN_6dxa z(nkUhGdh$j%wJu1V7O;Buu}qLpcVMiqv8S#i|nE#x z2f}=BSg$bbWhPn^VGG_K&3aC*j>Qyba%1?gUa)&GdcM_%2|>+2oP-%DL(vR^B^@dJtVnqMFaMfE%V?V$37} literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/corpus/f50a6d57a46d30184aa294af5b252ab9701af7c9-2 b/tests/fuzzers/rangeproof/corpus/f50a6d57a46d30184aa294af5b252ab9701af7c9-2 new file mode 100644 index 0000000000000000000000000000000000000000..af96210f204e19513ebde336e6db31430d771a7c GIT binary patch literal 1748 zcmV;_1}ph}I}WZ3f(26`!PcW;44V+4dFJ3 zqW*hpUjjP~hlXfc+zo+v^QgS@wGqHQDQ-Y57 z^2)G(Pm8QI$Uw$1>w){{4PY}SNj#{&OxwD4Krw~ezWCQ^35|BTx7)ZZtwo{kZ8PWb z3?9>8M&zy*7Hin@RQB)HKbcH_Cy*b~(e%}kMH%0&G`(bo^1wP*F9 zl>h~CbjzDh8QLcT%)4Q0%M>_LZ7!6jg;6Q9)tXk_+lDSss6Ux5*M+X+DClls{CQ%4 z*i}DSr11=0(V0_~xp;4zalg_6A%GSdr$REyu?ejqUiU@m6=Oxi*h{VPSWCh-*l>Jy zW(mwnh4ru+D!$W;W{n4mj&cQwm&C32C~jfi(W2xtn-00B5Rb0X1r}B+V%Mg>3G<{! zY;rq@!<7M>f@)Lenc~em?!EBi=X`mQ(cGuOAZYeNGYj+qgoI4P#B~9I*brIKB{CyJ z*F#zNA|hw$HyN-oh;*P~#Z?=_l>+#NP^N4USxF(D$wd4VwqQH##r$>GcuP9GX$_c= zH4&-oOHD%7eCjdb25Ln{p#5%?+n1+1NTwbGjYU0+IS@j#DJCtxQDK->uB;o8mi~Rv zG96-CF#0&nS*uQ2cD5py0)Z<#DK$C?P_qA+K_ggc*8 zRzPtfV}(^(ACm3fOHIEORPkN#xKI1#D%#~nD;zFpp$Jxx6++@HMDNkDfPALj`JYcn zWyDcp>C?2%RB~9`jSVOI5;y-AHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV z1$K3m7=T<0^t*d3MDt$!W}wi{FHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*; z!K<`d6ZtwGc*ZAJMxJ6G$DfFL?AgIE*GQ<#%Ulru|gwfOn}OXVzyoj2iNr z?+pv_-*76=4Xd5$f8NX5Ixu-F5tua4DIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_O zXA-LhPB1uQ)IdBC-x;ax$@r%zPR>+S`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ip zxex64`k1S2#aWJ!Zd2qrHqN`ZxC_%Qx=y_KiFDAQZ<+iPs{e_8t;AE@;!>o^X5&Sw zO*9<7@JcdZ;CKCi0&)iuTeS)v_+sF$w%Hm%oY<^d*iGbiVn`?as5#NJqtRdE+#_J^rwW_uS-XrCs74@PTEkN@>?}uS6^unR83->0iFgyuKk`_vb*0xDL$UT@Q zdt_^~K!6!K&uv$98H@mgWW{Ng=n5!WIz!K7w%l9}MxiD`=cw9b6w-ILNkwxRM}w=M zK9K4+TLtXLTZbal*QxY3U_+wY<=R5|FHTHlnjgJTB53U|!_aInFeD&8vV?ekI?4FH z0~Ea-w3=Z(N|Cx5fD=hSP*6K>h1O}JvR)Wb7Nqb_LhusjNB{*2v#D#Cr4HtHcG2Q$ zC73w)VI(w|^St1Q-nGK>FY`#aqTvdJlHZFe^imZZ0cYE&j}C`+)~kwh3=jzC zo^M975;N|U2{C4jW*h^+zHW@GyOrOB$uHu^sGgR{at;HhZ>KGE24;`fBItGCa|`#K zEirg4Abhns4?hdxdtiYg0XYkEI1tfyIU=vVuJ7grl*SUfuq z?f6g&1SQ-g78xigT0z@|i^hGY{NK1VH-4@U!L%s&^4QtoUBO}`vl+8%BX5;s@PV>o z_TY)faiJeZ#rCz6Q?lz-mhs>@WiM}C38le#3s=8QA7GmE`V2*$E5<%s`!w z{h(&U3fOb?*v%&h1=k3Qzg-FWZ_2iseeNiC0|rlS)?A>WwP1*S z>^+~04F@>QaB;7Ku_Xq--ZKONJhE~?FMuhJSGtt}bJEV6DPVv^mTf+(DeA`_W-=QX z^Bm&k8$pFdRRh8Wo1oz$THK(QTc>Lr&P0jDD+_Fl{=U-$v*=vWThFbta$09IhCU7!M(5?B^ z3Us_9+D}AEj|m;j*WsC$V=#b%$w;fvlIF^OqVR`jZj*RfA>lZAC@sQES&D z3!dV0*f|MYJ;>vbzrB$U8K*j=+MD%(0WGy_3_dg@AV|!YR!KCipIvZ{qflyBn(9ah z{c<5Nj6>%UA(IZ1%yKouHPbeCH<8Ue27CvN9(bBjx4@$zq;D=G5!4k)1OmHZq#eujX*de*R5#GdIdurO8MNMN9(PLR zXyKCBji)>sE63>;2UA3!a;KDP12qODfB~%C5SO>KYWWV(*wTm7^fv>s-s`=~+`Fm% z28QR59@=Q7cL`21h^h?Y!f{)I#=c{5pLg)!33;*f<$O$okpA%5em&(Y(YkaRp--q9 z=(P#evU>@T&9rzjoxOnMl{``s3yJ|Ur=8mK4|pRPjj|%r*TVuY)pArezYv{G+E~iq zyQA2J`dhOXJZvN*U6L0z{>+!2?7OYznH(BS=?BWBJ;Jq?^OJ5*$Kl#L&)5$<9;8hw zT!US}9^Ca%nSd|P-bfd=5F-Z*Oi`w&o?&=OK_9ZIxI@kFq_;$UI}WZ3f(26`!PcW; z44V+4dFJ3qW*hpUjjP~hlXfc+zo+v^QgS@wGqHQDQ-Y57^2)G(Pm8QI$Uw$1>w){{4PY}S zNj#{&OxwD4Krw~ezWCQ^35|BTx7)ZZtwo{kZ8PWb3?9>8M&zy*7H zin@RQB)HKbcH_Cy*b~(e%}kMH%0&G`(bo^1wP*F9l>h~CbjzDh8QLcT%)4Q0%M>_L zZ7!6jg;6Q9)tXk_+lDSss6Ux5*M+X+DClls{CQ%4*i}DSr11=0(V0_~xp;4zalg_6 zA%GSdr$REyu?ejqUiU@m6=Oxi*h{VPSWCh-*l>JyW(mwnh4ru+D!$W;W{n4mj&cQw zm&C32C~jfi(W2xtn-00B5Rb0X1r}B+V%Mg>3G<{!Y;rq@!<7M>f@)Lenc~em?!EBi z=X`mQ(cGuOAZYeNGYj+qgoI4P#B~9I*brIKB{CyJ*F#zNA|hw$HyN-oh;*P~#Z?=_ zl>+#NP^N4USxF(D$wd4VwqQH##r$>GcuP9GX$_c=H4&-oOHD%7eCjdb25Ln{p#5%? z+n1+1NTwbGjYU0+IS@j#DJCtxQDK->uB;o8mi~RvG96-CF#0&nS*uQ2cD5py0)Z<# zDK$C?P_qA+K_ggc*8RzPtfV}(^(ACm3fOHIEORPkN# zxKI1#D%#~nD;zFpp$Jxx6++@HMDNkDfPALj`JYcnWyDcp>C?2%RB~9`jSVOI5;y-A zHy+}7EBw;y`ZV?m@x1ebxo7q&L2P%Y_;@1fxh1OV1$K3m7=T<0^t*d3MDt$!W}wi{ zFHk|hPp(%R59v5vY5)K-riNy`*(h5y{|td7)Cm*;!K<`d6ZtwGc*ZAJMxJ6G$DfFL z?AgIE*GQ<#%Ulru|gwfOn}OXVzyoj2iNr?+pv_-*76=4Xd5$f8NX5Ixu-F z5tua4DIX~Ys59$B!01G5*{$k|Rkp300({%p4mf_OXA-LhPB1uQ)IdBC-x;ax$@r%z zPR>+S`yjr3Zfl1=XV)p}p};v7i;Gy~F$v@M2O&Ipxex64`k1S2#aWJ!Zd2qrHqN`Z zxC_%Qx=y_KiFDAQZ<+iPs{e_8t;AE@;!>o^X5&SwO*9<7@JcdZ;CKCi0&)iuTeS)v z_+sF$w%Hm%oY<^d*iGbiVn`?as5#NJqtRdE+#_J^rwW_uS-XrCs74@PT zEkN@>?}uS6^unR83->0iFgyuKk`_vb*0xDL$UT@Qdt_^~K!6!K&uv$98H@mgWW{Ng z=n5!WIz!K7w%l9}MxiD`=cw9b6w-ILNkwxRM}w=MK9K4+TLtXLTZbal*QxY3U_+wY z<=R5|FHTHlnjgJTB53U|!_aInFeD&8vV?ekI?4FH0~Ea-w3=Z(N|Cx5fD=hSP*6K> zh1O}JvR)Wb7Nqb_LhusjNB{*2v#D#Cr4HtHcG2Q$C73w)VI(w|^St1Q-nGK> zFY`#aqTvdJlHZFe^imZZ0cYE&j}C`+)~kwh3=jzCo^M975;N|U2{C4jW*h@~z`kyb ztGkuogvl@B$Eco`$Z`$?r*Ee%bOvUR*COb3;ByQ2oh>nVEg*co9$R@sQhOimGcamW zVd1}%fYab_n+^;} zkkpy)eVyzCR}YElc>38k;H9GZ6UUmH_e)YqpCbW6=MF>1ARX~>5NbLRwPbdUKA^&U zUVdTNX^$0`f@&$AM*Hv7IYKriDS3c`nV02^m_Q?Gc*yDxgf5T%djq`Z`+H`zzV#)JJg!o;;)!~(5=?i-64 zREC3p&G^qtGPdzIgkC#sks8jMvw+=mry>cDQ{4}*h+3sj zYaD5c!dP=dp3-7afD2Z7s$Zv1@c8w__{Y`KISKv2ymTrkmXoyTntuXOV&2q~Ww=${ z1w=0&T#`g4+G$EJLWAxLH-^&RTR?N24}0gYr@33qIyV8On2TsOw6bD7kt^T$x^ZPl zaYmQ?7`c?Do>fOd6eb-XTZ0!u4@89=0iu$^xcsol?RS1@F_aV3YisIV8;-QUf+N64D3yR~tn+Bel9ApbAbn7(I*CfEmF=O@d}Z4L zp~otzy=*ad1A_{lsSrEcNimXH6^mz{b2`2<1dq2lvtc>Un9iAOr#t3Mlaf$&Qp9Fw zRje`nIC_Ja&v`^95C@QOeP*GQ`H(VlxYQX)fITYQu=Hc1%+VO81xg$HW`8lx3iFem ztqG)TUk!tGVt_np99G6bC~ATak5^}%jgoi3z~t|ZakSs^G6C&YWo0xj;jm{qm8Tz< zDk+8HbA|9|^6I@WZItJc7hr=*i6=61cVJ6~!~>SKm`#C2yV7WlNWIA5Ols<}M57)T`o+X*zp_t-e+#o3&sT%r_x$E2 zgFm)=G@8|FB%`;6rs(O73l9=1bFwB~ZrQ!S{6*7G@AooRj@yOC3|Z&g>>NyG2v_4{ z^wB4((gf{CJd_e43+ByRf7!au=~K&VZv{2Bi7xK3J9s74hQ*BIB>iqpM9ge{2Q20b z%k4DSwl)P6t?Boy;f06SQem(Pu1$;v-6L_M@yi}<6003zgHhsPQZW2UShJWJmG6S8 zn~U$ue7=i@)Q6A6YPj8I)$^ux_NMk@_IIj?5PTY<=&F=J=4r8U8aY-F?)lRrL%HP|K$X((s{3)<+<-{S5 z7Z-cS#=OL)oXW za17j0Cs%iB4ZvG-X;9Ay4B_QZ7~8pIGt#pwPg6S~-CSqoS!Zd7-!f&g%ny(eL=GAk zZqy*w{v<;4&9P2dd zHJX%hGG1}&+5Q7%4G7t7!xsj-5_p%jb2$@!8>o*YCcved#b=*s``C0o7@$Z@aXa?s_(@HAe^7{rb!tTEIakbW^;=?ePp%c z6)nu)60O$q{z4CRk?-*GbMfMXRW*O9!JRTxfAW7-!mg@+?Rt6Mt6TY1Pr%Q!mq$q7 zHBh~51Mt_HIu`OWM+Qbhwn_XaqV^ADr^ar9Q_Q&(EhOg-9sf+R6mo@VZQM7ZT)udf zNGsr}pvb8P1YLd?JzLlDoL;)vkevL(F%`psWgU9#XpY$1`*^st+6k40H{e?iYrT-N z+GaVilE2=Js%eh0zTG~{Vbdz9;~t?5Rk#=~pN>w4OLQ=G!!G{hb>t|)`lbgg`jw&f ziW(N~q13CiLssfY&%5X>cB&atCEeKF(-`GW%o^rq15pmrFkoU_KgzdM%KIJ4J zskN4j(xIp3gy<;LIqMDV&3D+JV*GU=#PQ+=+grtB2-F`1i1kWd>QsAo=-!Y9rCn!Z zJme;5S7Ed83BZ@S$W4$t{6?2|SdxE*1iaUn7Uqf><@#3ATa*lcCvq_We1@mbOJbH? zZ}oNDxUTv_4~W&>11h31fc`QYu!K#jCE%IQf!>d52j*O4oj5l;D9u#=fsR+H4&V^F zRj=x44b4^rk(qw_hH{}8W|y(~KJ-3f^Jt!mo2ygVQ{>>%piTKOr@(FlNQNF;yI1vw zc=i%GGb?iu+;_L8#MaEq2Oy6yIuDPn^AbCTX_vsX?0iIYUU~VeHlv!3>o63&fVmgY z)D1$cISVTl0CW5FB5{6CF5prVl6qjN&Eq~CYCb@QMPCb|(gEFNRFx!k-IRe+3;30( zeq>A;@o3Cz1zq|lku;P-3O0%votA%v>F>Ybd8?gzY?2TyqLnj&`MAZ3!c11g#k zypaiPe|R6GBtih!cT69_l(UF9P+D^MuDxYOLit%E2quhUyL| zD4|7V`Op92F}HdYnS*G&?L^On1*9gzn@yd&k3h>#YDzR@q3UH$^7aWNtw(%2wWAtA z_)})Fb1dc#pbRSShcM?S^@mJp)2f(e^Sfzb0ppjKdvgt;?o=xu)Qr=gdVjqKHoa|i z+o&Fw)nq^43=KIl>Avlx7A|7-Wy>Y`IwY?6FfnQco75c7OA(PAXO;f&??NZ`@o9O zIBuW4rds?9&Ao+(n(;i9Dgapi`xSBner0hR(``!S zkdYrS#sG!VM|h}t^V4#Prc^vOQHxT)<9C|M#gA{O>5ARI+2nAh9)J&^R>QrE3U=+b z+KVCf2QMyTj3YyCZ@@?wMJ@GE={#Z7N|bxLi_-j*^u5d-cWyEgKtiLHu==Yw8)3Ik za$Jk^$G&C=1ReC)v8Mv#nj_jLsf~&1U1WacuuTtbb*xMKT{fIw&07x!HtC?cl4X4| z*#z&(y32!BdR|iE{}5?_AcJ*IJ0ve_<2w9`%ei=M6}p6s-DUnpV6NT`j2q5+WR?&x zHHzO{9lYuP?_EtUjCljgmGO-FaylaH{+y1-T5-fwYbfAAox+dD_0|O4biyoAwhxS6 zP>dO5t@d;0i8xR(j(1^;okB%mw;mxVKMxIQgIj@Y8s$n2MfYHks5&z6DQ2YazGm;5 z4wAZ$-6e}(>_j>ogk5g+KyUd;;++Y9(>LTmNMoiR>H=FkPWqk_Eo;05&ky&sQ=ICU zFCZY=F}wMm6gprVK12cbq=_el%CO!PN)(ITMdjO1qzJY~=LlNW_N&`J?Uf~|UF_Zl zVt8352>QI-7;SSD_C9z_XiN_j)e`PftrD&?ZP z@V4;=g-jze2-r|vz9d%hMFaQ=k-JW;^)y^9Jb9)k=!mc+9{sit!y_*DX#M-?o#VdQfrx}-fM(dK9c@736^aFgkhaRJiL_ymzLD)ccZaHsudOIc+-AT5 z;RaL`ibP4kg^<9aIJIW$nR*54yX}>W0yrS07klF;sa7dY;4#GIU!7C#PYu014+t_+ zwyMu5mowj63;htd*Gdt?kCS%H5A(+UyYi`$j>3mMRnDf*%2*9&46=!4$6xBfCJ%cW zqC;`e7sOS2{m$~vVXN$|iVqZ3Q7kqiD3 z6m`yy(bH=L=9QH$*Aw1L@d&7=wi`K=ecPr1+$S`}+IVf_6NU>w(S;_%dKR7vvQ(EQe!)l7g% zVE&IIW73d&gIP&d3nqsa)BIYg64D>dLrNRevKwCcB>2o*EvCVxyoPFTlU zg=}!Rj>;=9_~{1;M$ckwOX^eogm8H(VEtWV>tK;WIp}u!QOb@&Vc$S@HMXcS6Jg6*nD=f$dre4W zN!sO4dLSKovPQMSsF32_TOY_?$qAXT0&7-D1pX^Pfy@<1Wws>#Zx!iO9`~MhWnIC! ztBN~_)jEX3>lv_1DPBqJfkrI!9@bw_qUb}Ap>%uCAtsHW{*L!-%j!f#*6dT*4p4P- zb>zHpl~3alQ?my}V5Sm{X;{-8fQJtirtVBQlqK?Ub#={oJ!3z!`kFRYw#yyuXPu-> zYB&caBA4tOM2MFu=Iq+_m|b^-Y+lS%fITF(}V8zug^4T+hfvH&AmMMY(^|-ls*h zcR7JqqMdK_H|K5w49V4V778%m$L*;ifDeFZbu3O7@;`!=f1(OXZjJl=*n-jVN+>4C ztQC;~O9o>`zT&C%IKqnU>rYp3k!cxj#lAE7 z!32*2-6-Hu?c9VgI}0X}gTbufGIS0~VbP)lpn@N?u~w}T3`1kUPf-UkzZ|haN?}w( zO*_RnEhICnFstc|eOJoXqr`q_VLXP8{4Z#GYesVxlJtC3Yiv4;PApybd$)M{P!bg2 z_7F!V$h`dHXi79N^%F4cd{JdYiXzza)ZuBx1h{6V23!@I95QX<#?^KLX4FfX05z^& zm}k2BN5s1$j=HI`7#mmq95Yw9(C^1aM&e)UJ?CeR###gF3&GifA*Ymn z&H`y#&?2}Ogy=4oT}|AyR5o@6)bZ2~*{PCPgA8VYKP7n?~9CaX{2-q+8m z!}gL)*}>;pzpAu~k&Ys)-AA4iw@udhGJ&oF#tPgnpB*q<9Du<83(s^PQmxWUxVs3o zH*=wR-U4u;suiZrLn6a3?g1ETUJ&z~f}&V+N+?uR;*BABxa{y`4kc?1zw=Lmqc(Gh zNqv$0gaI}5{l=1I{Kv{SEJBvyl2PEv25k$W+M9=m30V9v6Ccq?6nv~|pyk#vF|d`| zC_`DTxyHv2=xAo{B2Rcj`NwX92MQp8v^uMNp>R1s1Ky%*Ono?`!slgt4jYYm=$2~j zxFYYi-+d-ii%UjM6>++slbbrxEoK|bg&=5-3~h$7&r;7bP>Xg>KQjvc5De#p5H1nD zisf_jez3g}N`Q|w;;=b-aCLtR#CT`sY|^+1N9&C04Ux(SZiF z<*hfZ`@C&9H}m|u8GCkrKyE(*C^ZsgO&R>DFF3g0I3U~-!9$PS5L+9&; z0u+At>PCKKf`vPh9-sR?LlVanKu_syvFkz;W3;6b3?OwJ3ZU9l;IrcYQ^uZvBTbTG zfIU^AWV=ER(C0HZ!pVUZw2#0F5OK(0zHtpbprYvrGZlD$H(5Q}dU05*yfsPvX z_~Ml#afbrpf=v~%l@iU3Bhk#Xz}%7=5m&wQ2Nz)G(`1aK-PX-dQJivAR*XxPJ^Nzf z>Q4?fuoL2Y*P+Xku#Pd(s$~RC*$?3)81iZ*-w;%;(?vWG;ea4CL zF@m{@<(!Rlz6j9TRfZ;a_+9*?O@%}DO9l6-l%74bp$jLq+zU45ajg^LBQuoPD^=R@9c3?Sa=quQDb*Z&`5 zLjib~+ct3j=sVv+qyh}6!DhYPo-O}RYUkvtY@(nnNIBR47Wsg z9frcuIr|7U7%rD21=55%T#6gwxw9h4zoULB6Q$phDB1~T2gnGmXNMc?AfyGvYj>ww z66qJ>z>`j6K61{GhhwvNrva8V0>x|nV$J8LmhB5g?;AZIP%AElrM$>q9~c? zhbrdm$=NH1Fc&A1=@>aBB-;;?9`5oXK!&E7=?8FK?4#lmn&`=m=({M^uq#_i?e9n3 z65Le92%T`$kv=hlM7q!Pn%C16*Y5PN1{B|71uhxsMDUJ*<7v41(F?cymhy!i z+&v3eQcCvd(*#$+>2!}nOKATK%yHT0Xb;kXjkQPa>BrM`Ck<4mCkN=Y`#_E!Cw7L5Ojm%f09hga_kTG

tqh-JF4rtl^ z#Vf-l_B@W!l!o}pHSb(%QvZ8f*rEyo1lTq}wmVL6>-RC6?e1WdfXS|uqbJ@7ej@gzPEB9^1t;VP_1T|!dc6w+R zLRxb|i95|KBHO(HoV3Q6;%)6_M9?cHuP^>hx7hIbmJ6_d;;!{kR#>y#V0?Cf>yrq}*(`wnQAks76(1 zdL1kjJ5``P#hy3Fer0V@HL!GI0y_n9|0ObLr!FuH!khzPQMNtP|H8wZ?b!8Uv>1ro^ocr-r{ zl~mj%Ys4ua*1vTX2{>hQbRv)e$O=Wtm^pD?G)rjgam6-#ak%hT-B) z(XoN$rLx|*jaODvxsyf<+LVzjDX4_g%ZfuK?l{526xfj?<-aRuT*U`)|ZB?$bXlK@@b< zD(8)iEiJB|{fzlh5`dz#*NF!i1)^dAOfx!k51LhlV>b)TkGlelepY2M6ih#gTA`Ja z5qF%sJ~M_;AEXh>@_u#GHPxJGP}DbL!GwMz8w=n!0AgvQT=~BeO^~;b5e{QeeHUH2 z`7@Ldqm66YkUoFr_av~9}YbZU8bd9+5fy%GNn__=+K z-N`6R(>zY;B%NHZfjAGCeFPA0I8devL`kqORBoMol?Ky}u1-1nAITO3Tw_%&TBYC% zTho;?`H_80;L#g!z=We}T(OFt&KVNdSccs!p*BQT-KPCf5daG=@LK;y8TaKNntLFU z*-sJFN72;X=s^)cO7BZl&?+Y%0bI1Eu?;WyYT{4!ge$$fl#hEYhJf;rFQ#%P-ZISqwn%Zq$>FTF;kJup&G=>V;-JjY^<&B%J(@^C zZz*hbG7;FJBC;!Job|Wx3!GUlNLjLnhAzta3qyy4Kzo!(T`|jT)mp{W88*VZU?M(* z{TZxbF|(m=#3Qw&ouw`-q5f|p0p+XJn}6`GKmLUqqHd}hnHZ1bS20Dd1r72OPkfBQ>bB!VR=iaBAzNfn^HlPA+D60oEUM$$yz|-RkQAo zonIhpL5hBy`^*3K4_}N~s2HIIuaGuB1g6ijSe5Er7dM)aNge7_f~soF0>{E}PeCez z&Tz*jG<1fC8o4n|L#sWnn>4ITaSQ5Eq3B_m=Q?I-5fGFIQXeAu8<17X0_WzX!bCX~Ofe)P z9fOmVw(x`gs|!GGRl^pz~;9Apg7<|jmzd|t&SRd=PJb{7{u+{1){ zAF~{<6;xN3LPTk>Ic1pi)RILWb5MpRE=$^&rPK_fM(4vpmH)P(?Ldnz$Ys4pns*Gz zJ>&IX_Yy>HlMPK;a(&+8#j|zuJbm1td&zz5GmDIcV==WY5t@mqR7>l>iYuoFuH-65 zB|e=s1Clu{vr+Xo8JS{rnIlJ~rrI{4VWlb==PN)geR*+nzboro!8-0gFe zC*Cd!fz5!VVpCQqc#KqWEnvWMHw?MSXM5l@!<;%7it}7citPjs$`98VYptzfoLt%T zxBI3gZ@Dm~4YZ@7xbqYAZ6p_w(^Ec;#U9|5n|1aQMGXn*gsPml@WNB(nzS`K{m7m( zj>7bDiFou+B!6#2h3HAKlUQEpf*+=0$k##g`oX5Bi{)v^OsRiMr@|H%s6K~qAtlLq z(!NjYqMk13z@GU?$lr#0K9gN^hpnEnGH^`!t<0!!`rNg3IU#KB3$S zq80;`g}JKMO56F~@oTzqYGz#W&UfGo;5B=uTSgZ-fp51Ie?ggd?fW+w`%(m2IQ|F{ zKyYGwfwu>bdpdxT-?qaF~&1)IJ;!wHQC^OM*lVA z`+8gYR@7hqgENFrE_e^jU2;`DYbb_)h&Ku&q8s~KOnM`3>gi)*X9jzd&?!0NQ*w>34LB z*kCW+mBa{uARHJJCCy&NK)Y7FmG`tV{YP ziUeMo)%k@J*MsK*%lRn6F|WiViu;$$I}e>ySgZv6X0?>6g$sqQeav@DROMhci%j^HJ;%DF2Qci0KO~!3@w9 zy4&xL)Y!swLJ)$5$!psWfoP|79d_}+%3=?>Trg0}tvvah|tqj|2DGA$y z(UR7^4lV3&AXRV;F%ie(N2xHLl0f~RZL6dUZis>%Le$^|J#jG%Wpq8oaUr>P-}NF+ zTIhFXPH2j$RZIQGL&3i{%9os!x~o1eOQJyd8yAt%OzAFgOW7ZwKQCJUYf2YM*{r5b znL0QS793pUYp98PCL_T^?%fz`|DT+v5$R38d4l4((oBx7V zuN`+%nmp|Eyjbx<#c@)f}&c2K{ZyivM+*(8xFA27{qcfqx@DdZ=WhXI82yTPO zOZ1bCMRD#dICw<+7=LQ(d##K^(}sgZ9_E#r*X{G;H+gwzUFv^5=Y>}U8ovU*)-}Sx zqN7$s%649$Q%B4{4%+>r3d40VPVid(>w8*(Eg|cewS>3~+wIr1o(D#%sos&Ax7!f3 zdUIzwSq3C(WQDGp#YAVN$^5d&c2|Xj75FAP)ZWZ};x>YbjJt1%X}v`lqAd{= zdH2}zfOx_Lw+eo_t!KA%TkkYU`?RVCr*pJYWU9T9k{(u4;E+F`N`0x8GK=j|=V7D6 zbGn)rJ)manmWHccNu0# zT1kA@4BI}yXRe2dS4A+}SkgQO4hthBydAU;8&bCB>}`Lmib*Xd5FU0qon~LoR8kPo zY1=d34}ViSCK_tm=etL~RW3{JR_j-vlLe9Yn}lb!QLop)E_BcnR-3G6>}p{MT@6qA zEP(zdzP+Z~>n*qwo*=4rdJ)MP(}n32VhakfYF{^$qz#iL z*|wg6e2_gZt(us%#X~6C`?HL9l)|uZA{Dr!Ev1M|@jn-8S`>nCFUL?st$8%UV|noe z=pwmNIvSd#RLSH5N^|@2wCjgk+q0fX%T;vw_tQm1zRG&czEk@hW<@i2To8X1;F7)f zg!?>~By%fAkdYsk@X&p4|v<~*UANOw$E2I-zXj*ve(fm+lEzC8x7c{Gxx zTp?QG$1-lRcPpHJ*P$tb_WK{pnQKR8P>kTnD!H=b)e&rqP3(X_&pbLMHCLZTsh!vH zFVXkLG`kdBhKgu}R}nostsb2y*1<9emh$FVb9dC@xvQV(X+ca(R_@4 zbX#oYLkOeVfhp%XP_@t2lB+0@H~1Bb^?TbeaE__uMCqF&wffBO+LMa7H^l!#A3S%A z-0KkIK0&dpcn4@WlUnr>)x#&Pj|C1z%FSStqE!WVqjXM8ytzn;$nAQzz9B#&xP7Q} zIO;v`YFtlPT=O+3iFsvwl59xBU@{V?5qoOHt`X*LgNqv<%8h1$e+=gc5NfSa9Ui}k zi4uDz+c6K7H;H={lvb(-iJijF3ie26G z8Y?(9R8Zb*fw4=$mn-e-eC3$tUo_tPMaiIGpUr&_QBf%vr>04bqmHJ3+>3zc|JjQc zd`yB3#(v+xDr-q4g7|i0nL6v!)JZNuN$hDHCc`1@!w`#Qw&nb{#Bs(moEs@g-=is-H+)n!(fh-mDs%Hlo!7>Upw=6AXB=PvxRPGv3VVX(Itl z=8R#E82S|jcdD{DoK9-tP0Y`dSjlJAhs77yssGV8^A$_z)FPa1fu;5!Tb*;t?wK-a zlLk$Qe`d&y)$tgewJk*(Lg1$nSGl^r$CN*iXTzg?y7^+ap z$U%km4X|gZjORQztyHSMb4DaILwpV(?7}l`mojk-Is|VT{J2UR#d}z?!UD^w))zL6 zqO>{PgVJ|m8}vmxhq&SV(+I#a@GKAbIg<0z0VW^X65P`Y{;Pr*ITO6k^e+e7xM z#i_ByGb1N!>nVJC-wER=Kv73GA!nCg)P0^#tbqsLgB9Se_*Z+S6PU}D?_7}u;#()Z z7p0S&KkOV^R=z3a-YDR;Jyp%|)3dP5ikCwga0fZ^(^~yGGsC~W zVKyY$bWjkWo!Ds>%h76^X)#CGp_g&zsX6t$O0X9lh~`KC#&2sTE$b5d#*c!OoYdEZ z&^n##J_g1_<~6#g_B}v;H9aJZle5U(`QaaP zR#{pg3t&pmwW!%L1axG-?m81Q8T#G!f3x$!GdqhItaWOn8YC|GFSU;W#G&e}2gvUN zz;{~N%06cKgF+;Y;%g!tgeT+yRL~>^&xoQ}T7Di;AF*L5P4-WpfL*E&ByBpt9LDxa zqGfEiWGsj_GdqKX@0npXJavwk%5S5?BnGpZWm%PN1W?6Lsb^a>U3IHNnaKINxp>9CRGZaxk3x z=1>YbVZhGAo*@w_SFVBjy~dnE*Q%>8^oJ1LY%$iMK&nZ4cnwo8tH2Qq zw65e%w@u@gxnb*x|ARvk-|{NL($@GXY$l;6OupfpXXxbXpgL;euunTIO{~6oaPH_; zN5=Gl6~|j9VRPpg!Ik!ZGqh0 z2YgCLtWVqgz)^Kaa{prJ80)pfH`kj2@`R4R0| zIcCR18)`fKI>L%$+Uo>hl)=^;USslT0Aka`w{WcEnSf8gjDR9c|iq1XE0(mT4 z)fdu_`<5Qsil%!H1j_e_8W;13&{=4U9H!rCmOI+AHsP9fiv^bW8e`&2)9Ia)T=b4| z3D6ZL4eEZW$Hg^iImbMZK-0Z7205h{ogplrOl@R!n@@qs^GHF${rhW?LxVYCYp)*Z zso~h`ifb>$@BYR2+g_{;CUUC^jaOPv?It^P6VJqA->sR6N@icXlqBs=TgwKJiR;W* z^IM)36M<9Mai+Z_SuhJM%7f0>Z5ux`DNz6)Dsc%qH=vdHz?5f1IN{(d-ZpqDbe)tV zd(gq`L;H;@jo-vN82a+ooy58P$HAk6K$(-=IXj{eDN&$0;2roXFVack9@+B1jklwh zb4QfyK)TvVsI3O+UuoyJQ~DX$2wAKAtT|C9ReBx<*+-RiFMcYPa}9&-L80oN_6dxa z(nkUhGdh$j%wJu1V7O;Buu}qLpcVMiqv8S#i|nE#x z2f}=BSg$bbWhPn^VGG_K&3aC*j>Qyba%1?gUa)&GdcM_%2|>+2oP-%DL(vR^B^@dJtVnqMFaMfE%X5CCvB$ literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rangeproof/debug/main.go b/tests/fuzzers/rangeproof/debug/main.go new file mode 100644 index 0000000000..a81c69fea5 --- /dev/null +++ b/tests/fuzzers/rangeproof/debug/main.go @@ -0,0 +1,41 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/tests/fuzzers/rangeproof" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: debug \n") + fmt.Fprintf(os.Stderr, "Example\n") + fmt.Fprintf(os.Stderr, " $ debug ../crashers/4bbef6857c733a87ecf6fd8b9e7238f65eb9862a\n") + os.Exit(1) + } + crasher := os.Args[1] + data, err := ioutil.ReadFile(crasher) + if err != nil { + fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) + os.Exit(1) + } + rangeproof.Fuzz(data) +} diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go new file mode 100644 index 0000000000..b82a380723 --- /dev/null +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -0,0 +1,218 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rangeproof + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "sort" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/trie" +) + +type kv struct { + k, v []byte + t bool +} + +type entrySlice []*kv + +func (p entrySlice) Len() int { return len(p) } +func (p entrySlice) Less(i, j int) bool { return bytes.Compare(p[i].k, p[j].k) < 0 } +func (p entrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type fuzzer struct { + input io.Reader + exhausted bool +} + +func (f *fuzzer) randBytes(n int) []byte { + r := make([]byte, n) + if _, err := f.input.Read(r); err != nil { + f.exhausted = true + } + return r +} + +func (f *fuzzer) readInt() uint64 { + var x uint64 + if err := binary.Read(f.input, binary.LittleEndian, &x); err != nil { + f.exhausted = true + } + return x +} + +func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { + + trie := new(trie.Trie) + vals := make(map[string]*kv) + size := f.readInt() + // Fill it with some fluff + for i := byte(0); i < byte(size); i++ { + value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} + value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false} + trie.Update(value.k, value.v) + trie.Update(value2.k, value2.v) + vals[string(value.k)] = value + vals[string(value2.k)] = value2 + } + if f.exhausted { + return nil, nil + } + // And now fill with some random + for i := 0; i < n; i++ { + k := f.randBytes(32) + v := f.randBytes(20) + value := &kv{k, v, false} + trie.Update(k, v) + vals[string(k)] = value + if f.exhausted { + return nil, nil + } + } + return trie, vals +} + +func (f *fuzzer) fuzz() int { + maxSize := 200 + tr, vals := f.randomTrie(1 + int(f.readInt())%maxSize) + if f.exhausted { + return 0 // input too short + } + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + if len(entries) <= 1 { + return 0 + } + sort.Sort(entries) + + var ok = 0 + for { + start := int(f.readInt() % uint64(len(entries))) + end := 1 + int(f.readInt()%uint64(len(entries)-1)) + testcase := int(f.readInt() % uint64(6)) + index := int(f.readInt() & 0xFFFFFFFF) + index2 := int(f.readInt() & 0xFFFFFFFF) + if f.exhausted { + break + } + proof := memorydb.New() + if err := tr.Prove(entries[start].k, 0, proof); err != nil { + panic(fmt.Sprintf("Failed to prove the first node %v", err)) + } + if err := tr.Prove(entries[end-1].k, 0, proof); err != nil { + panic(fmt.Sprintf("Failed to prove the last node %v", err)) + } + var keys [][]byte + var vals [][]byte + for i := start; i < end; i++ { + keys = append(keys, entries[i].k) + vals = append(vals, entries[i].v) + } + if len(keys) == 0 { + return 0 + } + var first, last = keys[0], keys[len(keys)-1] + testcase %= 6 + switch testcase { + case 0: + // Modified key + keys[index%len(keys)] = f.randBytes(32) // In theory it can't be same + case 1: + // Modified val + vals[index%len(vals)] = f.randBytes(20) // In theory it can't be same + case 2: + // Gapped entry slice + index = index % len(keys) + keys = append(keys[:index], keys[index+1:]...) + vals = append(vals[:index], vals[index+1:]...) + case 3: + // Out of order + index1 := index % len(keys) + index2 := index2 % len(keys) + keys[index1], keys[index2] = keys[index2], keys[index1] + vals[index1], vals[index2] = vals[index2], vals[index1] + case 4: + // Set random key to nil, do nothing + keys[index%len(keys)] = nil + case 5: + // Set random value to nil, deletion + vals[index%len(vals)] = nil + + // Other cases: + // Modify something in the proof db + // add stuff to proof db + // drop stuff from proof db + + } + if f.exhausted { + break + } + ok = 1 + //nodes, subtrie + nodes, subtrie, notary, hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, last, keys, vals, proof) + if err != nil { + if nodes != nil { + panic("err != nil && nodes != nil") + } + if subtrie != nil { + panic("err != nil && subtrie != nil") + } + if notary != nil { + panic("err != nil && notary != nil") + } + if hasMore { + panic("err != nil && hasMore == true") + } + } else { + if nodes == nil { + panic("err == nil && nodes == nil") + } + if subtrie == nil { + panic("err == nil && subtrie == nil") + } + if notary == nil { + panic("err == nil && subtrie == nil") + } + } + } + return ok +} + +// The function must return +// 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// -1 if the input must not be added to corpus even if gives new coverage; and +// 0 otherwise; other values are reserved for future use. +func Fuzz(input []byte) int { + if len(input) < 100 { + return 0 + } + r := bytes.NewReader(input) + f := fuzzer{ + input: r, + exhausted: false, + } + return f.fuzz() +} diff --git a/trie/notary.go b/trie/notary.go new file mode 100644 index 0000000000..5a64727aa7 --- /dev/null +++ b/trie/notary.go @@ -0,0 +1,57 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +// KeyValueNotary tracks which keys have been accessed through a key-value reader +// with te scope of verifying if certain proof datasets are maliciously bloated. +type KeyValueNotary struct { + ethdb.KeyValueReader + reads map[string]struct{} +} + +// NewKeyValueNotary wraps a key-value database with an access notary to track +// which items have bene accessed. +func NewKeyValueNotary(db ethdb.KeyValueReader) *KeyValueNotary { + return &KeyValueNotary{ + KeyValueReader: db, + reads: make(map[string]struct{}), + } +} + +// Get retrieves an item from the underlying database, but also tracks it as an +// accessed slot for bloat checks. +func (k *KeyValueNotary) Get(key []byte) ([]byte, error) { + k.reads[string(key)] = struct{}{} + return k.KeyValueReader.Get(key) +} + +// Accessed returns s snapshot of the original key-value store containing only the +// data accessed through the notary. +func (k *KeyValueNotary) Accessed() ethdb.KeyValueStore { + db := memorydb.New() + for keystr := range k.reads { + key := []byte(keystr) + val, _ := k.KeyValueReader.Get(key) + db.Put(key, val) + } + return db +} diff --git a/trie/proof.go b/trie/proof.go index 2f52438f98..e7102f12b7 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -426,7 +426,7 @@ func hasRightElement(node node, key []byte) bool { // VerifyRangeProof checks whether the given leaf nodes and edge proof // can prove the given trie leaves range is matched with the specific root. -// Besides, the range should be consecutive(no gap inside) and monotonic +// Besides, the range should be consecutive (no gap inside) and monotonic // increasing. // // Note the given proof actually contains two edge proofs. Both of them can @@ -454,96 +454,136 @@ func hasRightElement(node node, key []byte) bool { // // Except returning the error to indicate the proof is valid or not, the function will // also return a flag to indicate whether there exists more accounts/slots in the trie. -func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (error, bool) { +func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (ethdb.KeyValueStore, *Trie, *KeyValueNotary, bool, error) { if len(keys) != len(values) { - return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)), false + return nil, nil, nil, false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) } // Ensure the received batch is monotonic increasing. for i := 0; i < len(keys)-1; i++ { if bytes.Compare(keys[i], keys[i+1]) >= 0 { - return errors.New("range is not monotonically increasing"), false + return nil, nil, nil, false, errors.New("range is not monotonically increasing") } } + // Create a key-value notary to track which items from the given proof the + // range prover actually needed to verify the data + notary := NewKeyValueNotary(proof) + // Special case, there is no edge proof at all. The given range is expected // to be the whole leaf-set in the trie. if proof == nil { - emptytrie, err := New(common.Hash{}, NewDatabase(memorydb.New())) + var ( + diskdb = memorydb.New() + triedb = NewDatabase(diskdb) + ) + tr, err := New(common.Hash{}, triedb) if err != nil { - return err, false + return nil, nil, nil, false, err } for index, key := range keys { - emptytrie.TryUpdate(key, values[index]) + tr.TryUpdate(key, values[index]) + } + if tr.Hash() != rootHash { + return nil, nil, nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) + } + // Proof seems valid, serialize all the nodes into the database + if _, err := tr.Commit(nil); err != nil { + return nil, nil, nil, false, err } - if emptytrie.Hash() != rootHash { - return fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, emptytrie.Hash()), false + if err := triedb.Commit(rootHash, false, nil); err != nil { + return nil, nil, nil, false, err } - return nil, false // no more element. + return diskdb, tr, notary, false, nil // No more elements } // Special case, there is a provided edge proof but zero key/value // pairs, ensure there are no more accounts / slots in the trie. if len(keys) == 0 { - root, val, err := proofToPath(rootHash, nil, firstKey, proof, true) + root, val, err := proofToPath(rootHash, nil, firstKey, notary, true) if err != nil { - return err, false + return nil, nil, nil, false, err } if val != nil || hasRightElement(root, firstKey) { - return errors.New("more entries available"), false + return nil, nil, nil, false, errors.New("more entries available") } - return nil, false + // Since the entire proof is a single path, we can construct a trie and a + // node database directly out of the inputs, no need to generate them + diskdb := notary.Accessed() + tr := &Trie{ + db: NewDatabase(diskdb), + root: root, + } + return diskdb, tr, notary, hasRightElement(root, firstKey), nil } // Special case, there is only one element and two edge keys are same. // In this case, we can't construct two edge paths. So handle it here. if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { - root, val, err := proofToPath(rootHash, nil, firstKey, proof, false) + root, val, err := proofToPath(rootHash, nil, firstKey, notary, false) if err != nil { - return err, false + return nil, nil, nil, false, err } if !bytes.Equal(firstKey, keys[0]) { - return errors.New("correct proof but invalid key"), false + return nil, nil, nil, false, errors.New("correct proof but invalid key") } if !bytes.Equal(val, values[0]) { - return errors.New("correct proof but invalid data"), false + return nil, nil, nil, false, errors.New("correct proof but invalid data") + } + // Since the entire proof is a single path, we can construct a trie and a + // node database directly out of the inputs, no need to generate them + diskdb := notary.Accessed() + tr := &Trie{ + db: NewDatabase(diskdb), + root: root, } - return nil, hasRightElement(root, firstKey) + return diskdb, tr, notary, hasRightElement(root, firstKey), nil } // Ok, in all other cases, we require two edge paths available. // First check the validity of edge keys. if bytes.Compare(firstKey, lastKey) >= 0 { - return errors.New("invalid edge keys"), false + return nil, nil, nil, false, errors.New("invalid edge keys") } // todo(rjl493456442) different length edge keys should be supported if len(firstKey) != len(lastKey) { - return errors.New("inconsistent edge keys"), false + return nil, nil, nil, false, errors.New("inconsistent edge keys") } // Convert the edge proofs to edge trie paths. Then we can // have the same tree architecture with the original one. // For the first edge proof, non-existent proof is allowed. - root, _, err := proofToPath(rootHash, nil, firstKey, proof, true) + root, _, err := proofToPath(rootHash, nil, firstKey, notary, true) if err != nil { - return err, false + return nil, nil, nil, false, err } // Pass the root node here, the second path will be merged // with the first one. For the last edge proof, non-existent // proof is also allowed. - root, _, err = proofToPath(rootHash, root, lastKey, proof, true) + root, _, err = proofToPath(rootHash, root, lastKey, notary, true) if err != nil { - return err, false + return nil, nil, nil, false, err } // Remove all internal references. All the removed parts should // be re-filled(or re-constructed) by the given leaves range. if err := unsetInternal(root, firstKey, lastKey); err != nil { - return err, false + return nil, nil, nil, false, err } - // Rebuild the trie with the leave stream, the shape of trie + // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - newtrie := &Trie{root: root, db: NewDatabase(memorydb.New())} + var ( + diskdb = memorydb.New() + triedb = NewDatabase(diskdb) + ) + tr := &Trie{root: root, db: triedb} for index, key := range keys { - newtrie.TryUpdate(key, values[index]) + tr.TryUpdate(key, values[index]) + } + if tr.Hash() != rootHash { + return nil, nil, nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) + } + // Proof seems valid, serialize all the nodes into the database + if _, err := tr.Commit(nil); err != nil { + return nil, nil, nil, false, err } - if newtrie.Hash() != rootHash { - return fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, newtrie.Hash()), false + if err := triedb.Commit(rootHash, false, nil); err != nil { + return nil, nil, nil, false, err } - return nil, hasRightElement(root, keys[len(keys)-1]) + return diskdb, tr, notary, hasRightElement(root, keys[len(keys)-1]), nil } // get returns the child of the given node. Return nil if the diff --git a/trie/proof_test.go b/trie/proof_test.go index 6cdc242d9a..3ecd318886 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -19,6 +19,7 @@ package trie import ( "bytes" crand "crypto/rand" + "encoding/binary" mrand "math/rand" "sort" "testing" @@ -181,7 +182,7 @@ func TestRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -232,7 +233,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -253,7 +254,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatal("Failed to verify whole rang with non-existent edges") } @@ -288,7 +289,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -310,7 +311,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -334,7 +335,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -349,7 +350,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -364,7 +365,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -379,7 +380,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -401,7 +402,7 @@ func TestAllElementsProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -414,7 +415,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -429,7 +430,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -462,7 +463,7 @@ func TestSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -498,7 +499,7 @@ func TestReverseSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -570,7 +571,7 @@ func TestBadRangeProof(t *testing.T) { index = mrand.Intn(end - start) vals[index] = nil } - err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err == nil { t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) } @@ -604,7 +605,7 @@ func TestGappedRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err == nil { t.Fatal("expect error, got nil") } @@ -631,7 +632,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -647,7 +648,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -715,7 +716,7 @@ func TestHasRightElement(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) + _, _, _, hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -748,13 +749,57 @@ func TestEmptyRangeProof(t *testing.T) { if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - err, _ := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) + db, tr, not, _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) if c.err && err == nil { t.Fatalf("Expected error, got nil") } if !c.err && err != nil { t.Fatalf("Expected no error, got %v", err) } + // If no error was returned, ensure the returned trie and database contains + // the entire proof, since there's no value + if !c.err { + if err := tr.Prove(first, 0, memorydb.New()); err != nil { + t.Errorf("returned trie doesn't contain original proof: %v", err) + } + if memdb := db.(*memorydb.Database); memdb.Len() != proof.Len() { + t.Errorf("database entry count mismatch: have %d, want %d", memdb.Len(), proof.Len()) + } + if not == nil { + t.Errorf("missing notary") + } + } + } +} + +// TestBloatedProof tests a malicious proof, where the proof is more or less the +// whole trie. +func TestBloatedProof(t *testing.T) { + // Use a small trie + trie, kvs := nonRandomTrie(100) + var entries entrySlice + for _, kv := range kvs { + entries = append(entries, kv) + } + sort.Sort(entries) + var keys [][]byte + var vals [][]byte + + proof := memorydb.New() + for i, entry := range entries { + trie.Prove(entry.k, 0, proof) + if i == 50 { + keys = append(keys, entry.k) + vals = append(vals, entry.v) + } + } + want := memorydb.New() + trie.Prove(keys[0], 0, want) + trie.Prove(keys[len(keys)-1], 0, want) + + _, _, notary, _, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + if used := notary.Accessed().(*memorydb.Database); used.Len() != want.Len() { + t.Fatalf("notary proof size mismatch: have %d, want %d", used.Len(), want.Len()) } } @@ -858,7 +903,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { b.ResetTimer() for i := 0; i < b.N; i++ { - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) + _, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) if err != nil { b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -889,3 +934,20 @@ func randBytes(n int) []byte { crand.Read(r) return r } + +func nonRandomTrie(n int) (*Trie, map[string]*kv) { + trie := new(Trie) + vals := make(map[string]*kv) + max := uint64(0xffffffffffffffff) + for i := uint64(0); i < uint64(n); i++ { + value := make([]byte, 32) + key := make([]byte, 32) + binary.LittleEndian.PutUint64(key, i) + binary.LittleEndian.PutUint64(value, i-max) + //value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} + elem := &kv{key, value, false} + trie.Update(elem.k, elem.v) + vals[string(elem.k)] = elem + } + return trie, vals +} diff --git a/trie/sync_bloom.go b/trie/sync_bloom.go index 89f61d66d9..979f4748f3 100644 --- a/trie/sync_bloom.go +++ b/trie/sync_bloom.go @@ -125,14 +125,14 @@ func (b *SyncBloom) init(database ethdb.Iteratee) { it.Release() it = database.NewIterator(nil, key) - log.Info("Initializing fast sync bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Initializing state bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start))) swap = time.Now() } } it.Release() // Mark the bloom filter inited and return - log.Info("Initialized fast sync bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Initialized state bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start))) atomic.StoreUint32(&b.inited, 1) } @@ -162,7 +162,7 @@ func (b *SyncBloom) Close() error { b.pend.Wait() // Wipe the bloom, but mark it "uninited" just in case someone attempts an access - log.Info("Deallocated fast sync bloom", "items", b.bloom.N(), "errorrate", b.errorRate()) + log.Info("Deallocated state bloom", "items", b.bloom.N(), "errorrate", b.errorRate()) atomic.StoreUint32(&b.inited, 0) b.bloom = nil diff --git a/trie/trie.go b/trie/trie.go index 6ddbbd78d3..87b72ecf17 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -19,13 +19,13 @@ package trie import ( "bytes" + "errors" "fmt" "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -159,29 +159,26 @@ func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) { if item == nil { return nil, resolved, nil } - enc, err := rlp.EncodeToBytes(item) - if err != nil { - log.Error("Encoding existing trie node failed", "err", err) - return nil, resolved, err - } - return enc, resolved, err + return item, resolved, err } -func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item node, newnode node, resolved int, err error) { +func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) { // If we reached the requested path, return the current node if pos >= len(path) { - // Don't return collapsed hash nodes though - if _, ok := origNode.(hashNode); !ok { - // Short nodes have expanded keys, compact them before returning - item := origNode - if sn, ok := item.(*shortNode); ok { - item = &shortNode{ - Key: hexToCompact(sn.Key), - Val: sn.Val, - } - } - return item, origNode, 0, nil + // Although we most probably have the original node expanded, encoding + // that into consensus form can be nasty (needs to cascade down) and + // time consuming. Instead, just pull the hash up from disk directly. + var hash hashNode + if node, ok := origNode.(hashNode); ok { + hash = node + } else { + hash, _ = origNode.cache() + } + if hash == nil { + return nil, origNode, 0, errors.New("non-consensus node") } + blob, err := t.db.Node(common.BytesToHash(hash)) + return blob, origNode, 1, err } // Path still needs to be traversed, descend into children switch n := (origNode).(type) { @@ -491,7 +488,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { - hash, cached, _ := t.hashRoot(nil) + hash, cached, _ := t.hashRoot() t.root = cached return common.BytesToHash(hash.(hashNode)) } @@ -545,7 +542,7 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { } // hashRoot calculates the root hash of the given trie -func (t *Trie) hashRoot(db *Database) (node, node, error) { +func (t *Trie) hashRoot() (node, node, error) { if t.root == nil { return hashNode(emptyRoot.Bytes()), nil, nil } From 485992979827596d92e622fec25ce68fe1bfd35b Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 14 Dec 2020 14:08:53 +0100 Subject: [PATCH 013/709] cmd/geth: fixed parallelization flaw in account import test (#22002) --- cmd/geth/accountcmd_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/geth/accountcmd_test.go b/cmd/geth/accountcmd_test.go index e27adb6916..04f55e9e7a 100644 --- a/cmd/geth/accountcmd_test.go +++ b/cmd/geth/accountcmd_test.go @@ -102,6 +102,7 @@ func TestAccountImport(t *testing.T) { }, } for _, test := range tests { + test := test t.Run(test.name, func(t *testing.T) { t.Parallel() importAccountWithExpect(t, test.key, test.output) From 0fe66f8ae41d2ca773f6b01080ddda10bec24377 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Mon, 14 Dec 2020 15:31:23 +0200 Subject: [PATCH 014/709] eth/protocols/eth: remove magic numbers in test (#21999) --- eth/protocols/eth/handler_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 65c4a10b0a..30beae931b 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -254,8 +254,8 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - p2p.Send(peer.app, 0x03, tt.query) - if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } // If the test used number origins, repeat with hashes as the too @@ -263,8 +263,8 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - p2p.Send(peer.app, 0x03, tt.query) - if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } } @@ -343,8 +343,8 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - p2p.Send(peer.app, 0x05, hashes) - if err := p2p.ExpectMsg(peer.app, 0x06, bodies); err != nil { + p2p.Send(peer.app, GetBlockBodiesMsg, hashes) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { t.Errorf("test %d: bodies mismatch: %v", i, err) } } @@ -410,13 +410,13 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - p2p.Send(peer.app, 0x0d, hashes) + p2p.Send(peer.app, GetNodeDataMsg, hashes) msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) } - if msg.Code != 0x0e { - t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, 0x0c) + if msg.Code != NodeDataMsg { + t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } var data [][]byte if err := msg.Decode(&data); err != nil { @@ -512,8 +512,8 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - p2p.Send(peer.app, 0x0f, hashes) - if err := p2p.ExpectMsg(peer.app, 0x10, receipts); err != nil { + p2p.Send(peer.app, GetReceiptsMsg, hashes) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { t.Errorf("receipts mismatch: %v", err) } } From 8cde2966af916d85805a47a4350f3567d9e51dbe Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 15 Dec 2020 18:52:51 +0100 Subject: [PATCH 015/709] eth, core: speed up some tests (#22000) --- core/bloombits/matcher_test.go | 6 ++++++ core/bloombits/scheduler_test.go | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index 91143e525e..923579221f 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -30,6 +30,7 @@ const testSectionSize = 4096 // Tests that wildcard filter rules (nil) can be specified and are handled well. func TestMatcherWildcards(t *testing.T) { + t.Parallel() matcher := NewMatcher(testSectionSize, [][][]byte{ {common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard {common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()}, // Default hash is not a wildcard @@ -56,6 +57,7 @@ func TestMatcherWildcards(t *testing.T) { // Tests the matcher pipeline on a single continuous workflow without interrupts. func TestMatcherContinuous(t *testing.T) { + t.Parallel() testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75) testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81) testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36) @@ -64,6 +66,7 @@ func TestMatcherContinuous(t *testing.T) { // Tests the matcher pipeline on a constantly interrupted and resumed work pattern // with the aim of ensuring data items are requested only once. func TestMatcherIntermittent(t *testing.T) { + t.Parallel() testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75) testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81) testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36) @@ -71,6 +74,7 @@ func TestMatcherIntermittent(t *testing.T) { // Tests the matcher pipeline on random input to hopefully catch anomalies. func TestMatcherRandom(t *testing.T) { + t.Parallel() for i := 0; i < 10; i++ { testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0) testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0) @@ -84,6 +88,7 @@ func TestMatcherRandom(t *testing.T) { // shifter from a multiple of 8. This is needed to cover an optimisation with // bitset matching https://github.com/ethereum/go-ethereum/issues/15309. func TestMatcherShifted(t *testing.T) { + t.Parallel() // Block 0 always matches in the tests, skip ahead of first 8 blocks with the // start to get a potential zero byte in the matcher bitset. @@ -97,6 +102,7 @@ func TestMatcherShifted(t *testing.T) { // Tests that matching on everything doesn't crash (special case internally). func TestWildcardMatcher(t *testing.T) { + t.Parallel() testMatcherBothModes(t, nil, 0, 10000, 0) } diff --git a/core/bloombits/scheduler_test.go b/core/bloombits/scheduler_test.go index 70772e4ab9..707e8ea11d 100644 --- a/core/bloombits/scheduler_test.go +++ b/core/bloombits/scheduler_test.go @@ -35,6 +35,7 @@ func TestSchedulerMultiClientSingleFetcher(t *testing.T) { testScheduler(t, 10, func TestSchedulerMultiClientMultiFetcher(t *testing.T) { testScheduler(t, 10, 10, 5000) } func testScheduler(t *testing.T, clients int, fetchers int, requests int) { + t.Parallel() f := newScheduler(0) // Create a batch of handler goroutines that respond to bloom bit requests and @@ -88,10 +89,10 @@ func testScheduler(t *testing.T, clients int, fetchers int, requests int) { } close(in) }() - + b := new(big.Int) for j := 0; j < requests; j++ { bits := <-out - if want := new(big.Int).SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) { + if want := b.SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) { t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want) } } From c7f2536735a1a47ae63edb488e15ae597dbaf1d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 15 Dec 2020 20:12:14 +0100 Subject: [PATCH 016/709] les: les/4 minimalistic version (#21909) * les: allow tx unindexing in les/4 light server mode * les: minor fixes * les: more small fixes * les: add meaningful constants for recentTxIndex handshake field --- cmd/utils/flags.go | 7 +++---- les/odr_requests.go | 2 +- les/peer.go | 32 +++++++++++++++++++++++++++++++- les/protocol.go | 5 +++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0b1695d0a5..c51d7916ca 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1491,10 +1491,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag) - // todo(rjl493456442) make it available for les server - // Ancient tx indices pruning is not available for les server now - // since light client relies on the server for transaction status query. - CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, TxLookupLimitFlag) + if (ctx.GlobalIsSet(LegacyLightServFlag.Name) || ctx.GlobalIsSet(LightServeFlag.Name)) && ctx.GlobalIsSet(TxLookupLimitFlag.Name) { + log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") + } var ks *keystore.KeyStore if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 { ks = keystores[0].(*keystore.KeyStore) diff --git a/les/odr_requests.go b/les/odr_requests.go index eb1d3602e0..a8cf8f50a9 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -488,7 +488,7 @@ func (r *TxStatusRequest) GetCost(peer *serverPeer) uint64 { // CanSend tells if a certain peer is suitable for serving the given request func (r *TxStatusRequest) CanSend(peer *serverPeer) bool { - return peer.version >= lpv2 + return peer.serveTxLookup } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) diff --git a/les/peer.go b/les/peer.go index 6004af03f5..0e2ed52c12 100644 --- a/les/peer.go +++ b/les/peer.go @@ -341,6 +341,7 @@ type serverPeer struct { onlyAnnounce bool // The flag whether the server sends announcement only. chainSince, chainRecent uint64 // The range of chain server peer can serve. stateSince, stateRecent uint64 // The range of state server peer can serve. + serveTxLookup bool // The server peer can serve tx lookups. // Advertised checkpoint fields checkpointNumber uint64 // The block height which the checkpoint is registered. @@ -628,6 +629,18 @@ func (p *serverPeer) Handshake(genesis common.Hash, forkid forkid.ID, forkFilter if recv.get("txRelay", nil) != nil { p.onlyAnnounce = true } + if p.version >= lpv4 { + var recentTx uint + if err := recv.get("recentTxLookup", &recentTx); err != nil { + return err + } + // Note: in the current version we only consider the tx index service useful + // if it is unlimited. This can be made configurable in the future. + p.serveTxLookup = recentTx == txIndexUnlimited + } else { + p.serveTxLookup = true + } + if p.onlyAnnounce && !p.trusted { return errResp(ErrUselessPeer, "peer cannot serve requests") } @@ -969,6 +982,20 @@ func (p *clientPeer) freezeClient() { // Handshake executes the les protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter, server *LesServer) error { + recentTx := server.handler.blockchain.TxLookupLimit() + if recentTx != txIndexUnlimited { + if recentTx < blockSafetyMargin { + recentTx = txIndexDisabled + } else { + recentTx -= blockSafetyMargin - txIndexRecentOffset + } + } + if server.config.UltraLightOnlyAnnounce { + recentTx = txIndexDisabled + } + if recentTx != txIndexUnlimited && p.version < lpv4 { + return errors.New("Cannot serve old clients without a complete tx index") + } // Note: clientPeer.headInfo should contain the last head announced to the client by us. // The values announced in the handshake are dummy values for compatibility reasons and should be ignored. p.headInfo = blockInfo{Hash: head, Number: headNum, Td: td} @@ -981,13 +1008,16 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge // If local ethereum node is running in archive mode, advertise ourselves we have // all version state data. Otherwise only recent state is available. - stateRecent := uint64(core.TriesInMemory - 4) + stateRecent := uint64(core.TriesInMemory - blockSafetyMargin) if server.archiveMode { stateRecent = 0 } *lists = (*lists).add("serveRecentState", stateRecent) *lists = (*lists).add("txRelay", nil) } + if p.version >= lpv4 { + *lists = (*lists).add("recentTxLookup", recentTx) + } *lists = (*lists).add("flowControl/BL", server.defParams.BufLimit) *lists = (*lists).add("flowControl/MRR", server.defParams.MinRecharge) diff --git a/les/protocol.go b/les/protocol.go index aebe0f2c04..39d9f5152f 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -50,6 +50,11 @@ var ProtocolLengths = map[uint]uint64{lpv2: 22, lpv3: 24, lpv4: 24} const ( NetworkId = 1 ProtocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message + blockSafetyMargin = 4 // safety margin applied to block ranges specified relative to head block + + txIndexUnlimited = 0 // this value in the "recentTxLookup" handshake field means the entire tx index history is served + txIndexDisabled = 1 // this value means tx index is not served at all + txIndexRecentOffset = 1 // txIndexRecentOffset + N in the handshake field means then tx index of the last N blocks is supported ) // les protocol message codes From 3c46f5570bd674cf49c0113352ff79a4e026a5b8 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 17 Dec 2020 01:20:20 +0100 Subject: [PATCH 017/709] cmd/faucet: sort requests by newest first (#22018) --- cmd/faucet/faucet.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index d7927ac491..008cb14296 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -516,12 +516,12 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue } - f.reqs = append(f.reqs, &request{ + f.reqs = append([]*request{{ Avatar: avatar, Account: address, Time: time.Now(), Tx: signed, - }) + }}, f.reqs...) timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute grace := timeout / 288 // 24h timeout => 5m grace From c5a3ffa3638c8fbcb694881efa7e89f69717cdd3 Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 21 Dec 2020 18:54:39 +0800 Subject: [PATCH 018/709] eth/download/statesync : optimize to avoid a copy in state sync hashing (#22035) * eth/download/statesync : state hash sum optimized * go fmt with blank in imports * keccak read arg fix --- eth/downloader/statesync.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 69bd13c2f7..6231588ad2 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -18,13 +18,13 @@ package downloader import ( "fmt" - "hash" "sync" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" @@ -260,9 +260,9 @@ func (d *Downloader) spindownStateSync(active map[string]*stateReq, finished []* type stateSync struct { d *Downloader // Downloader instance to access and manage current peerset - root common.Hash // State root currently being synced - sched *trie.Sync // State trie sync scheduler defining the tasks - keccak hash.Hash // Keccak256 hasher to verify deliveries with + root common.Hash // State root currently being synced + sched *trie.Sync // State trie sync scheduler defining the tasks + keccak crypto.KeccakState // Keccak256 hasher to verify deliveries with trieTasks map[common.Hash]*trieTask // Set of trie node tasks currently queued for retrieval codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval @@ -299,7 +299,7 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { d: d, root: root, sched: state.NewStateSync(root, d.stateDB, d.stateBloom), - keccak: sha3.NewLegacyKeccak256(), + keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), trieTasks: make(map[common.Hash]*trieTask), codeTasks: make(map[common.Hash]*codeTask), deliver: make(chan *stateReq), @@ -590,7 +590,7 @@ func (s *stateSync) processNodeData(blob []byte) (common.Hash, error) { res := trie.SyncResult{Data: blob} s.keccak.Reset() s.keccak.Write(blob) - s.keccak.Sum(res.Hash[:0]) + s.keccak.Read(res.Hash[:]) err := s.sched.Process(res) return res.Hash, err } From 61469cfeaf2a6d0b1598afbaf35dd2d1872604ce Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 21 Dec 2020 22:39:58 +0800 Subject: [PATCH 019/709] eth/downloader: fix typo in comment (#22019) --- eth/downloader/modes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/downloader/modes.go b/eth/downloader/modes.go index 8ea7876a1f..3ea14d22d7 100644 --- a/eth/downloader/modes.go +++ b/eth/downloader/modes.go @@ -25,7 +25,7 @@ type SyncMode uint32 const ( FullSync SyncMode = iota // Synchronise the entire blockchain history from full blocks FastSync // Quickly download the headers, full sync only at the chain - SnapSync // Download the chain and the state via compact snashots + SnapSync // Download the chain and the state via compact snapshots LightSync // Download only the headers and terminate afterwards ) From 158f72cc0c889739f49dde42210328902073353d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 23 Dec 2020 13:43:22 +0100 Subject: [PATCH 020/709] internal/ethapi: restore net_version RPC method (#22061) During the snap and eth refactor, the net_version rpc call was falsely deprecated. This restores the net_version RPC handler as most eth2 nodes and other software depend on it. --- eth/backend.go | 2 +- internal/ethapi/api.go | 12 +++++++++--- les/client.go | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 987dee6d55..c1732d3ceb 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -230,7 +230,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { return nil, err } // Start the RPC service - eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer) + eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, config.NetworkId) // Register the backend on the node stack.RegisterAPIs(eth.APIs()) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9ff1781e4a..b424435b50 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1900,12 +1900,13 @@ func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) { // PublicNetAPI offers network related RPC methods type PublicNetAPI struct { - net *p2p.Server + net *p2p.Server + networkVersion uint64 } // NewPublicNetAPI creates a new net API instance. -func NewPublicNetAPI(net *p2p.Server) *PublicNetAPI { - return &PublicNetAPI{net} +func NewPublicNetAPI(net *p2p.Server, networkVersion uint64) *PublicNetAPI { + return &PublicNetAPI{net, networkVersion} } // Listening returns an indication if the node is listening for network connections. @@ -1918,6 +1919,11 @@ func (s *PublicNetAPI) PeerCount() hexutil.Uint { return hexutil.Uint(s.net.PeerCount()) } +// Version returns the current ethereum protocol version. +func (s *PublicNetAPI) Version() string { + return fmt.Sprintf("%d", s.networkVersion) +} + // checkTxFee is an internal function used to check whether the fee of // the given transaction is _reasonable_(under the cap). func checkTxFee(gasPrice *big.Int, gas uint64, cap float64) error { diff --git a/les/client.go b/les/client.go index 198255dc54..47997a098b 100644 --- a/les/client.go +++ b/les/client.go @@ -171,7 +171,7 @@ func New(stack *node.Node, config *eth.Config) (*LightEthereum, error) { leth.blockchain.DisableCheckFreq() } - leth.netRPCService = ethapi.NewPublicNetAPI(leth.p2pServer) + leth.netRPCService = ethapi.NewPublicNetAPI(leth.p2pServer, leth.config.NetworkId) // Register the backend on the node stack.RegisterAPIs(leth.APIs()) From b9012a039b8aaf3e68ccc3826bb17d27eaf0fa1c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 23 Dec 2020 17:44:45 +0100 Subject: [PATCH 021/709] common,crypto: move fuzzers out of core (#22029) * common,crypto: move fuzzers out of core * fuzzers: move vm fuzzer out from core * fuzzing: rework cover package logic * fuzzers: lint --- oss-fuzz.sh | 88 ++++++++++++++----- .../fuzzers}/bitutil/compress_fuzz.go | 14 +-- {crypto => tests/fuzzers}/bn256/bn256_fuzz.go | 6 +- .../fuzzers/runtime/runtime_fuzz.go | 14 +-- 4 files changed, 84 insertions(+), 38 deletions(-) rename {common => tests/fuzzers}/bitutil/compress_fuzz.go (84%) rename {crypto => tests/fuzzers}/bn256/bn256_fuzz.go (94%) rename core/vm/runtime/fuzz.go => tests/fuzzers/runtime/runtime_fuzz.go (82%) diff --git a/oss-fuzz.sh b/oss-fuzz.sh index e060ea88e1..5919b2077f 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -26,38 +26,80 @@ # $CFLAGS, $CXXFLAGS C and C++ compiler flags. # $LIB_FUZZING_ENGINE C++ compiler argument to link fuzz target against the prebuilt engine library (e.g. libFuzzer). +# This sets the -coverpgk for the coverage report when the corpus is executed through go test +coverpkg="github.com/ethereum/go-ethereum/..." + +function coverbuild { + path=$1 + function=$2 + fuzzer=$3 + tags="" + + if [[ $# -eq 4 ]]; then + tags="-tags $4" + fi + cd $path + fuzzed_package=`pwd | rev | cut -d'/' -f 1 | rev` + cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go + sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go + sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go + sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go + +cat << DOG > $OUT/$fuzzer +#/bin/sh + + cd $OUT/$path + go test -run Test${function}Corpus -v $tags -coverprofile \$1 -coverpkg $coverpkg + +DOG + + chmod +x $OUT/$fuzzer + #echo "Built script $OUT/$fuzzer" + #cat $OUT/$fuzzer + cd - +} + function compile_fuzzer { - path=$SRC/go-ethereum/$1 + # Inputs: + # $1: The package to fuzz, within go-ethereum + # $2: The name of the fuzzing function + # $3: The name to give to the final fuzzing-binary + + path=$GOPATH/src/github.com/ethereum/go-ethereum/$1 func=$2 fuzzer=$3 - corpusfile="${path}/testdata/${fuzzer}_seed_corpus.zip" - echo "Building $fuzzer (expecting corpus at $corpusfile)" - (cd $path && \ + + echo "Building $fuzzer" + + # Do a coverage-build or a regular build + if [[ $SANITIZER = *coverage* ]]; then + coverbuild $path $func $fuzzer $coverpkg + else + (cd $path && \ go-fuzz -func $func -o $WORK/$fuzzer.a . && \ - echo "First stage built OK" && \ - $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $WORK/$fuzzer.a -o $OUT/$fuzzer && \ - echo "Second stage built ok" ) - - ## Check if there exists a seed corpus file - if [ -f $corpusfile ] - then - cp $corpusfile $OUT/ - echo "Found seed corpus: $corpusfile" - fi + $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $WORK/$fuzzer.a -o $OUT/$fuzzer) + fi + + ## Check if there exists a seed corpus file + corpusfile="${path}/testdata/${fuzzer}_seed_corpus.zip" + if [ -f $corpusfile ] + then + cp $corpusfile $OUT/ + echo "Found seed corpus: $corpusfile" + fi } -compile_fuzzer common/bitutil Fuzz fuzzBitutilCompress -compile_fuzzer crypto/bn256 FuzzAdd fuzzBn256Add -compile_fuzzer crypto/bn256 FuzzMul fuzzBn256Mul -compile_fuzzer crypto/bn256 FuzzPair fuzzBn256Pair -compile_fuzzer core/vm/runtime Fuzz fuzzVmRuntime -compile_fuzzer crypto/blake2b Fuzz fuzzBlake2b +compile_fuzzer tests/fuzzers/bitutil Fuzz fuzzBitutilCompress +compile_fuzzer tests/fuzzers/bn256 FuzzAdd fuzzBn256Add +compile_fuzzer tests/fuzzers/bn256 FuzzMul fuzzBn256Mul +compile_fuzzer tests/fuzzers/bn256 FuzzPair fuzzBn256Pair +compile_fuzzer tests/fuzzers/runtime Fuzz fuzzVmRuntime compile_fuzzer tests/fuzzers/keystore Fuzz fuzzKeystore compile_fuzzer tests/fuzzers/txfetcher Fuzz fuzzTxfetcher compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie -compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty +compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul @@ -69,6 +111,10 @@ compile_fuzzer tests/fuzzers/bls12381 FuzzPairing fuzz_pairing compile_fuzzer tests/fuzzers/bls12381 FuzzMapG1 fuzz_map_g1 compile_fuzzer tests/fuzzers/bls12381 FuzzMapG2 fuzz_map_g2 +#TODO: move this to tests/fuzzers, if possible +compile_fuzzer crypto/blake2b Fuzz fuzzBlake2b + + # This doesn't work very well @TODO #compile_fuzzertests/fuzzers/abi Fuzz fuzzAbi diff --git a/common/bitutil/compress_fuzz.go b/tests/fuzzers/bitutil/compress_fuzz.go similarity index 84% rename from common/bitutil/compress_fuzz.go rename to tests/fuzzers/bitutil/compress_fuzz.go index 714bbcd131..0b77b2dc9e 100644 --- a/common/bitutil/compress_fuzz.go +++ b/tests/fuzzers/bitutil/compress_fuzz.go @@ -14,11 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build gofuzz - package bitutil -import "bytes" +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common/bitutil" +) // Fuzz implements a go-fuzz fuzzer method to test various encoding method // invocations. @@ -35,7 +37,7 @@ func Fuzz(data []byte) int { // fuzzEncode implements a go-fuzz fuzzer method to test the bitset encoding and // decoding algorithm. func fuzzEncode(data []byte) int { - proc, _ := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data)) + proc, _ := bitutil.DecompressBytes(bitutil.CompressBytes(data), len(data)) if !bytes.Equal(data, proc) { panic("content mismatch") } @@ -45,11 +47,11 @@ func fuzzEncode(data []byte) int { // fuzzDecode implements a go-fuzz fuzzer method to test the bit decoding and // reencoding algorithm. func fuzzDecode(data []byte) int { - blob, err := bitsetDecodeBytes(data, 1024) + blob, err := bitutil.DecompressBytes(data, 1024) if err != nil { return 0 } - if comp := bitsetEncodeBytes(blob); !bytes.Equal(comp, data) { + if comp := bitutil.CompressBytes(blob); !bytes.Equal(comp, data) { panic("content mismatch") } return 1 diff --git a/crypto/bn256/bn256_fuzz.go b/tests/fuzzers/bn256/bn256_fuzz.go similarity index 94% rename from crypto/bn256/bn256_fuzz.go rename to tests/fuzzers/bn256/bn256_fuzz.go index b34043487f..477fe0a160 100644 --- a/crypto/bn256/bn256_fuzz.go +++ b/tests/fuzzers/bn256/bn256_fuzz.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. -// +build gofuzz - package bn256 import ( @@ -24,7 +22,7 @@ func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1) { } xg := new(google.G1) if _, err := xg.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err)) } return xc, xg } @@ -37,7 +35,7 @@ func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2) { } xg := new(google.G2) if _, err := xg.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err)) } return xc, xg } diff --git a/core/vm/runtime/fuzz.go b/tests/fuzzers/runtime/runtime_fuzz.go similarity index 82% rename from core/vm/runtime/fuzz.go rename to tests/fuzzers/runtime/runtime_fuzz.go index cb9ff08b5b..9b96045752 100644 --- a/core/vm/runtime/fuzz.go +++ b/tests/fuzzers/runtime/runtime_fuzz.go @@ -14,23 +14,23 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build gofuzz - package runtime +import ( + "github.com/ethereum/go-ethereum/core/vm/runtime" +) + // Fuzz is the basic entry point for the go-fuzz tool // // This returns 1 for valid parsable/runable code, 0 // for invalid opcode. func Fuzz(input []byte) int { - _, _, err := Execute(input, input, &Config{ - GasLimit: 3000000, + _, _, err := runtime.Execute(input, input, &runtime.Config{ + GasLimit: 12000000, }) - // invalid opcode - if err != nil && len(err.Error()) > 6 && string(err.Error()[:7]) == "invalid" { + if err != nil && len(err.Error()) > 6 && err.Error()[:7] == "invalid" { return 0 } - return 1 } From 25c0bd9b43da2c5b64e17b3847a8490d6abbe5c1 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Sun, 27 Dec 2020 17:56:50 +0000 Subject: [PATCH 022/709] README.md: update Travis badge (#22079) The legacy dot-org URL was displaying a message about the repository having migrated to the dot-com service, which now covers open-source projects as well. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddb885dfdc..19357355a8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Official Golang implementation of the Ethereum protocol. https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 )](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) -[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) +[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) Automated builds are available for stable releases and the unstable master branch. Binary From 9c6b5b904a0ea050a0ffda7cf7b60678b457783d Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Sun, 27 Dec 2020 21:57:19 +0100 Subject: [PATCH 023/709] eth, eth/tracers: include intrinsic gas in calltracer, expose for all tracers (#22038) * eth/tracers: share tx gas price with js tracer * eth/tracers: use `go generate` * eth/tracers: try with another version of go-bindata * eth/tracers: export txGas * eth, eth/tracers: pass intrinsic gas to js tracers eth/tracers: include tx gas in tracers usedGas eth/tracers: fix prestate tracer's sender balance eth/tracers: rm unnecessary import eth/tracers: pass intrinsicGas separately to tracer eth/tracers: fix tests broken by lack of txdata eth, eth/tracers: minor fix * eth/tracers: regenerate assets + unexport test-struct + add testcase * eth/tracers: simplify tests + make table-driven Co-authored-by: Guillaume Ballet Co-authored-by: Martin Holst Swende --- eth/api_tracer.go | 2 +- eth/tracers/internal/tracers/assets.go | 6 +- .../internal/tracers/prestate_tracer.js | 2 +- eth/tracers/tracer.go | 21 ++- eth/tracers/tracer_test.go | 151 ++++++++---------- eth/tracers/tracers_test.go | 4 +- 6 files changed, 92 insertions(+), 94 deletions(-) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 2497c8d95e..c68b762255 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -794,7 +794,7 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v } } // Constuct the JavaScript tracer to execute with - if tracer, err = tracers.New(*config.Tracer); err != nil { + if tracer, err = tracers.New(*config.Tracer, txContext); err != nil { return nil, err } // Handle timeouts and RPC cancellations diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 432398ebb5..8b89ad4182 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -6,7 +6,7 @@ // evmdis_tracer.js (4.195kB) // noop_tracer.js (1.271kB) // opcount_tracer.js (1.372kB) -// prestate_tracer.js (4.234kB) +// prestate_tracer.js (4.287kB) // trigram_tracer.js (1.788kB) // unigram_tracer.js (1.51kB) @@ -197,7 +197,7 @@ func opcount_tracerJs() (*asset, error) { return a, nil } -var _prestate_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x57\xdd\x6f\xdb\x38\x12\x7f\x96\xfe\x8a\x41\x5f\x6c\xa3\xae\xdc\x64\x81\x3d\xc0\xb9\x1c\xa0\xba\x6e\x1b\x20\x9b\x04\xb6\x7b\xb9\xdc\x62\x1f\x28\x72\x24\x73\x4d\x93\x02\x49\xd9\xf1\x15\xf9\xdf\x0f\x43\x7d\xf8\xa3\x49\xd3\xdd\x37\x9b\x1c\xfe\xe6\xfb\x37\xa3\xd1\x08\x26\xa6\xdc\x59\x59\x2c\x3d\x9c\xbf\x3f\xfb\x07\x2c\x96\x08\x85\x79\x87\x7e\x89\x16\xab\x35\xa4\x95\x5f\x1a\xeb\xe2\xd1\x08\x16\x4b\xe9\x20\x97\x0a\x41\x3a\x28\x99\xf5\x60\x72\xf0\x27\xf2\x4a\x66\x96\xd9\x5d\x12\x8f\x46\xf5\x9b\x67\xaf\x09\x21\xb7\x88\xe0\x4c\xee\xb7\xcc\xe2\x18\x76\xa6\x02\xce\x34\x58\x14\xd2\x79\x2b\xb3\xca\x23\x48\x0f\x4c\x8b\x91\xb1\xb0\x36\x42\xe6\x3b\x82\x94\x1e\x2a\x2d\xd0\x06\xd5\x1e\xed\xda\xb5\x76\x7c\xbe\xf9\x0a\xd7\xe8\x1c\x5a\xf8\x8c\x1a\x2d\x53\x70\x57\x65\x4a\x72\xb8\x96\x1c\xb5\x43\x60\x0e\x4a\x3a\x71\x4b\x14\x90\x05\x38\x7a\xf8\x89\x4c\x99\x37\xa6\xc0\x27\x53\x69\xc1\xbc\x34\x7a\x08\x28\xc9\x72\xd8\xa0\x75\xd2\x68\xf8\xa5\x55\xd5\x00\x0e\xc1\x58\x02\xe9\x33\x4f\x0e\x58\x30\x25\xbd\x1b\x00\xd3\x3b\x50\xcc\xef\x9f\xfe\x44\x40\xf6\x7e\x0b\x90\x3a\xa8\x59\x9a\x12\xc1\x2f\x99\x27\xaf\xb7\x52\x29\xc8\x10\x2a\x87\x79\xa5\x86\x84\x96\x55\x1e\xee\xaf\x16\x5f\x6e\xbf\x2e\x20\xbd\x79\x80\xfb\x74\x36\x4b\x6f\x16\x0f\x17\xb0\x95\x7e\x69\x2a\x0f\xb8\xc1\x1a\x4a\xae\x4b\x25\x51\xc0\x96\x59\xcb\xb4\xdf\x81\xc9\x09\xe1\xb7\xe9\x6c\xf2\x25\xbd\x59\xa4\x1f\xae\xae\xaf\x16\x0f\x60\x2c\x7c\xba\x5a\xdc\x4c\xe7\x73\xf8\x74\x3b\x83\x14\xee\xd2\xd9\xe2\x6a\xf2\xf5\x3a\x9d\xc1\xdd\xd7\xd9\xdd\xed\x7c\x9a\xc0\x1c\xc9\x2a\xa4\xf7\xaf\xc7\x3c\x0f\xd9\xb3\x08\x02\x3d\x93\xca\xb5\x91\x78\x30\x15\xb8\xa5\xa9\x94\x80\x25\xdb\x20\x58\xe4\x28\x37\x28\x80\x01\x37\xe5\xee\xa7\x93\x4a\x58\x4c\x19\x5d\x04\x9f\x5f\x2c\x48\xb8\xca\x41\x1b\x3f\x04\x87\x08\xff\x5c\x7a\x5f\x8e\x47\xa3\xed\x76\x9b\x14\xba\x4a\x8c\x2d\x46\xaa\x86\x73\xa3\x7f\x25\x31\x61\x96\x16\x9d\x67\x1e\x17\x96\x71\xb4\x60\x2a\x5f\x56\xde\x81\xab\xf2\x5c\x72\x89\xda\x83\xd4\xb9\xb1\xeb\x50\x29\xe0\x0d\x70\x8b\xcc\x23\x30\x50\x86\x33\x05\xf8\x88\xbc\x0a\x77\x75\xa4\x43\xb9\x5a\xa6\x1d\xe3\xe1\x34\xb7\x66\x4d\xbe\x56\xce\xd3\x0f\xe7\x70\x9d\x29\x14\x50\xa0\x46\x27\x1d\x64\xca\xf0\x55\x12\x7f\x8b\xa3\x03\x63\xa8\x4e\x82\x87\x8d\x50\xa8\x8d\x2d\xf6\x2c\x42\x56\x49\x25\xa4\x2e\x92\x38\x6a\xa5\xc7\xa0\x2b\xa5\x86\x71\x80\x50\xc6\xac\xaa\x32\xe5\xdc\x54\xc1\xf6\x3f\x91\xfb\x1a\xcc\x95\xc8\x65\x4e\xc5\xc1\xba\x5b\x6f\xc2\x55\xa7\xd7\x64\x24\x9f\xc4\xd1\x11\xcc\x18\xf2\x4a\x07\x77\xfa\x4c\x08\x3b\x04\x91\x0d\xbe\xc5\x51\xb4\x61\x96\xb0\xe0\x12\xbc\xf9\x82\x8f\xe1\x72\x70\x11\x47\x91\xcc\xa1\xef\x97\xd2\x25\x2d\xf0\xef\x8c\xf3\x3f\xe0\xf2\xf2\x32\x34\x75\x2e\x35\x8a\x01\x10\x44\xf4\x9c\x58\x7d\x13\x65\x4c\x31\xcd\x71\x0c\xbd\xf7\x8f\x3d\x78\x0b\x22\x4b\x0a\xf4\x1f\xea\xd3\x5a\x59\xe2\xcd\xdc\x5b\xa9\x8b\xfe\xd9\xaf\x83\x61\x78\xa5\x4d\x78\x03\x8d\xf8\x8d\xe9\x84\xeb\x7b\x6e\x44\xb8\x6e\x6c\xae\xa5\x26\x46\x34\x42\x8d\x94\xf3\xc6\xb2\x02\xc7\xf0\xed\x89\xfe\x3f\x91\x57\x4f\x71\xf4\x74\x14\xe5\x79\x2d\xf4\x42\x94\x1b\x08\x40\xed\x6d\x57\xe7\x85\xa4\x4e\x3d\x4c\x40\xc0\xfb\x51\x12\xe6\xad\x29\x27\x49\x58\xe1\xee\xf5\x4c\xd0\x85\x14\x8f\xdd\xc5\x0a\x77\x83\x8b\xf8\xc5\x14\x25\x8d\xd1\xbf\x4b\xf1\xf8\xb3\xf9\x3a\x79\x73\x14\xd7\x39\x49\xed\xed\x1d\x0c\x4e\xe2\x68\xd1\x55\xca\x53\xb9\x4b\xbd\x31\x2b\x22\xae\x25\xc5\x47\xa9\x10\x12\x53\x52\xb6\x5c\xcd\x1c\x19\xa2\x06\xe9\xd1\x32\xa2\x4e\xb3\x41\x4b\x53\x03\x2c\xfa\xca\x6a\xd7\x85\x31\x97\x9a\xa9\x16\xb8\x89\xba\xb7\x8c\xd7\x3d\x53\x9f\x1f\xc4\x92\xfb\xc7\x10\xc5\xe0\xdd\x68\x04\xa9\x07\x72\x11\x4a\x23\xb5\x1f\xc2\x16\x41\x23\x0a\x6a\x7c\x81\xa2\xe2\x3e\xe0\xf5\x36\x4c\x55\xd8\xab\x9b\x9b\x28\x32\x3c\x35\x15\x4d\x82\x83\xe6\x1f\x06\x03\xd7\x66\x13\x46\x5c\xc6\xf8\x0a\x9a\x86\x33\x56\x16\x52\xc7\x4d\x38\x8f\x9a\x8d\x2c\x4a\x08\x38\x98\x15\x72\x45\x49\xa4\x93\x0f\x4c\xc1\x25\x64\xb2\xb8\xd2\xfe\x24\x79\x75\xd0\xdb\xa7\x83\x3f\x92\xa6\x79\x12\x47\x84\xd7\x3f\x1f\x0c\xe1\xec\xd7\xae\x22\xbc\x21\x28\x78\x1d\xcc\x9b\x97\xa1\xe2\xd3\x62\x78\xfe\x59\x50\x43\x1d\xfc\x36\x68\x4d\x5c\x95\x51\x3a\x6a\x3f\x43\x1c\x8f\xbb\xf8\xe2\x07\xb8\xc7\xbe\xb5\xb8\x4d\x68\x12\x26\xc4\xcb\xa0\x75\x8a\x3e\x22\xb7\xb8\x26\x56\xa7\x2c\x70\xa6\x14\xda\x9e\x83\xc0\x19\xc3\xa6\x9c\x42\xbe\x70\x5d\xfa\x5d\xcb\xf5\x9e\xd9\x02\xbd\x7b\xdd\xb0\x80\xf3\xee\x5d\x4b\x81\x21\x14\xbb\x12\xe1\xf2\x12\x7a\x93\xd9\x34\x5d\x4c\x7b\x4d\x1b\x8d\x46\x70\x8f\x61\x13\xca\x94\xcc\x84\xda\x81\x40\x85\x1e\x6b\xbb\x8c\x0e\x21\xea\x28\x61\x48\x2b\x0d\x2d\x1b\xf8\x28\x9d\x97\xba\x80\x9a\x29\xb6\x34\x57\x1b\xb8\xd0\x23\x9c\x55\x8e\xaa\xf5\x64\x08\x79\x43\x1b\x85\x45\xe2\x15\xe2\xff\xd0\x6e\x4c\xc9\x6e\x03\xc9\xa5\x75\x1e\x4a\xc5\x38\x26\x84\xd7\x19\xf3\x72\x7e\x9b\x4e\x26\xd5\xb3\xd0\x82\x01\x68\x3f\xe0\x98\xa2\x01\x49\xea\x1d\xf4\x5b\x8c\x41\x1c\x45\xb6\x95\x3e\xc0\xbe\xd8\x53\x82\xf3\x58\x1e\x12\x02\x2d\x16\xb8\x41\xa2\xd0\xc0\x06\xf5\x30\x24\x5d\xff\xfe\xad\x99\xbe\xe8\x92\x38\xa2\x77\x07\x7d\xad\x4c\x71\xdc\xd7\xa2\x0e\x0b\xaf\xac\xa5\xfc\x77\x14\x9c\x53\x8f\xff\x59\x39\x4f\x31\xb5\x14\x9e\x86\x2d\x9e\x23\xc9\x40\x89\x34\x6d\x07\xdf\x93\x21\xcd\xad\x30\x27\x48\x5d\x33\xa5\xea\x6d\xae\x34\x1e\xb5\x97\x4c\xa9\x1d\xe5\x61\x6b\x69\x8d\xa1\xc5\x65\x08\x4e\x92\x54\x60\x9c\x20\x2a\x35\x57\x95\xa8\xcb\x20\xd4\x71\x83\xe7\x82\xcd\xc7\xfb\xcf\x1a\x9d\x63\x05\x26\x54\x49\xb9\x7c\x6c\x36\x48\x0d\xbd\x9a\xe4\xfa\x83\x5e\xd2\x19\x79\x4c\x31\xca\x14\x49\x5b\x64\x44\xd3\xa9\x10\x16\x9d\xeb\x0f\x1a\xce\xe9\x32\x7b\xbf\x44\x4d\xc1\x07\x8d\x5b\xe8\x56\x13\xc6\x39\xad\x6a\x62\x08\x4c\x08\xa2\xb6\x93\x35\x22\x8e\x22\xb7\x95\x9e\x2f\x21\x68\x32\xe5\xbe\x17\x07\x4d\xfd\x73\xe6\x10\xde\x4c\xff\xb3\x98\xdc\x7e\x9c\x4e\x6e\xef\x1e\xde\x8c\xe1\xe8\x6c\x7e\xf5\xdf\x69\x77\xf6\x21\xbd\x4e\x6f\x26\xd3\x37\xe3\x30\x9b\x9f\x71\xc8\x9b\xd6\x05\x52\xe8\x3c\xe3\xab\xa4\x44\x5c\xf5\xdf\x1f\xf3\xc0\xde\xc1\x28\xca\x2c\xb2\xd5\xc5\xde\x98\xba\x41\x1b\x1d\x2d\xe5\xc2\x25\xbc\x18\xac\x8b\x97\xad\x99\x34\xf2\xfd\x96\xc8\xf7\xab\x48\xa0\x8a\xd7\xed\x38\xff\xcb\x86\x84\xde\x61\x7c\x35\x06\xc7\x14\x6d\xc0\xf2\x7f\xf4\xe5\x92\xe7\x0e\xfd\x10\x50\x0b\xb3\x25\xe6\xeb\x50\xeb\x9b\x06\xf7\x20\x64\x67\x83\x9a\x41\x6f\xf3\xfe\xa0\x13\x26\xb0\xef\x45\xcf\x9f\x13\x45\x2d\xe0\xb2\x45\x7f\x1b\x5e\xbe\x1e\xa8\xf3\x26\x52\x27\x0a\x7e\x39\xd9\xf0\xc2\xfd\x1a\xd7\xc6\xee\x9a\x71\x74\xe0\xdf\x8f\xa3\x9a\x5e\x5f\x77\xf5\x44\x7f\xa8\xc8\xba\x83\x8f\xd3\xeb\xe9\xe7\x74\x31\x3d\x92\x9a\x2f\xd2\xc5\xd5\xa4\x3e\xfa\xcb\x85\x77\xf6\xd3\x85\xd7\x9b\xcf\x17\xb7\xb3\x69\x6f\xdc\xfc\xbb\xbe\x4d\x3f\xf6\xbe\x53\xd8\x6c\x81\x3f\x6a\x5d\x6f\xee\x8d\x15\x7f\xa7\x03\x0e\x36\xb2\x9c\x3d\xb7\x90\x05\x6a\xe7\xbe\x3a\xf9\xe0\x01\xa6\x5b\x56\xce\xeb\x8f\xbe\x28\xbc\x7f\x96\x87\x9f\xe2\xa7\xf8\xff\x01\x00\x00\xff\xff\xb1\x28\x85\x2a\x8a\x10\x00\x00") +var _prestate_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x57\xdd\x6f\xdb\x38\x12\x7f\xb6\xfe\x8a\x41\x5f\x6c\x5d\x5d\xb9\xcd\x02\x7b\x80\x73\x39\x40\x75\xdd\x36\x40\x36\x09\x6c\xe7\x72\xb9\xc5\x3e\x50\xe4\x48\xe6\x9a\x26\x05\x92\xb2\xe3\x2b\xf2\xbf\x1f\x86\xfa\xf0\x47\x93\xa6\x7b\x6f\x16\x39\xfc\xcd\xf7\x6f\xc6\xa3\x11\x4c\x4c\xb9\xb3\xb2\x58\x7a\x38\x7b\xff\xe1\xef\xb0\x58\x22\x14\xe6\x1d\xfa\x25\x5a\xac\xd6\x90\x56\x7e\x69\xac\x8b\x46\x23\x58\x2c\xa5\x83\x5c\x2a\x04\xe9\xa0\x64\xd6\x83\xc9\xc1\x9f\xc8\x2b\x99\x59\x66\x77\x49\x34\x1a\xd5\x6f\x9e\xbd\x26\x84\xdc\x22\x82\x33\xb9\xdf\x32\x8b\x63\xd8\x99\x0a\x38\xd3\x60\x51\x48\xe7\xad\xcc\x2a\x8f\x20\x3d\x30\x2d\x46\xc6\xc2\xda\x08\x99\xef\x08\x52\x7a\xa8\xb4\x40\x1b\x54\x7b\xb4\x6b\xd7\xda\xf1\xe5\xfa\x0e\xae\xd0\x39\xb4\xf0\x05\x35\x5a\xa6\xe0\xb6\xca\x94\xe4\x70\x25\x39\x6a\x87\xc0\x1c\x94\x74\xe2\x96\x28\x20\x0b\x70\xf4\xf0\x33\x99\x32\x6f\x4c\x81\xcf\xa6\xd2\x82\x79\x69\xf4\x10\x50\x92\xe5\xb0\x41\xeb\xa4\xd1\xf0\x4b\xab\xaa\x01\x1c\x82\xb1\x04\x32\x60\x9e\x1c\xb0\x60\x4a\x7a\x17\x03\xd3\x3b\x50\xcc\xef\x9f\xfe\x44\x40\xf6\x7e\x0b\x90\x3a\xa8\x59\x9a\x12\xc1\x2f\x99\x27\xaf\xb7\x52\x29\xc8\x10\x2a\x87\x79\xa5\x86\x84\x96\x55\x1e\xee\x2f\x17\x5f\x6f\xee\x16\x90\x5e\x3f\xc0\x7d\x3a\x9b\xa5\xd7\x8b\x87\x73\xd8\x4a\xbf\x34\x95\x07\xdc\x60\x0d\x25\xd7\xa5\x92\x28\x60\xcb\xac\x65\xda\xef\xc0\xe4\x84\xf0\xdb\x74\x36\xf9\x9a\x5e\x2f\xd2\x8f\x97\x57\x97\x8b\x07\x30\x16\x3e\x5f\x2e\xae\xa7\xf3\x39\x7c\xbe\x99\x41\x0a\xb7\xe9\x6c\x71\x39\xb9\xbb\x4a\x67\x70\x7b\x37\xbb\xbd\x99\x4f\x13\x98\x23\x59\x85\xf4\xfe\xf5\x98\xe7\x21\x7b\x16\x41\xa0\x67\x52\xb9\x36\x12\x0f\xa6\x02\xb7\x34\x95\x12\xb0\x64\x1b\x04\x8b\x1c\xe5\x06\x05\x30\xe0\xa6\xdc\xfd\x74\x52\x09\x8b\x29\xa3\x8b\xe0\xf3\x8b\x05\x09\x97\x39\x68\xe3\x87\xe0\x10\xe1\x1f\x4b\xef\xcb\xf1\x68\xb4\xdd\x6e\x93\x42\x57\x89\xb1\xc5\x48\xd5\x70\x6e\xf4\xcf\x24\x22\xcc\xd2\xa2\xf3\xcc\xe3\xc2\x32\x8e\x16\x4c\xe5\xcb\xca\x3b\x70\x55\x9e\x4b\x2e\x51\x7b\x90\x3a\x37\x76\x1d\x2a\x05\xbc\x01\x6e\x91\x79\x04\x06\xca\x70\xa6\x00\x1f\x91\x57\xe1\xae\x8e\x74\x28\x57\xcb\xb4\x63\x3c\x9c\xe6\xd6\xac\xc9\xd7\xca\x79\xfa\xe1\x1c\xae\x33\x85\x02\x0a\xd4\xe8\xa4\x83\x4c\x19\xbe\x4a\xa2\x6f\x51\xef\xc0\x18\xaa\x93\xe0\x61\x23\x14\x6a\x63\x8b\x7d\x8b\x90\x55\x52\x09\xa9\x8b\x24\xea\xb5\xd2\x63\xd0\x95\x52\xc3\x28\x40\x28\x63\x56\x55\x99\x72\x6e\xaa\x60\xfb\x9f\xc8\x7d\x0d\xe6\x4a\xe4\x32\xa7\xe2\x60\xdd\xad\x37\xe1\xaa\xd3\x6b\x32\x92\x4f\xa2\xde\x11\xcc\x18\xf2\x4a\x07\x77\x06\x4c\x08\x3b\x04\x91\xc5\xdf\xa2\x5e\x6f\xc3\x2c\x61\xc1\x05\x78\xf3\x15\x1f\xc3\x65\x7c\x1e\xf5\x7a\x32\x87\x81\x5f\x4a\x97\xb4\xc0\xbf\x33\xce\xff\x80\x8b\x8b\x8b\xd0\xd4\xb9\xd4\x28\x62\x20\x88\xde\x73\x62\xf5\x4d\x2f\x63\x8a\x69\x8e\x63\xe8\xbf\x7f\xec\xc3\x5b\x10\x59\x52\xa0\xff\x58\x9f\xd6\xca\x12\x6f\xe6\xde\x4a\x5d\x0c\x3e\xfc\x1a\x0f\xc3\x2b\x6d\xc2\x1b\x68\xc4\xaf\x4d\x27\x5c\xdf\x73\x23\xc2\x75\x63\x73\x2d\x35\x31\xa2\x11\x6a\xa4\x9c\x37\x96\x15\x38\x86\x6f\x4f\xf4\xfd\x44\x5e\x3d\x45\xbd\xa7\xa3\x28\xcf\x6b\xa1\x17\xa2\xdc\x40\x00\x6a\x6f\xbb\x3a\x2f\x24\x75\xea\x61\x02\x02\xde\x8f\x92\x30\x6f\x4d\x39\x49\xc2\x0a\x77\xaf\x67\x82\x2e\xa4\x78\xec\x2e\x56\xb8\x8b\xcf\xa3\x17\x53\x94\x34\x46\xff\x2e\xc5\xe3\xcf\xe6\xeb\xe4\xcd\x51\x5c\xe7\x24\xb5\xb7\x37\x8e\x4f\xe2\x68\xd1\x55\xca\x53\xb9\x4b\xbd\x31\x2b\x22\xae\x25\xc5\x47\xa9\x10\x12\x53\x52\xb6\x5c\xcd\x1c\x19\xa2\x06\xe9\xd1\x32\xa2\x4e\xb3\x41\x4b\x53\x03\x2c\xfa\xca\x6a\xd7\x85\x31\x97\x9a\xa9\x16\xb8\x89\xba\xb7\x8c\xd7\x3d\x53\x9f\x1f\xc4\x92\xfb\xc7\x10\xc5\xe0\xdd\x68\x04\xa9\x07\x72\x11\x4a\x23\xb5\x1f\xc2\x16\x41\x23\x0a\x6a\x7c\x81\xa2\xe2\x3e\xe0\xf5\x37\x4c\x55\xd8\xaf\x9b\x9b\x28\x32\x3c\x35\x15\x4d\x82\x83\xe6\x1f\x06\x03\xd7\x66\x13\x46\x5c\xc6\xf8\x0a\x9a\x86\x33\x56\x16\x52\x47\x4d\x38\x8f\x9a\x8d\x2c\x4a\x08\x38\x98\x15\x72\x45\x49\xa4\x93\x8f\x4c\xc1\x05\x64\xb2\xb8\xd4\xfe\x24\x79\x75\xd0\xdb\xa7\xf1\x1f\x49\xd3\x3c\x89\x23\xc2\x1b\x9c\xc5\x43\xf8\xf0\x6b\x57\x11\xde\x10\x14\xbc\x0e\xe6\xcd\xcb\x50\xd1\x69\x31\x3c\xff\x2c\xa8\xa1\x0e\x7e\x1b\xb4\x26\xae\xca\x28\x1d\xb5\x9f\x21\x8e\xc7\x5d\x7c\xfe\x03\xdc\x63\xdf\x5a\xdc\x26\x34\x09\x13\xe2\x10\x94\x3e\xc3\x77\xc1\xdc\x9d\x43\x01\x6f\x81\xbe\xa4\x26\x55\x4e\xf2\x2f\xcc\xc5\xf0\x37\x68\x24\x6e\xad\xe4\xdf\x59\x52\xe7\xf5\x13\x72\x8b\x6b\x1a\x05\x94\x3a\xce\x94\x42\xdb\x77\x10\x88\x66\xd8\xd4\x60\x48\x32\xae\x4b\xbf\x6b\x07\x84\x67\xb6\x40\xef\x5e\xf7\x26\xe0\xbc\x7b\xd7\xf2\x66\x88\xdf\xae\x44\xb8\xb8\x80\xfe\x64\x36\x4d\x17\xd3\x7e\xd3\x7b\xa3\x11\xdc\x63\x58\x9f\x32\x25\x33\xa1\x76\x20\x50\xa1\xc7\xda\x2e\xa3\x43\x5c\x3b\x1e\x19\xd2\x1e\x44\x1b\x0a\x3e\x4a\xe7\xa5\x2e\xa0\xa6\x97\x2d\x0d\xe3\x06\x2e\x34\x16\x67\x15\x85\xe7\x74\x72\x79\x43\x6b\x88\x45\x22\x23\x1a\x1a\xa1\x47\x99\x92\xdd\xda\x92\x4b\xeb\x3c\x94\x8a\x71\x4c\x08\xaf\x33\xe6\xe5\xa2\x68\xda\x9f\x54\xcf\x42\xdf\x06\xa0\xfd\x54\x64\x8a\xa6\x2a\xa9\x77\x30\x68\x31\xe2\xa8\xd7\xb3\xad\xf4\x01\xf6\xf9\x9e\x47\x9c\xc7\xf2\x90\x45\x68\x1b\xc1\x0d\x12\xef\x06\x0a\xa9\x27\x28\xe9\xfa\xd7\x6f\xcd\xc8\x46\x97\x44\x3d\x7a\x77\x40\x06\xca\x14\xc7\x64\x20\xea\xb0\xf0\xca\x5a\xca\x7f\xc7\xdb\x39\x11\xc3\x9f\x95\xf3\x14\x53\x4b\xe1\x69\x28\xe6\x39\x66\x0d\x3c\x4a\x23\x3a\xfe\x9e\x41\x69\xd8\x85\xe1\x42\xea\x9a\xd1\x56\xaf\x80\xa5\xf1\xa8\xbd\x64\x4a\xed\x28\x0f\x5b\x4b\xbb\x0f\x6d\x3b\x43\x70\x92\xa4\x02\x4d\x05\x51\xa9\xb9\xaa\x44\x5d\x06\xa1\xf8\x1b\x3c\x17\x6c\x3e\x5e\x9a\xd6\xe8\x1c\x2b\x30\xa1\x4a\xca\xe5\x63\xb3\x76\x6a\xe8\xd7\xcc\x38\x88\xfb\x49\x67\xe4\x31\x2f\x29\x53\x24\x6d\x91\x11\xb7\xa7\x42\x58\x74\x6e\x10\x37\x44\xd5\x65\xf6\x7e\x89\x9a\x82\x0f\x1a\xb7\xd0\xed\x33\x8c\x73\xda\xef\xc4\x10\x98\x10\xc4\x87\x27\xbb\x47\xd4\xeb\xb9\xad\xf4\x7c\x09\x41\x93\x29\xf7\xbd\x18\x37\xf5\xcf\x99\x43\x78\x33\xfd\xf7\x62\x72\xf3\x69\x3a\xb9\xb9\x7d\x78\x33\x86\xa3\xb3\xf9\xe5\x7f\xa6\xdd\xd9\xc7\xf4\x2a\xbd\x9e\x4c\xdf\x8c\xc3\x40\x7f\xc6\x21\x6f\x5a\x17\x48\xa1\xf3\x8c\xaf\x92\x12\x71\x35\x78\x7f\xcc\x03\x7b\x07\x7b\xbd\xcc\x22\x5b\x9d\xef\x8d\xa9\x1b\xb4\xd1\xd1\xf2\x34\x5c\xc0\x8b\xc1\x3a\x7f\xd9\x9a\x49\x23\x3f\x68\xd9\x7f\xbf\xbf\x04\xaa\x78\xdd\x8e\xb3\xbf\x6c\x48\xe8\x1d\xc6\x57\x63\x70\x4c\xd1\xda\x2c\xff\x4b\x7f\x77\xf2\xdc\xa1\x1f\x02\x6a\x61\xb6\xc4\x7c\x1d\x6a\x7d\xd3\xe0\x1e\x84\xec\x43\x5c\xd3\xee\x4d\x3e\x88\x3b\x61\x02\xfb\x5e\xf4\xec\x39\x51\xd4\x02\x2e\x5a\xf4\xb7\xe1\xe5\xeb\x81\x3a\x6b\x22\x75\xa2\xe0\x97\x93\xb5\x30\xdc\xaf\x71\x6d\xec\xae\x99\x61\x07\xfe\xfd\x38\xaa\xe9\xd5\x55\x57\x4f\xf4\x41\x45\xd6\x1d\x7c\x9a\x5e\x4d\xbf\xa4\x8b\xe9\x91\xd4\x7c\x91\x2e\x2e\x27\xf5\xd1\x5f\x2e\xbc\x0f\x3f\x5d\x78\xfd\xf9\x7c\x71\x33\x9b\xf6\xc7\xcd\xd7\xd5\x4d\xfa\xa9\xff\x9d\xc2\x66\x75\xfc\x51\xeb\x7a\x73\x6f\xac\xf8\x7f\x3a\xe0\x60\x8d\xcb\xd9\x73\x5b\x5c\xa0\x76\xee\xab\x93\x7f\x49\xc0\x74\xcb\xca\x79\xfd\x4f\xb1\x17\xde\x3f\xcb\xc3\x4f\xd1\x53\xf4\xbf\x00\x00\x00\xff\xff\x3a\xb7\x37\x41\xbf\x10\x00\x00") func prestate_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -213,7 +213,7 @@ func prestate_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "prestate_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe9, 0x79, 0x70, 0x4f, 0xc5, 0x78, 0x57, 0x63, 0x6f, 0x5, 0x31, 0xce, 0x3e, 0x5d, 0xbd, 0x71, 0x4, 0x46, 0x78, 0xcd, 0x1d, 0xcd, 0xb9, 0xd8, 0x10, 0xff, 0xe6, 0xc5, 0x59, 0xb9, 0x25, 0x6e}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd4, 0x9, 0xf9, 0x44, 0x13, 0x31, 0x89, 0xf7, 0x35, 0x9a, 0xc6, 0xf0, 0x86, 0x9d, 0xb2, 0xe3, 0x57, 0xe2, 0xc0, 0xde, 0xc9, 0x3a, 0x4c, 0x4a, 0x94, 0x90, 0xa5, 0x92, 0x2f, 0xbf, 0xc0, 0xb8}} return a, nil } diff --git a/eth/tracers/internal/tracers/prestate_tracer.js b/eth/tracers/internal/tracers/prestate_tracer.js index e0a22bf157..084c04ec46 100644 --- a/eth/tracers/internal/tracers/prestate_tracer.js +++ b/eth/tracers/internal/tracers/prestate_tracer.js @@ -55,7 +55,7 @@ var toBal = bigInt(this.prestate[toHex(ctx.to)].balance.slice(2), 16); this.prestate[toHex(ctx.to)].balance = '0x'+toBal.subtract(ctx.value).toString(16); - this.prestate[toHex(ctx.from)].balance = '0x'+fromBal.add(ctx.value).toString(16); + this.prestate[toHex(ctx.from)].balance = '0x'+fromBal.add(ctx.value).add((ctx.gasUsed + ctx.intrinsicGas) * ctx.gasPrice).toString(16); // Decrement the caller's nonce, and remove empty create targets this.prestate[toHex(ctx.from)].nonce--; diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index 5d806d9026..c9f00d7371 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -27,10 +27,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "gopkg.in/olebedev/go-duktape.v3" + duktape "gopkg.in/olebedev/go-duktape.v3" ) // bigIntegerJS is the minified version of https://github.com/peterolson/BigInteger.js. @@ -316,7 +317,7 @@ type Tracer struct { // New instantiates a new tracer instance. code specifies a Javascript snippet, // which must evaluate to an expression returning an object with 'step', 'fault' // and 'result' functions. -func New(code string) (*Tracer, error) { +func New(code string, txCtx vm.TxContext) (*Tracer, error) { // Resolve any tracers by name and assemble the tracer object if tracer, ok := tracer(code); ok { code = tracer @@ -335,6 +336,8 @@ func New(code string) (*Tracer, error) { depthValue: new(uint), refundValue: new(uint), } + tracer.ctx["gasPrice"] = txCtx.GasPrice + // Set up builtins for this environment tracer.vm.PushGlobalGoFunction("toHex", func(ctx *duktape.Context) int { ctx.PushString(hexutil.Encode(popSlice(ctx))) @@ -546,6 +549,18 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost // Initialize the context if it wasn't done yet if !jst.inited { jst.ctx["block"] = env.Context.BlockNumber.Uint64() + // Compute intrinsic gas + isHomestead := env.ChainConfig().IsHomestead(env.Context.BlockNumber) + isIstanbul := env.ChainConfig().IsIstanbul(env.Context.BlockNumber) + var input []byte + if data, ok := jst.ctx["input"].([]byte); ok { + input = data + } + intrinsicGas, err := core.IntrinsicGas(input, jst.ctx["type"] == "CREATE", isHomestead, isIstanbul) + if err != nil { + return err + } + jst.ctx["intrinsicGas"] = intrinsicGas jst.inited = true } // If tracing was interrupted, set the error and stop @@ -597,8 +612,8 @@ func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost // CaptureEnd is called after the call finishes to finalize the tracing. func (jst *Tracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error { jst.ctx["output"] = output - jst.ctx["gasUsed"] = gasUsed jst.ctx["time"] = t.String() + jst.ctx["gasUsed"] = gasUsed if err != nil { jst.ctx["error"] = err.Error() diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 554b2282f1..f28e14864b 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -17,7 +17,6 @@ package tracers import ( - "bytes" "encoding/json" "errors" "math/big" @@ -50,94 +49,77 @@ type dummyStatedb struct { func (*dummyStatedb) GetRefund() uint64 { return 1337 } -func runTrace(tracer *Tracer) (json.RawMessage, error) { - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) - - contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) - contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} - - _, err := env.Interpreter().Run(contract, []byte{}, false) - if err != nil { - return nil, err - } - return tracer.GetResult() +type vmContext struct { + blockCtx vm.BlockContext + txCtx vm.TxContext } -// TestRegressionPanicSlice tests that we don't panic on bad arguments to memory access -func TestRegressionPanicSlice(t *testing.T) { - tracer, err := New("{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}") - if err != nil { - t.Fatal(err) - } - if _, err = runTrace(tracer); err != nil { - t.Fatal(err) - } -} - -// TestRegressionPanicSlice tests that we don't panic on bad arguments to stack peeks -func TestRegressionPanicPeek(t *testing.T) { - tracer, err := New("{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}") - if err != nil { - t.Fatal(err) - } - if _, err = runTrace(tracer); err != nil { - t.Fatal(err) - } -} - -// TestRegressionPanicSlice tests that we don't panic on bad arguments to memory getUint -func TestRegressionPanicGetUint(t *testing.T) { - tracer, err := New("{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}") - if err != nil { - t.Fatal(err) - } - if _, err = runTrace(tracer); err != nil { - t.Fatal(err) - } +func testCtx() *vmContext { + return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} } -func TestTracing(t *testing.T) { - tracer, err := New("{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}") - if err != nil { - t.Fatal(err) - } - - ret, err := runTrace(tracer) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(ret, []byte("3")) { - t.Errorf("Expected return value to be 3, got %s", string(ret)) - } -} - -func TestStack(t *testing.T) { - tracer, err := New("{depths: [], step: function(log) { this.depths.push(log.stack.length()); }, fault: function() {}, result: function() { return this.depths; }}") - if err != nil { - t.Fatal(err) - } +func runTrace(tracer *Tracer, vmctx *vmContext) (json.RawMessage, error) { + env := vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) + var ( + startGas uint64 = 10000 + value = big.NewInt(0) + ) + contract := vm.NewContract(account{}, account{}, value, startGas) + contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} - ret, err := runTrace(tracer) + tracer.CaptureStart(contract.Caller(), contract.Address(), false, []byte{}, startGas, value) + ret, err := env.Interpreter().Run(contract, []byte{}, false) + tracer.CaptureEnd(ret, startGas-contract.Gas, 1, err) if err != nil { - t.Fatal(err) - } - if !bytes.Equal(ret, []byte("[0,1,2]")) { - t.Errorf("Expected return value to be [0,1,2], got %s", string(ret)) + return nil, err } + return tracer.GetResult() } -func TestOpcodes(t *testing.T) { - tracer, err := New("{opcodes: [], step: function(log) { this.opcodes.push(log.op.toString()); }, fault: function() {}, result: function() { return this.opcodes; }}") - if err != nil { - t.Fatal(err) - } - - ret, err := runTrace(tracer) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(ret, []byte("[\"PUSH1\",\"PUSH1\",\"STOP\"]")) { - t.Errorf("Expected return value to be [\"PUSH1\",\"PUSH1\",\"STOP\"], got %s", string(ret)) +func TestTracer(t *testing.T) { + execTracer := func(code string) []byte { + t.Helper() + ctx := &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} + tracer, err := New(code, ctx.txCtx) + if err != nil { + t.Fatal(err) + } + ret, err := runTrace(tracer, ctx) + if err != nil { + t.Fatal(err) + } + return ret + } + for i, tt := range []struct { + code string + want string + }{ + { // tests that we don't panic on bad arguments to memory access + code: "{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}", + want: `[{},{},{}]`, + }, { // tests that we don't panic on bad arguments to stack peeks + code: "{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}", + want: `["0","0","0"]`, + }, { // tests that we don't panic on bad arguments to memory getUint + code: "{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}", + want: `["0","0","0"]`, + }, { // tests some general counting + code: "{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}", + want: `3`, + }, { // tests that depth is reported correctly + code: "{depths: [], step: function(log) { this.depths.push(log.stack.length()); }, fault: function() {}, result: function() { return this.depths; }}", + want: `[0,1,2]`, + }, { // tests to-string of opcodes + code: "{opcodes: [], step: function(log) { this.opcodes.push(log.op.toString()); }, fault: function() {}, result: function() { return this.opcodes; }}", + want: `["PUSH1","PUSH1","STOP"]`, + }, { // tests intrinsic gas + code: "{depths: [], step: function() {}, fault: function() {}, result: function(ctx) { return ctx.gasPrice+'.'+ctx.gasUsed+'.'+ctx.intrinsicGas; }}", + want: `"100000.6.21000"`, + }, + } { + if have := execTracer(tt.code); tt.want != string(have) { + t.Errorf("testcase %d: expected return value to be %s got %s\n\tcode: %v", i, tt.want, string(have), tt.code) + } } } @@ -145,7 +127,8 @@ func TestHalt(t *testing.T) { t.Skip("duktape doesn't support abortion") timeout := errors.New("stahp") - tracer, err := New("{step: function() { while(1); }, result: function() { return null; }}") + vmctx := testCtx() + tracer, err := New("{step: function() { while(1); }, result: function() { return null; }}", vmctx.txCtx) if err != nil { t.Fatal(err) } @@ -155,17 +138,17 @@ func TestHalt(t *testing.T) { tracer.Stop(timeout) }() - if _, err = runTrace(tracer); err.Error() != "stahp in server-side tracer function 'step'" { + if _, err = runTrace(tracer, vmctx); err.Error() != "stahp in server-side tracer function 'step'" { t.Errorf("Expected timeout error, got %v", err) } } func TestHaltBetweenSteps(t *testing.T) { - tracer, err := New("{step: function() {}, fault: function() {}, result: function() { return null; }}") + vmctx := testCtx() + tracer, err := New("{step: function() {}, fault: function() {}, result: function() { return null; }}", vmctx.txCtx) if err != nil { t.Fatal(err) } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index b749d832b4..9dc4c69631 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -173,7 +173,7 @@ func TestPrestateTracerCreate2(t *testing.T) { _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := New("prestateTracer") + tracer, err := New("prestateTracer", txContext) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -248,7 +248,7 @@ func TestCallTracer(t *testing.T) { _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := New("callTracer") + tracer, err := New("callTracer", txContext) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } From b13e9c4e3d603955f92c1542a5e86c740f43f33e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sun, 27 Dec 2020 21:58:39 +0100 Subject: [PATCH 024/709] tests/fuzzers: fix false positive in bitutil fuzzer (#22076) --- tests/fuzzers/bitutil/compress_fuzz.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/fuzzers/bitutil/compress_fuzz.go b/tests/fuzzers/bitutil/compress_fuzz.go index 0b77b2dc9e..5903cf2f93 100644 --- a/tests/fuzzers/bitutil/compress_fuzz.go +++ b/tests/fuzzers/bitutil/compress_fuzz.go @@ -51,7 +51,19 @@ func fuzzDecode(data []byte) int { if err != nil { return 0 } - if comp := bitutil.CompressBytes(blob); !bytes.Equal(comp, data) { + // re-compress it (it's OK if the re-compressed differs from the + // original - the first input may not have been compressed at all) + comp := bitutil.CompressBytes(blob) + if len(comp) > len(blob) { + // After compression, it must be smaller or equal + panic("bad compression") + } + // But decompressing it once again should work + decomp, err := bitutil.DecompressBytes(data, 1024) + if err != nil { + panic(err) + } + if !bytes.Equal(decomp, blob) { panic("content mismatch") } return 1 From 2f8100615ad1d8335e38024221143163f09d54fe Mon Sep 17 00:00:00 2001 From: Suriyaa Sundararuban Date: Sun, 27 Dec 2020 22:01:28 +0100 Subject: [PATCH 025/709] cmd/geth: replace wiki links with new doc pages (#22071) --- cmd/geth/consolecmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index cbecbe0a5f..7822c73b31 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -43,7 +43,7 @@ var ( Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`, +See https://geth.ethereum.org/docs/interface/javascript-console.`, } attachCommand = cli.Command{ @@ -56,7 +56,7 @@ See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`, Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console. +See https://geth.ethereum.org/docs/interface/javascript-console. This command allows to open a console on a running geth node.`, } @@ -69,7 +69,7 @@ This command allows to open a console on a running geth node.`, Category: "CONSOLE COMMANDS", Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp -JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console`, +JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console`, } ) From 0a09a39325814a2acbf4486c74b9aa9e6cff04d6 Mon Sep 17 00:00:00 2001 From: Suriyaa Sundararuban Date: Sun, 27 Dec 2020 22:09:05 +0100 Subject: [PATCH 026/709] eth/filters: replace wiki links with new doc pages (#22070) --- eth/filters/api.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 664f229a04..b6f974c3ba 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -101,7 +101,7 @@ func (api *PublicFilterAPI) timeoutLoop() { // It is part of the filter package because this filter can be used through the // `eth_getFilterChanges` polling method that is also used for log filters. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newpendingtransactionfilter +// https://eth.wiki/json-rpc/API#eth_newpendingtransactionfilter func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { var ( pendingTxs = make(chan []common.Hash) @@ -171,7 +171,7 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su // NewBlockFilter creates a filter that fetches blocks that are imported into the chain. // It is part of the filter package since polling goes with eth_getFilterChanges. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newblockfilter +// https://eth.wiki/json-rpc/API#eth_newblockfilter func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { var ( headers = make(chan *types.Header) @@ -287,7 +287,7 @@ type FilterCriteria ethereum.FilterQuery // // In case "fromBlock" > "toBlock" an error is returned. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter +// https://eth.wiki/json-rpc/API#eth_newfilter func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { logs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), logs) @@ -322,7 +322,7 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { // GetLogs returns logs matching the given argument that are stored within the state. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs +// https://eth.wiki/json-rpc/API#eth_getlogs func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { var filter *Filter if crit.BlockHash != nil { @@ -351,7 +351,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ // UninstallFilter removes the filter with the given filter id. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_uninstallfilter +// https://eth.wiki/json-rpc/API#eth_uninstallfilter func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { api.filtersMu.Lock() f, found := api.filters[id] @@ -369,7 +369,7 @@ func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { // GetFilterLogs returns the logs for the filter with the given id. // If the filter could not be found an empty array of logs is returned. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs +// https://eth.wiki/json-rpc/API#eth_getfilterlogs func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) { api.filtersMu.Lock() f, found := api.filters[id] @@ -410,7 +410,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty // For pending transaction and block filters the result is []common.Hash. // (pending)Log filters return []Log. // -// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterchanges +// https://eth.wiki/json-rpc/API#eth_getfilterchanges func (api *PublicFilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { api.filtersMu.Lock() defer api.filtersMu.Unlock() From ab0979f9306e87025d821b2629985e8bffe130ef Mon Sep 17 00:00:00 2001 From: Suriyaa Sundararuban Date: Sun, 27 Dec 2020 22:18:57 +0100 Subject: [PATCH 027/709] signer: docs - replace wiki links with new doc pages (#22069) --- signer/fourbyte/abi_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/signer/fourbyte/abi_test.go b/signer/fourbyte/abi_test.go index 314c12735b..68c027ecea 100644 --- a/signer/fourbyte/abi_test.go +++ b/signer/fourbyte/abi_test.go @@ -68,7 +68,7 @@ func TestNewUnpacker(t *testing.T) { [10]byte{49, 50, 51, 52, 53, 54, 55, 56, 57, 48}, common.Hex2Bytes("48656c6c6f2c20776f726c6421"), }, - }, { // https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI#examples + }, { // https://docs.soliditylang.org/en/develop/abi-spec.html#examples `[{"type":"function","name":"sam","inputs":[{"type":"bytes"},{"type":"bool"},{"type":"uint256[]"}]}]`, // "dave", true and [1,2,3] "a5643bf20000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000464617665000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003", @@ -124,7 +124,7 @@ func TestCalldataDecoding(t *testing.T) { "42958b5400000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000042", // Too short compareAndApprove "a52c101e00ff0000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000042", - // From https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI + // From https://docs.soliditylang.org/en/develop/abi-spec.html // contains a bool with illegal values "a5643bf20000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000464617665000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003", } { @@ -135,7 +135,7 @@ func TestCalldataDecoding(t *testing.T) { } // Expected success for i, hexdata := range []string{ - // From https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI + // From https://docs.soliditylang.org/en/develop/abi-spec.html "a5643bf20000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000464617665000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003", "a52c101e0000000000000000000000000000000000000000000000000000000000000012", "a52c101eFFffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", From 653e8b9dd9b7a1565b0e26c71ad70a83803ad529 Mon Sep 17 00:00:00 2001 From: jk-jeongkyun <45347815+jeongkyun-oh@users.noreply.github.com> Date: Mon, 28 Dec 2020 06:26:42 +0900 Subject: [PATCH 028/709] eth/downloader: remove unnecessary condition (#22052) --- eth/downloader/queue.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 2150842f8e..ac7edc2c68 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -893,9 +893,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, return accepted, nil } // If none of the data was good, it's a stale delivery - if errors.Is(failure, errInvalidChain) { - return accepted, failure - } if accepted > 0 { return accepted, fmt.Errorf("partial failure: %v", failure) } From c17a7733df3aa7f68d4e0ff5ce9d5c2919284faa Mon Sep 17 00:00:00 2001 From: Suriyaa Sundararuban Date: Sun, 27 Dec 2020 22:28:08 +0100 Subject: [PATCH 029/709] docs: replace wiki links with new doc pages in readme.md (#22065) (#22066) --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 19357355a8..57d431db1e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ archives are published at https://geth.ethereum.org/downloads/. ## Building the source -For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki. +For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). Building `geth` requires both a Go (version 1.13 or later) and a C compiler. You can install them using your favourite package manager. Once the dependencies are installed, run @@ -36,18 +36,18 @@ directory. | Command | Description | | :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. | -| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | +| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://eth.wiki/json-rpc/API) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | | `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | ## Running `geth` Going through all the possible command line flags is out of scope here (please consult our -[CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options)), +[CLI Wiki page](https://geth.ethereum.org/docs/interface/command-line-options)), but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own `geth` instance. @@ -66,9 +66,9 @@ This command will: * Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag), causing it to download more data in exchange for avoiding processing the entire history of the Ethereum network, which is very CPU intensive. - * Start up `geth`'s built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console), - (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API) - as well as `geth`'s own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs). + * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), + (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://web3js.readthedocs.io/en/) + as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). This tool is optional and if you leave it out you can always attach to an already running `geth` instance with `geth attach`. @@ -170,8 +170,8 @@ accessible from the outside. As a developer, sooner rather than later you'll want to start interacting with `geth` and the Ethereum network via your own programs and not manually through the console. To aid -this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) -and [`geth` specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). +this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) +and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows). @@ -277,7 +277,7 @@ $ bootnode --genkey=boot.key $ bootnode --nodekey=boot.key ``` -With the bootnode online, it will display an [`enode` URL](https://github.com/ethereum/wiki/wiki/enode-url-format) +With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) that other nodes can use to connect to it and exchange peer information. Make sure to replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. @@ -344,7 +344,7 @@ Please make sure your contributions adhere to our coding guidelines: * Commit messages should be prefixed with the package(s) they modify. * E.g. "eth, rpc: make trace configs optional" -Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide) +Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) for more details on configuring your environment, managing project dependencies, and testing procedures. From a425a47ddcb6078e2ae6ab062bb73f2c0939fd1d Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 28 Dec 2020 05:38:16 +0800 Subject: [PATCH 030/709] core/rawdb, eth/protocols : Method name typo fix (#22026) --- core/rawdb/accessors_snapshot.go | 4 ++-- eth/protocols/snap/sync.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go index 0a91d9353b..c3616ba3aa 100644 --- a/core/rawdb/accessors_snapshot.go +++ b/core/rawdb/accessors_snapshot.go @@ -176,8 +176,8 @@ func DeleteSnapshotRecoveryNumber(db ethdb.KeyValueWriter) { } } -// ReadSanpshotSyncStatus retrieves the serialized sync status saved at shutdown. -func ReadSanpshotSyncStatus(db ethdb.KeyValueReader) []byte { +// ReadSnapshotSyncStatus retrieves the serialized sync status saved at shutdown. +func ReadSnapshotSyncStatus(db ethdb.KeyValueReader) []byte { data, _ := db.Get(snapshotSyncStatusKey) return data } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 679b328283..437b0caab4 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -614,7 +614,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { func (s *Syncer) loadSyncStatus() { var progress syncProgress - if status := rawdb.ReadSanpshotSyncStatus(s.db); status != nil { + if status := rawdb.ReadSnapshotSyncStatus(s.db); status != nil { if err := json.Unmarshal(status, &progress); err != nil { log.Error("Failed to decode snap sync status", "err", err) } else { From 0a3993c558616868e35f9730e92c704ac16ee437 Mon Sep 17 00:00:00 2001 From: yumiel yoomee1313 Date: Wed, 30 Dec 2020 21:10:11 +0900 Subject: [PATCH 031/709] accounts/abi/bind: fix erroneous test (#22053) closes #22049 --- accounts/abi/bind/bind_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 4a504516bb..db87703d03 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1640,11 +1640,7 @@ var bindTests = []struct { contract NewFallbacks { event Fallback(bytes data); fallback() external { - bytes memory data; - assembly { - calldatacopy(data, 0, calldatasize()) - } - emit Fallback(data); + emit Fallback(msg.data); } event Received(address addr, uint value); @@ -1653,7 +1649,7 @@ var bindTests = []struct { } } `, - []string{"60806040523480156100115760006000fd5b50610017565b61016e806100266000396000f3fe60806040526004361061000d575b36610081575b7f88a5966d370b9919b20f3e2c13ff65706f196a4e32cc2c12bf57088f885258743334604051808373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020018281526020019250505060405180910390a15b005b34801561008e5760006000fd5b505b606036600082377f9043988963722edecc2099c75b0af0ff76af14ffca42ed6bce059a20a2a9f986816040518080602001828103825283818151815260200191508051906020019080838360005b838110156100fa5780820151818401525b6020810190506100de565b50505050905090810190601f1680156101275780820380516001836020036101000a031916815260200191505b509250505060405180910390a1505b00fea26469706673582212205643ca37f40c2b352dc541f42e9e6720de065de756324b7fcc9fb1d67eda4a7d64736f6c63430006040033"}, + []string{"6080604052348015600f57600080fd5b506101078061001f6000396000f3fe608060405236605f577f88a5966d370b9919b20f3e2c13ff65706f196a4e32cc2c12bf57088f885258743334604051808373ffffffffffffffffffffffffffffffffffffffff1681526020018281526020019250505060405180910390a1005b348015606a57600080fd5b507f9043988963722edecc2099c75b0af0ff76af14ffca42ed6bce059a20a2a9f98660003660405180806020018281038252848482818152602001925080828437600081840152601f19601f820116905080830192505050935050505060405180910390a100fea26469706673582212201f994dcfbc53bf610b19176f9a361eafa77b447fd9c796fa2c615dfd0aaf3b8b64736f6c634300060c0033"}, []string{`[{"anonymous":false,"inputs":[{"indexed":false,"internalType":"bytes","name":"data","type":"bytes"}],"name":"Fallback","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"addr","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Received","type":"event"},{"stateMutability":"nonpayable","type":"fallback"},{"stateMutability":"payable","type":"receive"}]`}, ` "bytes" @@ -1701,6 +1697,7 @@ var bindTests = []struct { } // Test fallback function + gotEvent = false opts.Value = nil calldata := []byte{0x01, 0x02, 0x03} c.Fallback(opts, calldata) From 167ff563d16a405a89ce449fdb34eb6d99631053 Mon Sep 17 00:00:00 2001 From: Melvin Junhee Woo Date: Mon, 4 Jan 2021 17:07:43 +0900 Subject: [PATCH 032/709] core/state/snapshot: gethring -> gathering typo (#22104) --- core/state/snapshot/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 4a2fa78d3a..17f1ca6078 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -151,7 +151,7 @@ func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorSta // generate is a background thread that iterates over the state and storage tries, // constructing the state snapshot. All the arguments are purely for statistics -// gethering and logging, since the method surfs the blocks as they arrive, often +// gathering and logging, since the method surfs the blocks as they arrive, often // being restarted. func (dl *diskLayer) generate(stats *generatorStats) { // If a database wipe is in operation, wait until it's done From f83fc302a504919f6668060110cbb8b64c26dd07 Mon Sep 17 00:00:00 2001 From: Vie Date: Mon, 4 Jan 2021 18:52:23 +0800 Subject: [PATCH 033/709] cmd/geth: update copyright year (#22099) --- cmd/geth/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e587a5f86d..0d1b569b8b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -218,7 +218,7 @@ func init() { // Initialize the CLI app and start Geth app.Action = geth app.HideVersion = true // we have a command to print the version - app.Copyright = "Copyright 2013-2020 The go-ethereum Authors" + app.Copyright = "Copyright 2013-2021 The go-ethereum Authors" app.Commands = []cli.Command{ // See chaincmd.go: initCommand, From 47820ef726a7b08ea2e22baff8fff64231c3046b Mon Sep 17 00:00:00 2001 From: Suriyaa Sundararuban Date: Mon, 4 Jan 2021 11:58:51 +0100 Subject: [PATCH 034/709] .github: Replace wiki links with new doc pages (#22065) (#22068) --- .github/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f87996cdcb..a08542df25 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -30,11 +30,11 @@ Please make sure your contributions adhere to our coding guidelines: Before you submit a feature request, please check and make sure that it isn't possible through some other means. The JavaScript-enabled console is a powerful feature in the right hands. Please check our -[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info +[Geth documentation page](https://geth.ethereum.org/docs/) for more info and help. ## Configuration, dependencies, and tests -Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide) +Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) for more details on configuring your environment, managing project dependencies and testing procedures. From 5c2a7ce2ccace9f453bcd320b4ac52e1f5ce3ab2 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:39:25 +0100 Subject: [PATCH 035/709] node: rename startNetworking to openEndpoints (#22105) --- node/node.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/node/node.go b/node/node.go index c66ebb89d0..b58594ef1d 100644 --- a/node/node.go +++ b/node/node.go @@ -159,12 +159,13 @@ func (n *Node) Start() error { return ErrNodeStopped } n.state = runningState - err := n.startNetworking() + // open networking and RPC endpoints + err := n.openEndpoints() lifecycles := make([]Lifecycle, len(n.lifecycles)) copy(lifecycles, n.lifecycles) n.lock.Unlock() - // Check if networking startup failed. + // Check if endpoint startup failed. if err != nil { n.doClose(nil) return err @@ -247,12 +248,14 @@ func (n *Node) doClose(errs []error) error { } } -// startNetworking starts all network endpoints. -func (n *Node) startNetworking() error { +// openEndpoints starts all network and RPC endpoints. +func (n *Node) openEndpoints() error { + // start networking endpoints n.log.Info("Starting peer-to-peer node", "instance", n.server.Name) if err := n.server.Start(); err != nil { return convertFileLockError(err) } + // start RPC endpoints err := n.startRPC() if err != nil { n.stopRPC() From 1951e20d1040627faf3b6722c88ddf0e86ecf50e Mon Sep 17 00:00:00 2001 From: Suriyaa Sundararuban Date: Mon, 4 Jan 2021 12:42:47 +0100 Subject: [PATCH 036/709] SECURITY.md: link to release page (#22067) Add links to go-ethereum's GitHub release page. Co-authored-by: Felix Lange --- SECURITY.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index bc54ede42f..bdce7b8d2a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,31 +2,29 @@ ## Supported Versions -Please see Releases. We recommend to use the most recent released version. +Please see [Releases](https://github.com/ethereum/go-ethereum/releases). We recommend using the [most recently released version](https://github.com/ethereum/go-ethereum/releases/latest). ## Audit reports Audit reports are published in the `docs` folder: https://github.com/ethereum/go-ethereum/tree/master/docs/audits - | Scope | Date | Report Link | | ------- | ------- | ----------- | | `geth` | 20170425 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2017-04-25_Geth-audit_Truesec.pdf) | | `clef` | 20180914 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2018-09-14_Clef-audit_NCC.pdf) | - - ## Reporting a Vulnerability **Please do not file a public ticket** mentioning the vulnerability. -To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. +To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publically disclosed security vulnerabilities. + +Use the built-in `geth version-check` feature to check whether the software is affected by any known vulnerability. This command will fetch the latest [`vulnerabilities.json`](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json) file which contains known security vulnerabilities concerning `geth`, and cross-check the data against its own version number. The following key may be used to communicate sensitive information to developers. Fingerprint: `AE96 ED96 9E47 9B00 84F3 E17F E88D 3334 FA5F 6A0A` - ``` -----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 From e4571d8c12be5aaa380ee58ea35a5617823e5324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 4 Jan 2021 13:58:46 +0200 Subject: [PATCH 037/709] cmd: support v1.1 Twitter API in faucet, fix puppeth --- cmd/faucet/README.md | 50 ++++++++++++++++++ cmd/faucet/faucet.go | 100 ++++++++++++++++++++++------------- cmd/puppeth/module_faucet.go | 2 +- cmd/puppeth/wizard_faucet.go | 18 ++----- 4 files changed, 118 insertions(+), 52 deletions(-) create mode 100644 cmd/faucet/README.md diff --git a/cmd/faucet/README.md b/cmd/faucet/README.md new file mode 100644 index 0000000000..364689a782 --- /dev/null +++ b/cmd/faucet/README.md @@ -0,0 +1,50 @@ +# Faucet + +The `faucet` is a simplistic web application with the goal of distributing small amounts of Ether in private and test networks. + +Users need to post their Ethereum addresses to fund in a Twitter status update or public Facebook post and share the link to the faucet. The faucet will in turn deduplicate user requests and send the Ether. After a funding round, the faucet prevents the same user requesting again for a pre-configured amount of time, proportional to the amount of Ether requested. + +## Operation + +The `faucet` is a single binary app (everything included) with all configurations set via command line flags and a few files. + +First thing's first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set: + +- `--genesis` is a path to a file containin the network `genesis.json` +- `--network` is the devp2p network id used during connection +- `--bootnodes` is a list of `enode://` ids to join the network through + +The `faucet` will use the `les` protocol to join the configured Ethereum network and will store its data in `$HOME/.faucet` (currently not configurable). + +## Funding + +To be able to distribute funds, the `faucet` needs access to an already funded Ethereum account. This can be configured via: + +- `--account.json` is a path to the Ethereum account's JSON key file +- `--account.pass` is a path to a text file with the decryption passphrase + +The faucet is able to distribute various amounts of Ether in exchange for various timeouts. These can be configured via: + +- `--faucet.amount` is the number of Ethers to send by default +- `--faucet.minutes` is the time to wait before allowing a rerequest +- `--faucet.tiers` is the funding tiers to support (x3 time, x2.5 funds) + +## Sybil protection + +To prevent the same user from exhausting funds in a loop, the `faucet` ties requests to social networks and captcha resolvers. + +Captcha protection uses Google's invisible ReCaptcha, thus the `faucet` needs to run on a live domain. The domain needs to be registered in Google's systems to retrieve the captcha API token and secrets. After doing so, captcha protection may be enabled via: + +- `--captcha.token` is the API token for ReCaptcha +- `--captcha.secret` is the API secret for ReCaptcha + +Sybil protection via Twitter requires an API key as of 15th December, 2020. To obtain it, a Twitter user must be upgraded to developer status and a new Twitter App deployed with it. The app's `Bearer` token is required by the faucet to retrieve tweet data: + +- `--twitter.token` is the Bearer token for `v2` API access +- `--twitter.token.v1` is the Bearer token for `v1` API access + +Sybil protection via Facebook uses the website to directly download post data thus does not currently require an API configuration. + +## Miscellaneous + +Beside the above - mostly essential - CLI flags, there are a number that can be used to fine tune the `faucet`'s operation. Please see `faucet --help` for a full list. \ No newline at end of file diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 008cb14296..79a84fd24e 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -84,7 +84,8 @@ var ( noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication") logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet") - twitterBearerToken = flag.String("twitter.token", "", "Twitter bearer token to authenticate with the twitter API") + twitterTokenFlag = flag.String("twitter.token", "", "Bearer token to authenticate with the v2 Twitter API") + twitterTokenV1Flag = flag.String("twitter.token.v1", "", "Bearer token to authenticate with the v1.1 Twitter API") ) var ( @@ -245,6 +246,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u cfg.NetworkId = network cfg.Genesis = genesis utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash()) + lesBackend, err := les.New(stack, &cfg) if err != nil { return nil, fmt.Errorf("Failed to register the Ethereum service: %w", err) @@ -388,8 +390,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { if err = conn.ReadJSON(&msg); err != nil { return } - if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://gist.github.com/") && !strings.HasPrefix(msg.URL, "https://twitter.com/") && - !strings.HasPrefix(msg.URL, "https://plus.google.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") { + if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://twitter.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") { if err = sendError(conn, errors.New("URL doesn't link to supported services")); err != nil { log.Warn("Failed to send URL error to client", "err", err) return @@ -451,21 +452,8 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { address common.Address ) switch { - case strings.HasPrefix(msg.URL, "https://gist.github.com/"): - if err = sendError(conn, errors.New("GitHub authentication discontinued at the official request of GitHub")); err != nil { - log.Warn("Failed to send GitHub deprecation to client", "err", err) - return - } - continue - case strings.HasPrefix(msg.URL, "https://plus.google.com/"): - //lint:ignore ST1005 Google is a company name and should be capitalized. - if err = sendError(conn, errors.New("Google+ authentication discontinued as the service was sunset")); err != nil { - log.Warn("Failed to send Google+ deprecation to client", "err", err) - return - } - continue case strings.HasPrefix(msg.URL, "https://twitter.com/"): - id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken) + id, username, avatar, address, err = authTwitter(msg.URL, *twitterTokenV1Flag, *twitterTokenFlag) case strings.HasPrefix(msg.URL, "https://www.facebook.com/"): username, avatar, address, err = authFacebook(msg.URL) id = username @@ -690,23 +678,31 @@ func sendSuccess(conn *websocket.Conn, msg string) error { // authTwitter tries to authenticate a faucet request using Twitter posts, returning // the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. -func authTwitter(url string, token string) (string, string, string, common.Address, error) { +func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, common.Address, error) { // Ensure the user specified a meaningful URL, no fancy nonsense parts := strings.Split(url, "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") } - + // Strip any query parameters from the tweet id and ensure it's numeric + tweetID := strings.Split(parts[len(parts)-1], "?")[0] + if !regexp.MustCompile("^[0-9]+$").MatchString(tweetID) { + return "", "", "", common.Address{}, errors.New("Invalid Tweet URL") + } // Twitter's API isn't really friendly with direct links. // It is restricted to 300 queries / 15 minute with an app api key. // Anything more will require read only authorization from the users and that we want to avoid. - // If twitter bearer token is provided, use the twitter api - if token != "" { - return authTwitterWithToken(parts[len(parts)-1], token) + // If Twitter bearer token is provided, use the API, selecting the version + // the user would prefer (currently there's a limit of 1 v2 app / developer + // but unlimited v1.1 apps). + switch { + case tokenV1 != "": + return authTwitterWithTokenV1(tweetID, tokenV1) + case tokenV2 != "": + return authTwitterWithTokenV2(tweetID, tokenV2) } - // Twiter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. @@ -742,19 +738,49 @@ func authTwitter(url string, token string) (string, string, string, common.Addre return username + "@twitter", username, avatar, address, nil } -// authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning -// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. -func authTwitterWithToken(tweetID string, token string) (string, string, string, common.Address, error) { - // Strip any query parameters from the tweet id - sanitizedTweetID := strings.Split(tweetID, "?")[0] +// authTwitterWithTokenV1 tries to authenticate a faucet request using Twitter's v1 +// API, returning the user id, username, avatar URL and Ethereum address to fund on +// success. +func authTwitterWithTokenV1(tweetID string, token string) (string, string, string, common.Address, error) { + // Query the tweet details from Twitter + url := fmt.Sprintf("https://api.twitter.com/1.1/statuses/show.json?id=%s", tweetID) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", "", common.Address{}, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", "", common.Address{}, err + } + defer res.Body.Close() - // Ensure numeric tweetID - if !regexp.MustCompile("^[0-9]+$").MatchString(sanitizedTweetID) { - return "", "", "", common.Address{}, errors.New("Invalid Tweet URL") + var result struct { + Text string `json:"text"` + User struct { + ID string `json:"id_str"` + Username string `json:"screen_name"` + Avatar string `json:"profile_image_url"` + } `json:"user"` } + err = json.NewDecoder(res.Body).Decode(&result) + if err != nil { + return "", "", "", common.Address{}, err + } + address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Text)) + if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") + } + return result.User.ID + "@twitter", result.User.Username, result.User.Avatar, address, nil +} +// authTwitterWithTokenV2 tries to authenticate a faucet request using Twitter's v2 +// API, returning the user id, username, avatar URL and Ethereum address to fund on +// success. +func authTwitterWithTokenV2(tweetID string, token string) (string, string, string, common.Address, error) { // Query the tweet details from Twitter - url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID) + url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", tweetID) req, err := http.NewRequest("GET", url, nil) if err != nil { return "", "", "", common.Address{}, err @@ -769,15 +795,13 @@ func authTwitterWithToken(tweetID string, token string) (string, string, string, var result struct { Data struct { AuthorID string `json:"author_id"` - ID string `json:"id"` Text string `json:"text"` } `json:"data"` Includes struct { Users []struct { - ProfileImageURL string `json:"profile_image_url"` - Username string `json:"username"` - ID string `json:"id"` - Name string `json:"name"` + ID string `json:"id"` + Username string `json:"username"` + Avatar string `json:"profile_image_url"` } `json:"users"` } `json:"includes"` } @@ -792,7 +816,7 @@ func authTwitterWithToken(tweetID string, token string) (string, string, string, //lint:ignore ST1005 This error is to be displayed in the browser return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") } - return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil + return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].Avatar, address, nil } // authFacebook tries to authenticate a faucet request using Facebook posts, diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 2527e137f2..88cb80ae4c 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -46,7 +46,7 @@ ENTRYPOINT [ \ "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}", \ "--account.json", "/account.json", "--account.pass", "/account.pass" \ {{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}} \ - {{if .TwitterToken}}, "--twitter.token", "{{.TwitterToken}}", + {{if .TwitterToken}}, "--twitter.token.v1", "{{.TwitterToken}}"{{end}} \ ]` // faucetComposefile is the docker-compose.yml file required to deploy and maintain diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 47e05cd9c1..65d4e8b8ed 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -102,11 +102,10 @@ func (w *wizard) deployFaucet() { infos.captchaSecret = w.readPassword() } } - - // Accessing the twitter api requires a bearer token, request it + // Accessing the Twitter API requires a bearer token, request it if infos.twitterToken != "" { fmt.Println() - fmt.Println("Reuse previous twitter API Bearer token (y/n)? (default = yes)") + fmt.Println("Reuse previous Twitter API token (y/n)? (default = yes)") if !w.readDefaultYesNo(true) { infos.twitterToken = "" } @@ -114,17 +113,10 @@ func (w *wizard) deployFaucet() { if infos.twitterToken == "" { // No previous twitter token (or old one discarded) fmt.Println() - fmt.Println("Enable twitter API (y/n)? (default = no)") - if !w.readDefaultYesNo(false) { - log.Warn("The faucet will fallback to using direct calls") - } else { - // Twitter api explicitly requested, read the bearer token - fmt.Println() - fmt.Printf("What is the twitter API Bearer token?\n") - infos.twitterToken = w.readString() - } + fmt.Println() + fmt.Printf("What is the Twitter API app Bearer token?\n") + infos.twitterToken = w.readString() } - // Figure out where the user wants to store the persistent data fmt.Println() if infos.node.datadir == "" { From 9584f56b9d2fe950b8fa70f5c7398de404f6b71c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Jan 2021 10:44:33 +0100 Subject: [PATCH 038/709] miner: avoid sleeping in miner (#22108) This PR removes a logic in the miner, which was originally intended to help temporary testnets based on ethash from "running off into the future". If the difficulty was low, and a few computers started mining several blocks per second, the ethash rules (which demand 1s delay between blocks) would push the blocktimes further and further away. The solution was to make the miner sleep while this happened. Nowadays, this problem is solved instead by PoA chains, and it's recommended to let testnets and devnets be based on clique instead. The existing logic is problematic, since it can cause stalls within the miner making it difficult for remote workers to submit work if the channel is blocked on a sleep. Credits to Saar Tochner for reporting this via the bug bounty --- miner/worker.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 5f07affdc4..2c5032c656 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -861,13 +861,6 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) if parent.Time() >= uint64(timestamp) { timestamp = int64(parent.Time() + 1) } - // this will ensure we're not going off too far in the future - if now := time.Now().Unix(); timestamp > now+1 { - wait := time.Duration(timestamp-now) * time.Second - log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) - time.Sleep(wait) - } - num := parent.Number() header := &types.Header{ ParentHash: parent.Hash(), From 664903dc889ec295b8eea292964547dc7910a26d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 5 Jan 2021 10:18:22 +0000 Subject: [PATCH 039/709] cmd/geth: usb is off by default (#21984) --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 3 ++- cmd/utils/flags.go | 11 +++++++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0d1b569b8b..a2d5c36e76 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -68,6 +68,7 @@ var ( utils.KeyStoreDirFlag, utils.ExternalSignerFlag, utils.NoUSBFlag, + utils.USBFlag, utils.SmartCardDaemonPathFlag, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 0e70451ed3..73fdcacac5 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -37,7 +37,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.DataDirFlag, utils.AncientFlag, utils.KeyStoreDirFlag, - utils.NoUSBFlag, + utils.USBFlag, utils.SmartCardDaemonPathFlag, utils.NetworkIdFlag, utils.GoerliFlag, @@ -219,6 +219,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ { Name: "ALIASED (deprecated)", Flags: append([]cli.Flag{ + utils.NoUSBFlag, utils.LegacyRPCEnabledFlag, utils.LegacyRPCListenAddrFlag, utils.LegacyRPCPortFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c51d7916ca..20b1744fdd 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -119,7 +119,11 @@ var ( } NoUSBFlag = cli.BoolFlag{ Name: "nousb", - Usage: "Disables monitoring for and managing USB hardware wallets", + Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", + } + USBFlag = cli.BoolFlag{ + Name: "usb", + Usage: "Enable monitoring and management of USB hardware wallets", } SmartCardDaemonPathFlag = cli.StringFlag{ Name: "pcscdpath", @@ -1225,8 +1229,11 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LightKDFFlag.Name) { cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name) } + if ctx.GlobalIsSet(USBFlag.Name) { + cfg.NoUSB = !ctx.GlobalBool(USBFlag.Name) + } if ctx.GlobalIsSet(NoUSBFlag.Name) { - cfg.NoUSB = ctx.GlobalBool(NoUSBFlag.Name) + log.Warn("Option nousb is deprecated and USB is deactivated by default. Use --usb to enable") } if ctx.GlobalIsSet(InsecureUnlockAllowedFlag.Name) { cfg.InsecureUnlockAllowed = ctx.GlobalBool(InsecureUnlockAllowedFlag.Name) From eb2a1dfdd21eeb89fcd6f9c06d42770e5078a6ed Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 5 Jan 2021 02:22:32 -0800 Subject: [PATCH 040/709] graphql: use a decimal representation for gas limit and gas used (#21883) This changes the JSON encoding of blocks returned by the API to have decimal instead of hexadecimal numbers. The spec wants it this way. Co-authored-by: Martin Holst Swende --- graphql/graphql.go | 56 ++++++++++--- graphql/graphql_test.go | 175 ++++++++++++++++++++++++---------------- 2 files changed, 150 insertions(+), 81 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 22cfcf6637..66c581628b 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -20,6 +20,8 @@ package graphql import ( "context" "errors" + "fmt" + "strconv" "time" "github.com/ethereum/go-ethereum" @@ -39,6 +41,37 @@ var ( errBlockInvariant = errors.New("block objects must be instantiated with at least one of num or hash") ) +type Long int64 + +// ImplementsGraphQLType returns true if Long implements the provided GraphQL type. +func (b Long) ImplementsGraphQLType(name string) bool { return name == "Long" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Long) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + // uncomment to support hex values + //if strings.HasPrefix(input, "0x") { + // // apply leniency and support hex representations of longs. + // value, err := hexutil.DecodeUint64(input) + // *b = Long(value) + // return err + //} else { + value, err := strconv.ParseInt(input, 10, 64) + *b = Long(value) + return err + //} + case int32: + *b = Long(input) + case int64: + *b = Long(input) + default: + err = fmt.Errorf("unexpected type %T for Long", input) + } + return err +} + // Account represents an Ethereum account at a particular block. type Account struct { backend ethapi.Backend @@ -415,13 +448,13 @@ func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) { return b.receipts, nil } -func (b *Block) Number(ctx context.Context) (hexutil.Uint64, error) { +func (b *Block) Number(ctx context.Context) (Long, error) { header, err := b.resolveHeader(ctx) if err != nil { return 0, err } - return hexutil.Uint64(header.Number.Uint64()), nil + return Long(header.Number.Uint64()), nil } func (b *Block) Hash(ctx context.Context) (common.Hash, error) { @@ -435,20 +468,20 @@ func (b *Block) Hash(ctx context.Context) (common.Hash, error) { return b.hash, nil } -func (b *Block) GasLimit(ctx context.Context) (hexutil.Uint64, error) { +func (b *Block) GasLimit(ctx context.Context) (Long, error) { header, err := b.resolveHeader(ctx) if err != nil { return 0, err } - return hexutil.Uint64(header.GasLimit), nil + return Long(header.GasLimit), nil } -func (b *Block) GasUsed(ctx context.Context) (hexutil.Uint64, error) { +func (b *Block) GasUsed(ctx context.Context) (Long, error) { header, err := b.resolveHeader(ctx) if err != nil { return 0, err } - return hexutil.Uint64(header.GasUsed), nil + return Long(header.GasUsed), nil } func (b *Block) Parent(ctx context.Context) (*Block, error) { @@ -902,11 +935,14 @@ type Resolver struct { } func (r *Resolver) Block(ctx context.Context, args struct { - Number *hexutil.Uint64 + Number *Long Hash *common.Hash }) (*Block, error) { var block *Block if args.Number != nil { + if *args.Number < 0 { + return nil, nil + } number := rpc.BlockNumber(*args.Number) numberOrHash := rpc.BlockNumberOrHashWithNumber(number) block = &Block{ @@ -939,10 +975,10 @@ func (r *Resolver) Block(ctx context.Context, args struct { } func (r *Resolver) Blocks(ctx context.Context, args struct { - From hexutil.Uint64 - To *hexutil.Uint64 + From *Long + To *Long }) ([]*Block, error) { - from := rpc.BlockNumber(args.From) + from := rpc.BlockNumber(*args.From) var to rpc.BlockNumber if args.To != nil { diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 98c8622ee6..77129673c0 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -19,18 +19,17 @@ package graphql import ( "fmt" "io/ioutil" + "math/big" "net/http" "strings" "testing" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" - "github.com/stretchr/testify/assert" + "github.com/ethereum/go-ethereum/params" ) func TestBuildSchema(t *testing.T) { @@ -45,29 +44,95 @@ func TestBuildSchema(t *testing.T) { } // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint -func TestGraphQLHTTPOnSamePort_GQLRequest_Successful(t *testing.T) { +func TestGraphQLBlockSerialization(t *testing.T) { stack := createNode(t, true) defer stack.Close() // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) } - // create http request - body := strings.NewReader("{\"query\": \"{block{number}}\",\"variables\": null}") - gqlReq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), body) - if err != nil { - t.Error("could not issue new http request ", err) - } - gqlReq.Header.Set("Content-Type", "application/json") - // read from response - resp := doHTTPRequest(t, gqlReq) - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatalf("could not read from response body: %v", err) + + for i, tt := range []struct { + body string + want string + code int + }{ + { // Should return latest block + body: `{"query": "{block{number}}","variables": null}`, + want: `{"data":{"block":{"number":10}}}`, + code: 200, + }, + { // Should return info about latest block + body: `{"query": "{block{number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":{"number":10,"gasUsed":0,"gasLimit":11500000}}}`, + code: 200, + }, + { + body: `{"query": "{block(number:0){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":{"number":0,"gasUsed":0,"gasLimit":11500000}}}`, + code: 200, + }, + { + body: `{"query": "{block(number:-1){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":null}}`, + code: 200, + }, + { + body: `{"query": "{block(number:-500){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":null}}`, + code: 200, + }, + { + body: `{"query": "{block(number:\"0\"){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":{"number":0,"gasUsed":0,"gasLimit":11500000}}}`, + code: 200, + }, + { + body: `{"query": "{block(number:\"-33\"){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":null}}`, + code: 200, + }, + { + body: `{"query": "{block(number:\"1337\"){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"data":{"block":null}}`, + code: 200, + }, + { + body: `{"query": "{block(number:\"0xbad\"){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"errors":[{"message":"strconv.ParseInt: parsing \"0xbad\": invalid syntax"}],"data":{}}`, + code: 400, + }, + { // hex strings are currently not supported. If that's added to the spec, this test will need to change + body: `{"query": "{block(number:\"0x0\"){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"errors":[{"message":"strconv.ParseInt: parsing \"0x0\": invalid syntax"}],"data":{}}`, + code: 400, + }, + { + body: `{"query": "{block(number:\"a\"){number,gasUsed,gasLimit}}","variables": null}`, + want: `{"errors":[{"message":"strconv.ParseInt: parsing \"a\": invalid syntax"}],"data":{}}`, + code: 400, + }, + { + body: `{"query": "{bleh{number}}","variables": null}"`, + want: `{"errors":[{"message":"Cannot query field \"bleh\" on type \"Query\".","locations":[{"line":1,"column":2}]}]}`, + code: 400, + }, + } { + resp, err := http.Post(fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), "application/json", strings.NewReader(tt.body)) + if err != nil { + t.Fatalf("could not post: %v", err) + } + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("could not read from response body: %v", err) + } + if have := string(bodyBytes); have != tt.want { + t.Errorf("testcase %d %s,\nhave:\n%v\nwant:\n%v", i, tt.body, have, tt.want) + } + if tt.code != resp.StatusCode { + t.Errorf("testcase %d %s,\nwrong statuscode, have: %v, want: %v", i, tt.body, resp.StatusCode, tt.code) + } } - expected := "{\"data\":{\"block\":{\"number\":\"0x0\"}}}" - assert.Equal(t, 200, resp.StatusCode) - assert.Equal(t, expected, string(bodyBytes)) } // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint @@ -77,49 +142,22 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) } - - // create http request - body := strings.NewReader("{\"query\": \"{block{number}}\",\"variables\": null}") - gqlReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), body) + body := strings.NewReader(`{"query": "{block{number}}","variables": null}`) + resp, err := http.Post(fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), "application/json", body) if err != nil { - t.Error("could not issue new http request ", err) + t.Fatalf("could not post: %v", err) } - gqlReq.Header.Set("Content-Type", "application/json") - // read from response - resp := doHTTPRequest(t, gqlReq) bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatalf("could not read from response body: %v", err) } // make sure the request is not handled successfully - assert.Equal(t, 404, resp.StatusCode) - assert.Equal(t, "404 page not found\n", string(bodyBytes)) -} - -// Tests that 400 is returned when an invalid RPC request is made. -func TestGraphQL_BadRequest(t *testing.T) { - stack := createNode(t, true) - defer stack.Close() - // start node - if err := stack.Start(); err != nil { - t.Fatalf("could not start node: %v", err) + if want, have := "404 page not found\n", string(bodyBytes); have != want { + t.Errorf("have:\n%v\nwant:\n%v", have, want) } - // create http request - body := strings.NewReader("{\"query\": \"{bleh{number}}\",\"variables\": null}") - gqlReq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), body) - if err != nil { - t.Error("could not issue new http request ", err) - } - gqlReq.Header.Set("Content-Type", "application/json") - // read from response - resp := doHTTPRequest(t, gqlReq) - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatalf("could not read from response body: %v", err) + if want, have := 404, resp.StatusCode; want != have { + t.Errorf("wrong statuscode, have:\n%v\nwant:%v", have, want) } - expected := "{\"errors\":[{\"message\":\"Cannot query field \\\"bleh\\\" on type \\\"Query\\\".\",\"locations\":[{\"line\":1,\"column\":2}]}]}" - assert.Equal(t, expected, string(bodyBytes)) - assert.Equal(t, 400, resp.StatusCode) } func createNode(t *testing.T, gqlEnabled bool) *node.Node { @@ -135,21 +173,20 @@ func createNode(t *testing.T, gqlEnabled bool) *node.Node { if !gqlEnabled { return stack } - createGQLService(t, stack, "127.0.0.1:9393") - return stack } func createGQLService(t *testing.T, stack *node.Node, endpoint string) { - // create backend (use a config which is light on mem consumption) + // create backend ethConf := ð.Config{ - Genesis: core.DeveloperGenesisBlock(15, common.Address{}), - Miner: miner.Config{ - Etherbase: common.HexToAddress("0xaabb"), + Genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), }, Ethash: ethash.Config{ - PowMode: ethash.ModeTest, + PowMode: ethash.ModeFake, }, NetworkId: 1337, TrieCleanCache: 5, @@ -163,20 +200,16 @@ func createGQLService(t *testing.T, stack *node.Node, endpoint string) { if err != nil { t.Fatalf("could not create eth backend: %v", err) } - + // Create some blocks and import them + chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), + ethash.NewFaker(), ethBackend.ChainDb(), 10, func(i int, gen *core.BlockGen) {}) + _, err = ethBackend.BlockChain().InsertChain(chain) + if err != nil { + t.Fatalf("could not create import blocks: %v", err) + } // create gql service err = New(stack, ethBackend.APIBackend, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } } - -func doHTTPRequest(t *testing.T, req *http.Request) *http.Response { - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - t.Fatal("could not issue a GET request to the given endpoint", err) - - } - return resp -} From 4714ce9430d2519e869a993d9f973a062dfc52d6 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 5 Jan 2021 14:31:23 +0100 Subject: [PATCH 041/709] cmd/geth: added --mainnet flag (#21932) * cmd/geth: added --mainnet flag * cmd/utils: set default genesis if --mainnet is specified * cmd/utils: addressed comments --- cmd/geth/chaincmd.go | 2 ++ cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 12 +++++++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6418f90957..f539322654 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -159,6 +159,7 @@ The export-preimages command export hash preimages to an RLP encoded stream`, utils.CacheFlag, utils.SyncModeFlag, utils.FakePoWFlag, + utils.MainnetFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.TxLookupLimitFlag, @@ -210,6 +211,7 @@ Use "ethereum dump 0" to dump the genesis block.`, utils.DataDirFlag, utils.AncientFlag, utils.CacheFlag, + utils.MainnetFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a2d5c36e76..7af24e6523 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -140,6 +140,7 @@ var ( utils.NodeKeyFileFlag, utils.NodeKeyHexFlag, utils.DNSDiscoveryFlag, + utils.MainnetFlag, utils.DeveloperFlag, utils.DeveloperPeriodFlag, utils.LegacyTestnetFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 73fdcacac5..78ebb807e1 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -40,6 +40,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.USBFlag, utils.SmartCardDaemonPathFlag, utils.NetworkIdFlag, + utils.MainnetFlag, utils.GoerliFlag, utils.RinkebyFlag, utils.YoloV2Flag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 20b1744fdd..764d7ad73e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -135,6 +135,10 @@ var ( Usage: "Explicitly set network id (integer)(For testnets: use --ropsten, --rinkeby, --goerli instead)", Value: eth.DefaultConfig.NetworkId, } + MainnetFlag = cli.BoolFlag{ + Name: "mainnet", + Usage: "Ethereum mainnet", + } GoerliFlag = cli.BoolFlag{ Name: "goerli", Usage: "Görli network: pre-configured proof-of-authority test network", @@ -1494,7 +1498,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV2Flag) + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV2Flag) CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag) @@ -1608,6 +1612,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } // Override any default configs for hard coded networks. switch { + case ctx.GlobalBool(MainnetFlag.Name): + if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + cfg.NetworkId = 1 + } + cfg.Genesis = core.DefaultGenesisBlock() + SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 3 From 9ba306d47ef3211de83bb858643abab77faf0528 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 5 Jan 2021 14:48:22 +0100 Subject: [PATCH 042/709] common/compiler: fix parsing of solc output with solidity v.0.8.0 (#22092) Solidity 0.8.0 changes the way that output is marshalled. This patch allows to parse both the legacy format used previously and the new format. See also https://docs.soliditylang.org/en/breaking/080-breaking-changes.html#interface-changes --- common/compiler/solidity.go | 50 +++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index b689f258a3..01de3d4c65 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -44,6 +44,20 @@ type solcOutput struct { Version string } +// solidity v.0.8 changes the way ABI, Devdoc and Userdoc are serialized +type solcOutputV8 struct { + Contracts map[string]struct { + BinRuntime string `json:"bin-runtime"` + SrcMapRuntime string `json:"srcmap-runtime"` + Bin, SrcMap, Metadata string + Abi interface{} + Devdoc interface{} + Userdoc interface{} + Hashes map[string]string + } + Version string +} + func (s *Solidity) makeArgs() []string { p := []string{ "--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc", @@ -125,7 +139,6 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro if err := cmd.Run(); err != nil { return nil, fmt.Errorf("solc: %v\n%s", err, stderr.Bytes()) } - return ParseCombinedJSON(stdout.Bytes(), source, s.Version, s.Version, strings.Join(s.makeArgs(), " ")) } @@ -141,7 +154,8 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion string, compilerVersion string, compilerOptions string) (map[string]*Contract, error) { var output solcOutput if err := json.Unmarshal(combinedJSON, &output); err != nil { - return nil, err + // Try to parse the output with the new solidity v.0.8.0 rules + return parseCombinedJSONV8(combinedJSON, source, languageVersion, compilerVersion, compilerOptions) } // Compilation succeeded, assemble and return the contracts. contracts := make(map[string]*Contract) @@ -176,3 +190,35 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin } return contracts, nil } + +// parseCombinedJSONV8 parses the direct output of solc --combined-output +// and parses it using the rules from solidity v.0.8.0 and later. +func parseCombinedJSONV8(combinedJSON []byte, source string, languageVersion string, compilerVersion string, compilerOptions string) (map[string]*Contract, error) { + var output solcOutputV8 + if err := json.Unmarshal(combinedJSON, &output); err != nil { + return nil, err + } + // Compilation succeeded, assemble and return the contracts. + contracts := make(map[string]*Contract) + for name, info := range output.Contracts { + contracts[name] = &Contract{ + Code: "0x" + info.Bin, + RuntimeCode: "0x" + info.BinRuntime, + Hashes: info.Hashes, + Info: ContractInfo{ + Source: source, + Language: "Solidity", + LanguageVersion: languageVersion, + CompilerVersion: compilerVersion, + CompilerOptions: compilerOptions, + SrcMap: info.SrcMap, + SrcMapRuntime: info.SrcMapRuntime, + AbiDefinition: info.Abi, + UserDoc: info.Userdoc, + DeveloperDoc: info.Devdoc, + Metadata: info.Metadata, + }, + } + } + return contracts, nil +} From 618454214b70124646dc1c6333a59138b97d6b0a Mon Sep 17 00:00:00 2001 From: jk-jeongkyun <45347815+jeongkyun-oh@users.noreply.github.com> Date: Tue, 5 Jan 2021 22:56:01 +0900 Subject: [PATCH 043/709] eth/downloader: enhanced test cases for downloader queue (#22114) --- eth/downloader/queue_test.go | 44 ++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index aedfba4565..f43ad67a41 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -97,6 +97,9 @@ func dummyPeer(id string) *peerConnection { } func TestBasics(t *testing.T) { + numOfBlocks := len(emptyChain.blocks) + numOfReceipts := len(emptyChain.blocks) / 2 + q := newQueue(10, 10) if !q.Idle() { t.Errorf("new queue should be idle") @@ -135,6 +138,12 @@ func TestBasics(t *testing.T) { t.Fatalf("expected header %d, got %d", exp, got) } } + if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { + t.Errorf("expected block task queue to be %d, got %d", exp, got) + } + if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { + t.Errorf("expected receipt task queue to be %d, got %d", exp, got) + } { peer := dummyPeer("peer-2") fetchReq, _, throttle := q.ReserveBodies(peer, 50) @@ -148,8 +157,12 @@ func TestBasics(t *testing.T) { t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) } } - //fmt.Printf("blockTaskQueue len: %d\n", q.blockTaskQueue.Size()) - //fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size()) + if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { + t.Errorf("expected block task queue to be %d, got %d", exp, got) + } + if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { + t.Errorf("expected receipt task queue to be %d, got %d", exp, got) + } { // The receipt delivering peer should not be affected // by the throttling of body deliveries @@ -168,12 +181,20 @@ func TestBasics(t *testing.T) { } } - //fmt.Printf("blockTaskQueue len: %d\n", q.blockTaskQueue.Size()) - //fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size()) - //fmt.Printf("processable: %d\n", q.resultCache.countCompleted()) + if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { + t.Errorf("expected block task queue to be %d, got %d", exp, got) + } + if exp, got := q.receiptTaskQueue.Size(), numOfReceipts-5; exp != got { + t.Errorf("expected receipt task queue to be %d, got %d", exp, got) + } + if got, exp := q.resultCache.countCompleted(), 0; got != exp { + t.Errorf("wrong processable count, got %d, exp %d", got, exp) + } } func TestEmptyBlocks(t *testing.T) { + numOfBlocks := len(emptyChain.blocks) + q := newQueue(10, 10) q.Prepare(1, FastSync) @@ -208,13 +229,12 @@ func TestEmptyBlocks(t *testing.T) { } } - if q.blockTaskQueue.Size() != len(emptyChain.blocks)-10 { - t.Errorf("expected block task queue to be 0, got %d", q.blockTaskQueue.Size()) + if q.blockTaskQueue.Size() != numOfBlocks-10 { + t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) } if q.receiptTaskQueue.Size() != 0 { - t.Errorf("expected receipt task queue to be 0, got %d", q.receiptTaskQueue.Size()) + t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) } - //fmt.Printf("receiptTaskQueue len: %d\n", q.receiptTaskQueue.Size()) { peer := dummyPeer("peer-3") fetchReq, _, _ := q.ReserveReceipts(peer, 50) @@ -224,6 +244,12 @@ func TestEmptyBlocks(t *testing.T) { t.Fatal("there should be no body fetch tasks remaining") } } + if q.blockTaskQueue.Size() != numOfBlocks-10 { + t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) + } + if q.receiptTaskQueue.Size() != 0 { + t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) + } if got, exp := q.resultCache.countCompleted(), 10; got != exp { t.Errorf("wrong processable count, got %d, exp %d", got, exp) } From 83d317cff937940395fcb7ece29effc9c7779c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 6 Jan 2021 08:37:45 +0200 Subject: [PATCH 044/709] cmd/utils, eth/downloader: minor snap nitpicks --- cmd/utils/flags.go | 1 - eth/downloader/downloader.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 764d7ad73e..684e3428ba 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1568,7 +1568,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { log.Info("Snap sync requested, enabling --snapshot") - ctx.Set(SnapshotFlag.Name, "true") } else { cfg.TrieCleanCache += cfg.SnapshotCache cfg.SnapshotCache = 0 // Disabled diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 3123598437..315354ea99 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -89,7 +89,7 @@ var ( errCancelContentProcessing = errors.New("content processing canceled (requested)") errCanceled = errors.New("syncing canceled (requested)") errNoSyncActive = errors.New("no sync active") - errTooOld = errors.New("peer doesn't speak recent enough protocol version (need version >= 64)") + errTooOld = errors.New("peer's protocol version too old") ) type Downloader struct { @@ -460,7 +460,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I } }() if p.version < 64 { - return fmt.Errorf("%w, peer version: %d", errTooOld, p.version) + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, 64) } mode := d.getMode() From d667ee2d1063faad0b4347db498bda00f34a1ba6 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 6 Jan 2021 12:06:44 +0100 Subject: [PATCH 045/709] crypto: fix ineffectual assignments (#22124) * crypto/bls12381: fixed ineffectual assignment * crypto/signify: fix ineffectual assignment --- crypto/bls12381/arithmetic_fallback.go | 2 +- crypto/signify/signify_fuzz.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crypto/bls12381/arithmetic_fallback.go b/crypto/bls12381/arithmetic_fallback.go index 19fb589104..91cabf4f3d 100644 --- a/crypto/bls12381/arithmetic_fallback.go +++ b/crypto/bls12381/arithmetic_fallback.go @@ -207,7 +207,7 @@ func lsubAssign(z, x *fe) { z[2], b = bits.Sub64(z[2], x[2], b) z[3], b = bits.Sub64(z[3], x[3], b) z[4], b = bits.Sub64(z[4], x[4], b) - z[5], b = bits.Sub64(z[5], x[5], b) + z[5], _ = bits.Sub64(z[5], x[5], b) } func neg(z *fe, x *fe) { diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index d1bcf356a4..f9167900ad 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -25,7 +25,6 @@ import ( "log" "os" "os/exec" - "runtime" fuzz "github.com/google/gofuzz" "github.com/jedisct1/go-minisign" @@ -129,6 +128,9 @@ func getKey(fileS string) (string, error) { func createKeyPair() (string, string) { // Create key and put it in correct format tmpKey, err := ioutil.TempFile("", "") + if err != nil { + panic(err) + } defer os.Remove(tmpKey.Name()) defer os.Remove(tmpKey.Name() + ".pub") defer os.Remove(tmpKey.Name() + ".sec") From 072fd9625472b585dc7c0bd5fab17e289c0d91a9 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 6 Jan 2021 17:19:16 +0100 Subject: [PATCH 046/709] graphql: return decimal for `estimateGas` and `cumulativeGas` queries (#22126) * estimateGas, cumulativeGas * linted * add test for estimateGas --- graphql/graphql.go | 27 ++++++++++++++------------- graphql/graphql_test.go | 6 ++++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 66c581628b..ea587106b4 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -301,21 +301,21 @@ func (t *Transaction) Status(ctx context.Context) (*hexutil.Uint64, error) { return &ret, nil } -func (t *Transaction) GasUsed(ctx context.Context) (*hexutil.Uint64, error) { +func (t *Transaction) GasUsed(ctx context.Context) (*Long, error) { receipt, err := t.getReceipt(ctx) if err != nil || receipt == nil { return nil, err } - ret := hexutil.Uint64(receipt.GasUsed) + ret := Long(receipt.GasUsed) return &ret, nil } -func (t *Transaction) CumulativeGasUsed(ctx context.Context) (*hexutil.Uint64, error) { +func (t *Transaction) CumulativeGasUsed(ctx context.Context) (*Long, error) { receipt, err := t.getReceipt(ctx) if err != nil || receipt == nil { return nil, err } - ret := hexutil.Uint64(receipt.CumulativeGasUsed) + ret := Long(receipt.CumulativeGasUsed) return &ret, nil } @@ -811,7 +811,7 @@ type CallData struct { // CallResult encapsulates the result of an invocation of the `call` accessor. type CallResult struct { data hexutil.Bytes // The return data from the call - gasUsed hexutil.Uint64 // The amount of gas used + gasUsed Long // The amount of gas used status hexutil.Uint64 // The return status of the call - 0 for failure or 1 for success. } @@ -819,7 +819,7 @@ func (c *CallResult) Data() hexutil.Bytes { return c.data } -func (c *CallResult) GasUsed() hexutil.Uint64 { +func (c *CallResult) GasUsed() Long { return c.gasUsed } @@ -847,22 +847,22 @@ func (b *Block) Call(ctx context.Context, args struct { return &CallResult{ data: result.ReturnData, - gasUsed: hexutil.Uint64(result.UsedGas), + gasUsed: Long(result.UsedGas), status: status, }, nil } func (b *Block) EstimateGas(ctx context.Context, args struct { Data ethapi.CallArgs -}) (hexutil.Uint64, error) { +}) (Long, error) { if b.numberOrHash == nil { _, err := b.resolveHeader(ctx) if err != nil { - return hexutil.Uint64(0), err + return 0, err } } gas, err := ethapi.DoEstimateGas(ctx, b.backend, args.Data, *b.numberOrHash, b.backend.RPCGasCap()) - return gas, err + return Long(gas), err } type Pending struct { @@ -917,16 +917,17 @@ func (p *Pending) Call(ctx context.Context, args struct { return &CallResult{ data: result.ReturnData, - gasUsed: hexutil.Uint64(result.UsedGas), + gasUsed: Long(result.UsedGas), status: status, }, nil } func (p *Pending) EstimateGas(ctx context.Context, args struct { Data ethapi.CallArgs -}) (hexutil.Uint64, error) { +}) (Long, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - return ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) + return Long(gas), err } // Resolver is the top-level object in the GraphQL hierarchy. diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 77129673c0..fc62da1813 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -117,6 +117,12 @@ func TestGraphQLBlockSerialization(t *testing.T) { want: `{"errors":[{"message":"Cannot query field \"bleh\" on type \"Query\".","locations":[{"line":1,"column":2}]}]}`, code: 400, }, + // should return `estimateGas` as decimal + { + body: `{"query": "{block{ estimateGas(data:{}) }}"}`, + want: `{"data":{"block":{"estimateGas":53000}}}`, + code: 200, + }, } { resp, err := http.Post(fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), "application/json", strings.NewReader(tt.body)) if err != nil { From d2e1b17f1828424906c9033b1dcbd3f2756a0c2c Mon Sep 17 00:00:00 2001 From: Melvin Junhee Woo Date: Thu, 7 Jan 2021 15:36:21 +0900 Subject: [PATCH 047/709] snapshot, trie: fixed typos, mostly in snapshot pkg (#22133) --- core/state/snapshot/conversion.go | 2 +- core/state/snapshot/difflayer.go | 10 +++++----- core/state/snapshot/difflayer_test.go | 2 +- core/state/snapshot/disklayer.go | 2 +- core/state/snapshot/disklayer_test.go | 2 +- core/state/snapshot/iterator.go | 4 ++-- core/state/snapshot/iterator_binary.go | 8 ++++---- core/state/snapshot/iterator_fast.go | 6 +++--- core/state/snapshot/snapshot.go | 4 ++-- trie/database.go | 2 +- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index dee9ff0bf2..9832225345 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -240,7 +240,7 @@ func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieGenerato } in <- leaf - // Accumulate the generaation statistic if it's required. + // Accumulate the generation statistic if it's required. processed++ if time.Since(logged) > 3*time.Second && stats != nil { if account == (common.Hash{}) { diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 0aef6cf570..3be78cae88 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -44,7 +44,7 @@ var ( // aggregatorItemLimit is an approximate number of items that will end up // in the agregator layer before it's flushed out to disk. A plain account // weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot - // 0B (+hash). Slots are mostly set/unset in lockstep, so thet average at + // 0B (+hash). Slots are mostly set/unset in lockstep, so that average at // 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a // smaller number to be on the safe side. aggregatorItemLimit = aggregatorMemoryLimit / 42 @@ -114,9 +114,9 @@ type diffLayer struct { // deleted, all data in other set belongs to the "new" A. destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil - accountData map[common.Hash][]byte // Keyed accounts for direct retrival (nil means deleted) + accountData map[common.Hash][]byte // Keyed accounts for direct retrieval (nil means deleted) storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil - storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrival. one per account (nil means deleted) + storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted) diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer @@ -482,7 +482,7 @@ func (dl *diffLayer) flatten() snapshot { } } -// AccountList returns a sorted list of all accounts in this difflayer, including +// AccountList returns a sorted list of all accounts in this diffLayer, including // the deleted ones. // // Note, the returned slice is not a copy, so do not modify it. @@ -513,7 +513,7 @@ func (dl *diffLayer) AccountList() []common.Hash { return dl.accountList } -// StorageList returns a sorted list of all storage slot hashes in this difflayer +// StorageList returns a sorted list of all storage slot hashes in this diffLayer // for the given account. If the whole storage is destructed in this layer, then // an additional flag *destructed = true* will be returned, otherwise the flag is // false. Besides, the returned list will include the hash of deleted storage slot. diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 31636ee133..919af5fa86 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -314,7 +314,7 @@ func BenchmarkSearchSlot(b *testing.B) { // With accountList and sorting // BenchmarkFlatten-6 50 29890856 ns/op // -// Without sorting and tracking accountlist +// Without sorting and tracking accountList // BenchmarkFlatten-6 300 5511511 ns/op func BenchmarkFlatten(b *testing.B) { fill := func(parent snapshot) *diffLayer { diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index e8f2bc853f..7cbf6e293d 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -31,7 +31,7 @@ import ( // diskLayer is a low level persistent snapshot built on top of a key-value store. type diskLayer struct { diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot - triedb *trie.Database // Trie node cache for reconstuction purposes + triedb *trie.Database // Trie node cache for reconstruction purposes cache *fastcache.Cache // Cache to avoid hitting the disk for direct access root common.Hash // Root hash of the base snapshot diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 40ff5ade4c..6beb944e07 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -482,7 +482,7 @@ func TestDiskGeneratorPersistence(t *testing.T) { if !bytes.Equal(generator.Marker, genMarker) { t.Fatalf("Generator marker is not matched") } - // Test senario 2, the disk layer is fully generated + // Test scenario 2, the disk layer is fully generated // Modify or delete some accounts, flatten everything onto disk if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ accThree: accThree.Bytes(), diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index 5f943fea9f..1d9340bbad 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -133,7 +133,7 @@ func (it *diffAccountIterator) Hash() common.Hash { // Account returns the RLP encoded slim account the iterator is currently at. // This method may _fail_, if the underlying layer has been flattened between -// the call to Next and Acccount. That type of error will set it.Err. +// the call to Next and Account. That type of error will set it.Err. // This method assumes that flattening does not delete elements from // the accountdata mapping (writing nil into it is fine though), and will panic // if elements have been deleted. @@ -243,7 +243,7 @@ type diffStorageIterator struct { } // StorageIterator creates a storage iterator over a single diff layer. -// Execept the storage iterator is returned, there is an additional flag +// Except the storage iterator is returned, there is an additional flag // "destructed" returned. If it's true then it means the whole storage is // destructed in this layer(maybe recreated too), don't bother deeper layer // for storage retrieval. diff --git a/core/state/snapshot/iterator_binary.go b/core/state/snapshot/iterator_binary.go index f82f750029..22184b2545 100644 --- a/core/state/snapshot/iterator_binary.go +++ b/core/state/snapshot/iterator_binary.go @@ -37,7 +37,7 @@ type binaryIterator struct { } // initBinaryAccountIterator creates a simplistic iterator to step over all the -// accounts in a slow, but eaily verifiable way. Note this function is used for +// accounts in a slow, but easily verifiable way. Note this function is used for // initialization, use `newBinaryAccountIterator` as the API. func (dl *diffLayer) initBinaryAccountIterator() Iterator { parent, ok := dl.parent.(*diffLayer) @@ -62,7 +62,7 @@ func (dl *diffLayer) initBinaryAccountIterator() Iterator { } // initBinaryStorageIterator creates a simplistic iterator to step over all the -// storage slots in a slow, but eaily verifiable way. Note this function is used +// storage slots in a slow, but easily verifiable way. Note this function is used // for initialization, use `newBinaryStorageIterator` as the API. func (dl *diffLayer) initBinaryStorageIterator(account common.Hash) Iterator { parent, ok := dl.parent.(*diffLayer) @@ -199,14 +199,14 @@ func (it *binaryIterator) Release() { } // newBinaryAccountIterator creates a simplistic account iterator to step over -// all the accounts in a slow, but eaily verifiable way. +// all the accounts in a slow, but easily verifiable way. func (dl *diffLayer) newBinaryAccountIterator() AccountIterator { iter := dl.initBinaryAccountIterator() return iter.(AccountIterator) } // newBinaryStorageIterator creates a simplistic account iterator to step over -// all the storage slots in a slow, but eaily verifiable way. +// all the storage slots in a slow, but easily verifiable way. func (dl *diffLayer) newBinaryStorageIterator(account common.Hash) StorageIterator { iter := dl.initBinaryStorageIterator(account) return iter.(StorageIterator) diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 291d52900d..48069b8fcf 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -75,7 +75,7 @@ type fastIterator struct { fail error } -// newFastIterator creates a new hierarhical account or storage iterator with one +// newFastIterator creates a new hierarchical account or storage iterator with one // element per diff layer. The returned combo iterator can be used to walk over // the entire snapshot diff stack simultaneously. func newFastIterator(tree *Tree, root common.Hash, account common.Hash, seek common.Hash, accountIterator bool) (*fastIterator, error) { @@ -335,14 +335,14 @@ func (fi *fastIterator) Debug() { fmt.Println() } -// newFastAccountIterator creates a new hierarhical account iterator with one +// newFastAccountIterator creates a new hierarchical account iterator with one // element per diff layer. The returned combo iterator can be used to walk over // the entire snapshot diff stack simultaneously. func newFastAccountIterator(tree *Tree, root common.Hash, seek common.Hash) (AccountIterator, error) { return newFastIterator(tree, root, common.Hash{}, seek, true) } -// newFastStorageIterator creates a new hierarhical storage iterator with one +// newFastStorageIterator creates a new hierarchical storage iterator with one // element per diff layer. The returned combo iterator can be used to walk over // the entire snapshot diff stack simultaneously. func newFastStorageIterator(tree *Tree, root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 60b4158b56..443fc8e5c7 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -368,7 +368,7 @@ func (t *Tree) Cap(root common.Hash, layers int) error { // crossed. All diffs beyond the permitted number are flattened downwards. If the // layer limit is reached, memory cap is also enforced (but not before). // -// The method returns the new disk layer if diffs were persistend into it. +// The method returns the new disk layer if diffs were persisted into it. func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer { // Dive until we run out of layers or reach the persistent database for ; layers > 2; layers-- { @@ -647,7 +647,7 @@ func (t *Tree) Rebuild(root common.Hash) { panic(fmt.Sprintf("unknown layer type: %T", layer)) } } - // Start generating a new snapshot from scratch on a backgroung thread. The + // Start generating a new snapshot from scratch on a background thread. The // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") t.layers = map[common.Hash]snapshot{ diff --git a/trie/database.go b/trie/database.go index 7c2f207097..b18665770e 100644 --- a/trie/database.go +++ b/trie/database.go @@ -736,7 +736,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch.Replay(uncacher) batch.Reset() - // Reset the storage counters and bumpd metrics + // Reset the storage counters and bumped metrics if db.preimages != nil { db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 } From 44208d925811dc309b9e26df6dc1752ba359a0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 7 Jan 2021 10:23:50 +0200 Subject: [PATCH 048/709] cmd/faucet: fix websocket race regression after switching to gorilla --- cmd/faucet/faucet.go | 59 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 79a84fd24e..a0fc28c0d4 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -213,7 +213,7 @@ type faucet struct { nonce uint64 // Current pending nonce of the faucet price *big.Int // Current gas price to issue funds with - conns []*websocket.Conn // Currently live websocket connections + conns []*wsConn // Currently live websocket connections timeouts map[string]time.Time // History of users and their funding timeouts reqs []*request // Currently pending funding requests update chan struct{} // Channel to signal request updates @@ -221,6 +221,13 @@ type faucet struct { lock sync.RWMutex // Lock protecting the faucet's internals } +// wsConn wraps a websocket connection with a write mutex as the underlying +// websocket library does not synchronize access to the stream. +type wsConn struct { + conn *websocket.Conn + wlock sync.Mutex +} + func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) { // Assemble the raw devp2p protocol stack stack, err := node.New(&node.Config{ @@ -321,13 +328,14 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { defer conn.Close() f.lock.Lock() - f.conns = append(f.conns, conn) + wsconn := &wsConn{conn: conn} + f.conns = append(f.conns, wsconn) f.lock.Unlock() defer func() { f.lock.Lock() for i, c := range f.conns { - if c == conn { + if c.conn == conn { f.conns = append(f.conns[:i], f.conns[i+1:]...) break } @@ -355,7 +363,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { if head == nil || balance == nil { // Report the faucet offline until initial stats are ready //lint:ignore ST1005 This error is to be displayed in the browser - if err = sendError(conn, errors.New("Faucet offline")); err != nil { + if err = sendError(wsconn, errors.New("Faucet offline")); err != nil { log.Warn("Failed to send faucet error to client", "err", err) return } @@ -366,7 +374,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { f.lock.RLock() reqs := f.reqs f.lock.RUnlock() - if err = send(conn, map[string]interface{}{ + if err = send(wsconn, map[string]interface{}{ "funds": new(big.Int).Div(balance, ether), "funded": nonce, "peers": f.stack.Server().PeerCount(), @@ -375,7 +383,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { log.Warn("Failed to send initial stats to client", "err", err) return } - if err = send(conn, head, 3*time.Second); err != nil { + if err = send(wsconn, head, 3*time.Second); err != nil { log.Warn("Failed to send initial header to client", "err", err) return } @@ -391,7 +399,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { return } if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://twitter.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") { - if err = sendError(conn, errors.New("URL doesn't link to supported services")); err != nil { + if err = sendError(wsconn, errors.New("URL doesn't link to supported services")); err != nil { log.Warn("Failed to send URL error to client", "err", err) return } @@ -399,7 +407,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } if msg.Tier >= uint(*tiersFlag) { //lint:ignore ST1005 This error is to be displayed in the browser - if err = sendError(conn, errors.New("Invalid funding tier requested")); err != nil { + if err = sendError(wsconn, errors.New("Invalid funding tier requested")); err != nil { log.Warn("Failed to send tier error to client", "err", err) return } @@ -415,7 +423,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form) if err != nil { - if err = sendError(conn, err); err != nil { + if err = sendError(wsconn, err); err != nil { log.Warn("Failed to send captcha post error to client", "err", err) return } @@ -428,7 +436,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { err = json.NewDecoder(res.Body).Decode(&result) res.Body.Close() if err != nil { - if err = sendError(conn, err); err != nil { + if err = sendError(wsconn, err); err != nil { log.Warn("Failed to send captcha decode error to client", "err", err) return } @@ -437,7 +445,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { if !result.Success { log.Warn("Captcha verification failed", "err", string(result.Errors)) //lint:ignore ST1005 it's funny and the robot won't mind - if err = sendError(conn, errors.New("Beep-bop, you're a robot!")); err != nil { + if err = sendError(wsconn, errors.New("Beep-bop, you're a robot!")); err != nil { log.Warn("Failed to send captcha failure to client", "err", err) return } @@ -465,7 +473,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues") } if err != nil { - if err = sendError(conn, err); err != nil { + if err = sendError(wsconn, err); err != nil { log.Warn("Failed to send prefix error to client", "err", err) return } @@ -489,7 +497,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainID) if err != nil { f.lock.Unlock() - if err = sendError(conn, err); err != nil { + if err = sendError(wsconn, err); err != nil { log.Warn("Failed to send transaction creation error to client", "err", err) return } @@ -498,7 +506,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { // Submit the transaction and mark as funded if successful if err := f.client.SendTransaction(context.Background(), signed); err != nil { f.lock.Unlock() - if err = sendError(conn, err); err != nil { + if err = sendError(wsconn, err); err != nil { log.Warn("Failed to send transaction transmission error to client", "err", err) return } @@ -520,13 +528,13 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { // Send an error if too frequent funding, othewise a success if !fund { - if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(time.Until(timeout)))); err != nil { // nolint: gosimple + if err = sendError(wsconn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(time.Until(timeout)))); err != nil { // nolint: gosimple log.Warn("Failed to send funding error to client", "err", err) return } continue } - if err = sendSuccess(conn, fmt.Sprintf("Funding request accepted for %s into %s", username, address.Hex())); err != nil { + if err = sendSuccess(wsconn, fmt.Sprintf("Funding request accepted for %s into %s", username, address.Hex())); err != nil { log.Warn("Failed to send funding success to client", "err", err) return } @@ -619,12 +627,12 @@ func (f *faucet) loop() { "requests": f.reqs, }, time.Second); err != nil { log.Warn("Failed to send stats to client", "err", err) - conn.Close() + conn.conn.Close() continue } if err := send(conn, head, time.Second); err != nil { log.Warn("Failed to send header to client", "err", err) - conn.Close() + conn.conn.Close() } } f.lock.RUnlock() @@ -646,7 +654,7 @@ func (f *faucet) loop() { for _, conn := range f.conns { if err := send(conn, map[string]interface{}{"requests": f.reqs}, time.Second); err != nil { log.Warn("Failed to send requests to client", "err", err) - conn.Close() + conn.conn.Close() } } f.lock.RUnlock() @@ -656,23 +664,26 @@ func (f *faucet) loop() { // sends transmits a data packet to the remote end of the websocket, but also // setting a write deadline to prevent waiting forever on the node. -func send(conn *websocket.Conn, value interface{}, timeout time.Duration) error { +func send(conn *wsConn, value interface{}, timeout time.Duration) error { if timeout == 0 { timeout = 60 * time.Second } - conn.SetWriteDeadline(time.Now().Add(timeout)) - return conn.WriteJSON(value) + conn.wlock.Lock() + defer conn.wlock.Unlock() + + conn.conn.SetWriteDeadline(time.Now().Add(timeout)) + return conn.conn.WriteJSON(value) } // sendError transmits an error to the remote end of the websocket, also setting // the write deadline to 1 second to prevent waiting forever. -func sendError(conn *websocket.Conn, err error) error { +func sendError(conn *wsConn, err error) error { return send(conn, map[string]string{"error": err.Error()}, time.Second) } // sendSuccess transmits a success message to the remote end of the websocket, also // setting the write deadline to 1 second to prevent waiting forever. -func sendSuccess(conn *websocket.Conn, msg string) error { +func sendSuccess(conn *wsConn, msg string) error { return send(conn, map[string]string{"success": msg}, time.Second) } From 58b9db5f7cf7a3db4d9d0afa3772fb8da32ebc3a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 7 Jan 2021 11:58:07 +0100 Subject: [PATCH 049/709] eth/protocols/snap: track reverts when peer rejects request (#22016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/protocols/snap: reschedule missed deliveries * eth/protocols/snap: clarify log message * eth/protocols/snap: revert failures async and update runloop Co-authored-by: Péter Szilágyi --- eth/protocols/snap/sync.go | 172 +++++++++++++++++++++++++------------ 1 file changed, 116 insertions(+), 56 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 437b0caab4..82b21c4701 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -792,10 +792,7 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { } req.timeout = time.AfterFunc(requestTimeout, func() { log.Debug("Account range request timed out") - select { - case s.accountReqFails <- req: - default: - } + s.scheduleRevertAccountRequest(req) }) s.accountReqs[reqid] = req delete(s.accountIdlers, idle) @@ -807,12 +804,8 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { // Attempt to send the remote request and revert if it fails if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, maxRequestSize); err != nil { peer.Log().Debug("Failed to request account range", "err", err) - select { - case s.accountReqFails <- req: - default: - } + s.scheduleRevertAccountRequest(req) } - // Request successfully sent, start a }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists // Inject the request into the task to block further assignments @@ -886,10 +879,7 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { } req.timeout = time.AfterFunc(requestTimeout, func() { log.Debug("Bytecode request timed out") - select { - case s.bytecodeReqFails <- req: - default: - } + s.scheduleRevertBytecodeRequest(req) }) s.bytecodeReqs[reqid] = req delete(s.bytecodeIdlers, idle) @@ -901,12 +891,8 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { // Attempt to send the remote request and revert if it fails if err := peer.RequestByteCodes(reqid, hashes, maxRequestSize); err != nil { log.Debug("Failed to request bytecodes", "err", err) - select { - case s.bytecodeReqFails <- req: - default: - } + s.scheduleRevertBytecodeRequest(req) } - // Request successfully sent, start a }(s.peers[idle]) // We're in the lock, peers[id] surely exists } } @@ -1018,10 +1004,7 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { } req.timeout = time.AfterFunc(requestTimeout, func() { log.Debug("Storage request timed out") - select { - case s.storageReqFails <- req: - default: - } + s.scheduleRevertStorageRequest(req) }) s.storageReqs[reqid] = req delete(s.storageIdlers, idle) @@ -1037,12 +1020,8 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { } if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, maxRequestSize); err != nil { log.Debug("Failed to request storage", "err", err) - select { - case s.storageReqFails <- req: - default: - } + s.scheduleRevertStorageRequest(req) } - // Request successfully sent, start a }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists // Inject the request into the subtask to block further assignments @@ -1140,10 +1119,7 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { } req.timeout = time.AfterFunc(requestTimeout, func() { log.Debug("Trienode heal request timed out") - select { - case s.trienodeHealReqFails <- req: - default: - } + s.scheduleRevertTrienodeHealRequest(req) }) s.trienodeHealReqs[reqid] = req delete(s.trienodeHealIdlers, idle) @@ -1155,12 +1131,8 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { // Attempt to send the remote request and revert if it fails if err := peer.RequestTrieNodes(reqid, root, pathsets, maxRequestSize); err != nil { log.Debug("Failed to request trienode healers", "err", err) - select { - case s.trienodeHealReqFails <- req: - default: - } + s.scheduleRevertTrienodeHealRequest(req) } - // Request successfully sent, start a }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists } } @@ -1245,10 +1217,7 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { } req.timeout = time.AfterFunc(requestTimeout, func() { log.Debug("Bytecode heal request timed out") - select { - case s.bytecodeHealReqFails <- req: - default: - } + s.scheduleRevertBytecodeHealRequest(req) }) s.bytecodeHealReqs[reqid] = req delete(s.bytecodeHealIdlers, idle) @@ -1260,12 +1229,8 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { // Attempt to send the remote request and revert if it fails if err := peer.RequestByteCodes(reqid, hashes, maxRequestSize); err != nil { log.Debug("Failed to request bytecode healers", "err", err) - select { - case s.bytecodeHealReqFails <- req: - default: - } + s.scheduleRevertBytecodeHealRequest(req) } - // Request successfully sent, start a }(s.peers[idle]) // We're in the lock, peers[id] surely exists } } @@ -1325,10 +1290,26 @@ func (s *Syncer) revertRequests(peer string) { } } +// scheduleRevertAccountRequest asks the event loop to clean up an account range +// request and return all failed retrieval tasks to the scheduler for reassignment. +func (s *Syncer) scheduleRevertAccountRequest(req *accountRequest) { + select { + case s.accountReqFails <- req: + // Sync event loop notified + case <-req.cancel: + // Sync cycle got cancelled + case <-req.stale: + // Request already reverted + } +} + // revertAccountRequest cleans up an account range request and returns all failed // retrieval tasks to the scheduler for reassignment. +// +// Note, this needs to run on the event runloop thread to reschedule to idle peers. +// On peer threads, use scheduleRevertAccountRequest. func (s *Syncer) revertAccountRequest(req *accountRequest) { - log.Trace("Reverting account request", "peer", req.peer, "reqid", req.id) + log.Debug("Reverting account request", "peer", req.peer, "reqid", req.id) select { case <-req.stale: log.Trace("Account request already reverted", "peer", req.peer, "reqid", req.id) @@ -1350,10 +1331,26 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { } } -// revertBytecodeRequest cleans up an bytecode request and returns all failed +// scheduleRevertBytecodeRequest asks the event loop to clean up a bytecode request +// and return all failed retrieval tasks to the scheduler for reassignment. +func (s *Syncer) scheduleRevertBytecodeRequest(req *bytecodeRequest) { + select { + case s.bytecodeReqFails <- req: + // Sync event loop notified + case <-req.cancel: + // Sync cycle got cancelled + case <-req.stale: + // Request already reverted + } +} + +// revertBytecodeRequest cleans up a bytecode request and returns all failed // retrieval tasks to the scheduler for reassignment. +// +// Note, this needs to run on the event runloop thread to reschedule to idle peers. +// On peer threads, use scheduleRevertBytecodeRequest. func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { - log.Trace("Reverting bytecode request", "peer", req.peer) + log.Debug("Reverting bytecode request", "peer", req.peer) select { case <-req.stale: log.Trace("Bytecode request already reverted", "peer", req.peer, "reqid", req.id) @@ -1375,10 +1372,26 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { } } +// scheduleRevertStorageRequest asks the event loop to clean up a storage range +// request and return all failed retrieval tasks to the scheduler for reassignment. +func (s *Syncer) scheduleRevertStorageRequest(req *storageRequest) { + select { + case s.storageReqFails <- req: + // Sync event loop notified + case <-req.cancel: + // Sync cycle got cancelled + case <-req.stale: + // Request already reverted + } +} + // revertStorageRequest cleans up a storage range request and returns all failed // retrieval tasks to the scheduler for reassignment. +// +// Note, this needs to run on the event runloop thread to reschedule to idle peers. +// On peer threads, use scheduleRevertStorageRequest. func (s *Syncer) revertStorageRequest(req *storageRequest) { - log.Trace("Reverting storage request", "peer", req.peer) + log.Debug("Reverting storage request", "peer", req.peer) select { case <-req.stale: log.Trace("Storage request already reverted", "peer", req.peer, "reqid", req.id) @@ -1404,10 +1417,26 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { } } -// revertTrienodeHealRequest cleans up an trienode heal request and returns all +// scheduleRevertTrienodeHealRequest asks the event loop to clean up a trienode heal +// request and return all failed retrieval tasks to the scheduler for reassignment. +func (s *Syncer) scheduleRevertTrienodeHealRequest(req *trienodeHealRequest) { + select { + case s.trienodeHealReqFails <- req: + // Sync event loop notified + case <-req.cancel: + // Sync cycle got cancelled + case <-req.stale: + // Request already reverted + } +} + +// revertTrienodeHealRequest cleans up a trienode heal request and returns all // failed retrieval tasks to the scheduler for reassignment. +// +// Note, this needs to run on the event runloop thread to reschedule to idle peers. +// On peer threads, use scheduleRevertTrienodeHealRequest. func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { - log.Trace("Reverting trienode heal request", "peer", req.peer) + log.Debug("Reverting trienode heal request", "peer", req.peer) select { case <-req.stale: log.Trace("Trienode heal request already reverted", "peer", req.peer, "reqid", req.id) @@ -1429,10 +1458,26 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { } } -// revertBytecodeHealRequest cleans up an bytecode request and returns all failed -// retrieval tasks to the scheduler for reassignment. +// scheduleRevertBytecodeHealRequest asks the event loop to clean up a bytecode heal +// request and return all failed retrieval tasks to the scheduler for reassignment. +func (s *Syncer) scheduleRevertBytecodeHealRequest(req *bytecodeHealRequest) { + select { + case s.bytecodeHealReqFails <- req: + // Sync event loop notified + case <-req.cancel: + // Sync cycle got cancelled + case <-req.stale: + // Request already reverted + } +} + +// revertBytecodeHealRequest cleans up a bytecode heal request and returns all +// failed retrieval tasks to the scheduler for reassignment. +// +// Note, this needs to run on the event runloop thread to reschedule to idle peers. +// On peer threads, use scheduleRevertBytecodeHealRequest. func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { - log.Trace("Reverting bytecode heal request", "peer", req.peer) + log.Debug("Reverting bytecode heal request", "peer", req.peer) select { case <-req.stale: log.Trace("Bytecode heal request already reverted", "peer", req.peer, "reqid", req.id) @@ -1768,7 +1813,7 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { if err := batch.Write(); err != nil { log.Crit("Failed to persist healing data", "err", err) } - log.Debug("Persisted set of healing data", "bytes", common.StorageSize(batch.ValueSize())) + log.Debug("Persisted set of healing data", "type", "trienodes", "bytes", common.StorageSize(batch.ValueSize())) } // processBytecodeHealResponse integrates an already validated bytecode response @@ -1804,7 +1849,7 @@ func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) { if err := batch.Write(); err != nil { log.Crit("Failed to persist healing data", "err", err) } - log.Debug("Persisted set of healing data", "bytes", common.StorageSize(batch.ValueSize())) + log.Debug("Persisted set of healing data", "type", "bytecode", "bytes", common.StorageSize(batch.ValueSize())) } // forwardAccountTask takes a filled account task and persists anything available @@ -1940,6 +1985,9 @@ func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, account logger.Debug("Peer rejected account range request", "root", s.root) s.statelessPeers[peer.id] = struct{}{} s.lock.Unlock() + + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertAccountRequest(req) return nil } root := s.root @@ -2055,6 +2103,9 @@ func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { logger.Debug("Peer rejected bytecode request") s.statelessPeers[peer.id] = struct{}{} s.lock.Unlock() + + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertBytecodeRequest(req) return nil } s.lock.Unlock() @@ -2166,6 +2217,9 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots logger.Debug("Peer rejected storage request") s.statelessPeers[peer.id] = struct{}{} s.lock.Unlock() + + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertStorageRequest(req) return nil } s.lock.Unlock() @@ -2287,6 +2341,9 @@ func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { logger.Debug("Peer rejected trienode heal request") s.statelessPeers[peer.id] = struct{}{} s.lock.Unlock() + + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertTrienodeHealRequest(req) return nil } s.lock.Unlock() @@ -2371,6 +2428,9 @@ func (s *Syncer) onHealByteCodes(peer *Peer, id uint64, bytecodes [][]byte) erro logger.Debug("Peer rejected bytecode heal request") s.statelessPeers[peer.id] = struct{}{} s.lock.Unlock() + + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertBytecodeHealRequest(req) return nil } s.lock.Unlock() From 4bb5c6ca7a19f5ad0230879205380adaca12ef4f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 7 Jan 2021 17:12:41 +0100 Subject: [PATCH 050/709] eth/protocols/snap: speed up hash checks (#22023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/protocols/snap: speed up hash checks * eth/protocols/snap: nit fix Co-authored-by: Péter Szilágyi --- eth/protocols/snap/sync.go | 15 +++-- eth/protocols/snap/sync_test.go | 98 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 eth/protocols/snap/sync_test.go diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 82b21c4701..d6f0eb5472 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2112,14 +2112,15 @@ func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { // Cross reference the requested bytecodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256() + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash := make([]byte, 32) codes := make([][]byte, len(req.hashes)) for i, j := 0, 0; i < len(bytecodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() hasher.Write(bytecodes[i]) - hash := hasher.Sum(nil) + hasher.Read(hash) for j < len(req.hashes) && !bytes.Equal(hash, req.hashes[j][:]) { j++ @@ -2350,14 +2351,15 @@ func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { // Cross reference the requested trienodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256() + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash := make([]byte, 32) nodes := make([][]byte, len(req.hashes)) for i, j := 0, 0; i < len(trienodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() hasher.Write(trienodes[i]) - hash := hasher.Sum(nil) + hasher.Read(hash) for j < len(req.hashes) && !bytes.Equal(hash, req.hashes[j][:]) { j++ @@ -2437,14 +2439,15 @@ func (s *Syncer) onHealByteCodes(peer *Peer, id uint64, bytecodes [][]byte) erro // Cross reference the requested bytecodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256() + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash := make([]byte, 32) codes := make([][]byte, len(req.hashes)) for i, j := 0, 0; i < len(bytecodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() hasher.Write(bytecodes[i]) - hash := hasher.Sum(nil) + hasher.Read(hash) for j < len(req.hashes) && !bytes.Equal(hash, req.hashes[j][:]) { j++ diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go new file mode 100644 index 0000000000..4f28b99bfe --- /dev/null +++ b/eth/protocols/snap/sync_test.go @@ -0,0 +1,98 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "crypto/rand" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "golang.org/x/crypto/sha3" +) + +func TestHashing(t *testing.T) { + var bytecodes = make([][]byte, 10) + for i := 0; i < len(bytecodes); i++ { + buf := make([]byte, 100) + rand.Read(buf) + bytecodes[i] = buf + } + var want, got string + var old = func() { + hasher := sha3.NewLegacyKeccak256() + for i := 0; i < len(bytecodes); i++ { + hasher.Reset() + hasher.Write(bytecodes[i]) + hash := hasher.Sum(nil) + got = fmt.Sprintf("%v\n%v", got, hash) + } + } + var new = func() { + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + var hash = make([]byte, 32) + for i := 0; i < len(bytecodes); i++ { + hasher.Reset() + hasher.Write(bytecodes[i]) + hasher.Read(hash) + want = fmt.Sprintf("%v\n%v", want, hash) + } + } + old() + new() + if want != got { + t.Errorf("want\n%v\ngot\n%v\n", want, got) + } +} + +func BenchmarkHashing(b *testing.B) { + var bytecodes = make([][]byte, 10000) + for i := 0; i < len(bytecodes); i++ { + buf := make([]byte, 100) + rand.Read(buf) + bytecodes[i] = buf + } + var old = func() { + hasher := sha3.NewLegacyKeccak256() + for i := 0; i < len(bytecodes); i++ { + hasher.Reset() + hasher.Write(bytecodes[i]) + hasher.Sum(nil) + } + } + var new = func() { + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + var hash = make([]byte, 32) + for i := 0; i < len(bytecodes); i++ { + hasher.Reset() + hasher.Write(bytecodes[i]) + hasher.Read(hash) + } + } + b.Run("old", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + old() + } + }) + b.Run("new", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + new() + } + }) +} From 3c6665e7d62ed166d9f1cf4519ad23ab77c5cae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 7 Jan 2021 13:04:20 +0200 Subject: [PATCH 051/709] cmd/faucet: switch Facebook auth over to mobile site --- cmd/faucet/faucet.go | 10 +++++++-- cmd/faucet/faucet_test.go | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 cmd/faucet/faucet_test.go diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index a0fc28c0d4..2c0881f5a2 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// faucet is a Ether faucet backed by a light client. +// faucet is an Ether faucet backed by a light client. package main //go:generate go-bindata -nometadata -o website.go faucet.html @@ -847,7 +847,13 @@ func authFacebook(url string) (string, string, common.Address, error) { // Facebook's Graph API isn't really friendly with direct links. Still, we don't // want to do ask read permissions from users, so just load the public posts and // scrape it for the Ethereum address and profile URL. - res, err := http.Get(url) + // + // Facebook recently changed their desktop webpage to use AJAX for loading post + // content, so switch over to the mobile site for now. Will probably end up having + // to use the API eventually. + crawl := strings.Replace(url, "www.facebook.com", "m.facebook.com", 1) + + res, err := http.Get(crawl) if err != nil { return "", "", common.Address{}, err } diff --git a/cmd/faucet/faucet_test.go b/cmd/faucet/faucet_test.go new file mode 100644 index 0000000000..4f3e47084e --- /dev/null +++ b/cmd/faucet/faucet_test.go @@ -0,0 +1,43 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestFacebook(t *testing.T) { + for _, tt := range []struct { + url string + want common.Address + }{ + { + "https://www.facebook.com/fooz.gazonk/posts/2837228539847129", + common.HexToAddress("0xDeadDeaDDeaDbEefbEeFbEEfBeeFBeefBeeFbEEF"), + }, + } { + _, _, gotAddress, err := authFacebook(tt.url) + if err != nil { + t.Fatal(err) + } + if gotAddress != tt.want { + t.Fatalf("address wrong, have %v want %v", gotAddress, tt.want) + } + } +} From 165f53fc6e9e904054c67462000a19fc83dcc12f Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 8 Jan 2021 06:39:35 +0800 Subject: [PATCH 052/709] les: remove transaction propagation limits (#22125) --- les/txrelay.go | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/les/txrelay.go b/les/txrelay.go index 57f2412eba..9d29b2f234 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -25,13 +25,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -type ltrInfo struct { - tx *types.Transaction - sentTo map[*serverPeer]struct{} -} - type lesTxRelay struct { - txSent map[common.Hash]*ltrInfo + txSent map[common.Hash]*types.Transaction txPending map[common.Hash]struct{} peerList []*serverPeer peerStartPos int @@ -43,7 +38,7 @@ type lesTxRelay struct { func newLesTxRelay(ps *serverPeerSet, retriever *retrieveManager) *lesTxRelay { r := &lesTxRelay{ - txSent: make(map[common.Hash]*ltrInfo), + txSent: make(map[common.Hash]*types.Transaction), txPending: make(map[common.Hash]struct{}), retriever: retriever, stop: make(chan struct{}), @@ -80,8 +75,7 @@ func (ltrx *lesTxRelay) unregisterPeer(p *serverPeer) { } } -// send sends a list of transactions to at most a given number of peers at -// once, never resending any particular transaction to the same peer twice +// send sends a list of transactions to at most a given number of peers. func (ltrx *lesTxRelay) send(txs types.Transactions, count int) { sendTo := make(map[*serverPeer]types.Transactions) @@ -92,26 +86,18 @@ func (ltrx *lesTxRelay) send(txs types.Transactions, count int) { for _, tx := range txs { hash := tx.Hash() - ltr, ok := ltrx.txSent[hash] + _, ok := ltrx.txSent[hash] if !ok { - ltr = <rInfo{ - tx: tx, - sentTo: make(map[*serverPeer]struct{}), - } - ltrx.txSent[hash] = ltr + ltrx.txSent[hash] = tx ltrx.txPending[hash] = struct{}{} } - if len(ltrx.peerList) > 0 { cnt := count pos := ltrx.peerStartPos for { peer := ltrx.peerList[pos] - if _, ok := ltr.sentTo[peer]; !ok { - sendTo[peer] = append(sendTo[peer], tx) - ltr.sentTo[peer] = struct{}{} - cnt-- - } + sendTo[peer] = append(sendTo[peer], tx) + cnt-- if cnt == 0 { break // sent it to the desired number of peers } @@ -174,7 +160,7 @@ func (ltrx *lesTxRelay) NewHead(head common.Hash, mined []common.Hash, rollback txs := make(types.Transactions, len(ltrx.txPending)) i := 0 for hash := range ltrx.txPending { - txs[i] = ltrx.txSent[hash].tx + txs[i] = ltrx.txSent[hash] i++ } ltrx.send(txs, 1) From 6b88ab75bcbc5eaecaf5619ec730aa00f5e7c941 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 8 Jan 2021 11:17:15 +0100 Subject: [PATCH 053/709] cmd/faucet: fix nonce-gap problem (#22145) * cmd/faucet: avoid encoding for each client * cmd/faucet: fix flaw in clearing of txs, avoid sending more than necessary * cmd/faucet: fix flaw in tx cropping * cmd/faucet: revert change to not always send tx info * cmd/faucet: review fixes * cmd/faucet: revert #22018, fix order in UI * cmd/faucet: fix lock error * cmd/faucet: revert json changes * squashme --- cmd/faucet/faucet.go | 5 ++--- cmd/faucet/faucet.html | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 2c0881f5a2..b9c4e1819a 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -512,12 +512,12 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue } - f.reqs = append([]*request{{ + f.reqs = append(f.reqs, &request{ Avatar: avatar, Account: address, Time: time.Now(), Tx: signed, - }}, f.reqs...) + }) timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute grace := timeout / 288 // 24h timeout => 5m grace @@ -670,7 +670,6 @@ func send(conn *wsConn, value interface{}, timeout time.Duration) error { } conn.wlock.Lock() defer conn.wlock.Unlock() - conn.conn.SetWriteDeadline(time.Now().Add(timeout)) return conn.conn.WriteJSON(value) } diff --git a/cmd/faucet/faucet.html b/cmd/faucet/faucet.html index ba14333186..dad5ad84f2 100644 --- a/cmd/faucet/faucet.html +++ b/cmd/faucet/faucet.html @@ -177,7 +177,7 @@

How does this work?

} // Iterate over our entire local collection and re-render the funding table var content = ""; - for (var i=0; i= 0; i--) { var done = requests[i].time == ""; var elapsed = moment().unix()-moment(requests[i].time).unix(); From 889f5645b57dde5b5d4cccf1561bdb449293d2d8 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 8 Jan 2021 21:29:25 +0100 Subject: [PATCH 054/709] ethclient: better test suite for ethclient package (#22127) This commit extends the ethclient test suite and increases code coverage of the ethclient package from ~15% to >55%. These tests act as early smoke tests to signal issues in the RPC-interface. E.g. if a functionality like eth_chainId or eth_call breaks, the test will break. --- ethclient/ethclient_test.go | 259 ++++++++++++++++++++++++++++++++---- 1 file changed, 231 insertions(+), 28 deletions(-) diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 0ca72c6ee7..d700022e8f 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -17,6 +17,7 @@ package ethclient import ( + "bytes" "context" "errors" "fmt" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" ) // Verify that Client implements the ethereum interfaces. @@ -229,12 +231,48 @@ func generateTestChain() (*core.Genesis, []*types.Block) { return genesis, blocks } -func TestHeader(t *testing.T) { +func TestEthClient(t *testing.T) { backend, chain := newTestBackend(t) client, _ := backend.Attach() defer backend.Close() defer client.Close() + tests := map[string]struct { + test func(t *testing.T) + }{ + "TestHeader": { + func(t *testing.T) { testHeader(t, chain, client) }, + }, + "TestBalanceAt": { + func(t *testing.T) { testBalanceAt(t, client) }, + }, + "TestTxInBlockInterrupted": { + func(t *testing.T) { testTransactionInBlockInterrupted(t, client) }, + }, + "TestChainID": { + func(t *testing.T) { testChainID(t, client) }, + }, + "TestGetBlock": { + func(t *testing.T) { testGetBlock(t, client) }, + }, + "TestStatusFunctions": { + func(t *testing.T) { testStatusFunctions(t, client) }, + }, + "TestCallContract": { + func(t *testing.T) { testCallContract(t, client) }, + }, + "TestAtFunctions": { + func(t *testing.T) { testAtFunctions(t, client) }, + }, + } + + t.Parallel() + for name, tt := range tests { + t.Run(name, tt.test) + } +} + +func testHeader(t *testing.T, chain []*types.Block, client *rpc.Client) { tests := map[string]struct { block *big.Int want *types.Header @@ -273,12 +311,7 @@ func TestHeader(t *testing.T) { } } -func TestBalanceAt(t *testing.T) { - backend, _ := newTestBackend(t) - client, _ := backend.Attach() - defer backend.Close() - defer client.Close() - +func testBalanceAt(t *testing.T, client *rpc.Client) { tests := map[string]struct { account common.Address block *big.Int @@ -319,31 +352,32 @@ func TestBalanceAt(t *testing.T) { } } -func TestTransactionInBlockInterrupted(t *testing.T) { - backend, _ := newTestBackend(t) - client, _ := backend.Attach() - defer backend.Close() - defer client.Close() - +func testTransactionInBlockInterrupted(t *testing.T, client *rpc.Client) { ec := NewClient(client) + + // Get current block by number + block, err := ec.BlockByNumber(context.Background(), nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // Test tx in block interupted ctx, cancel := context.WithCancel(context.Background()) cancel() - tx, err := ec.TransactionInBlock(ctx, common.Hash{1}, 1) + tx, err := ec.TransactionInBlock(ctx, block.Hash(), 1) if tx != nil { t.Fatal("transaction should be nil") } - if err == nil { - t.Fatal("error should not be nil") + if err == nil || err == ethereum.NotFound { + t.Fatal("error should not be nil/notfound") + } + // Test tx in block not found + if _, err := ec.TransactionInBlock(context.Background(), block.Hash(), 1); err != ethereum.NotFound { + t.Fatal("error should be ethereum.NotFound") } } -func TestChainID(t *testing.T) { - backend, _ := newTestBackend(t) - client, _ := backend.Attach() - defer backend.Close() - defer client.Close() +func testChainID(t *testing.T, client *rpc.Client) { ec := NewClient(client) - id, err := ec.ChainID(context.Background()) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -353,13 +387,9 @@ func TestChainID(t *testing.T) { } } -func TestBlockNumber(t *testing.T) { - backend, _ := newTestBackend(t) - client, _ := backend.Attach() - defer backend.Close() - defer client.Close() +func testGetBlock(t *testing.T, client *rpc.Client) { ec := NewClient(client) - + // Get current block number blockNumber, err := ec.BlockNumber(context.Background()) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -367,4 +397,177 @@ func TestBlockNumber(t *testing.T) { if blockNumber != 1 { t.Fatalf("BlockNumber returned wrong number: %d", blockNumber) } + // Get current block by number + block, err := ec.BlockByNumber(context.Background(), new(big.Int).SetUint64(blockNumber)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if block.NumberU64() != blockNumber { + t.Fatalf("BlockByNumber returned wrong block: want %d got %d", blockNumber, block.NumberU64()) + } + // Get current block by hash + blockH, err := ec.BlockByHash(context.Background(), block.Hash()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if block.Hash() != blockH.Hash() { + t.Fatalf("BlockByHash returned wrong block: want %v got %v", block.Hash().Hex(), blockH.Hash().Hex()) + } + // Get header by number + header, err := ec.HeaderByNumber(context.Background(), new(big.Int).SetUint64(blockNumber)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if block.Header().Hash() != header.Hash() { + t.Fatalf("HeaderByNumber returned wrong header: want %v got %v", block.Header().Hash().Hex(), header.Hash().Hex()) + } + // Get header by hash + headerH, err := ec.HeaderByHash(context.Background(), block.Hash()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if block.Header().Hash() != headerH.Hash() { + t.Fatalf("HeaderByHash returned wrong header: want %v got %v", block.Header().Hash().Hex(), headerH.Hash().Hex()) + } +} + +func testStatusFunctions(t *testing.T, client *rpc.Client) { + ec := NewClient(client) + + // Sync progress + progress, err := ec.SyncProgress(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if progress != nil { + t.Fatalf("unexpected progress: %v", progress) + } + // NetworkID + networkID, err := ec.NetworkID(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if networkID.Cmp(big.NewInt(0)) != 0 { + t.Fatalf("unexpected networkID: %v", networkID) + } + // SuggestGasPrice (should suggest 1 Gwei) + gasPrice, err := ec.SuggestGasPrice(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gasPrice.Cmp(big.NewInt(1000000000)) != 0 { + t.Fatalf("unexpected gas price: %v", gasPrice) + } +} + +func testCallContract(t *testing.T, client *rpc.Client) { + ec := NewClient(client) + + // EstimateGas + msg := ethereum.CallMsg{ + From: testAddr, + To: &common.Address{}, + Gas: 21000, + GasPrice: big.NewInt(1), + Value: big.NewInt(1), + } + gas, err := ec.EstimateGas(context.Background(), msg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gas != 21000 { + t.Fatalf("unexpected gas price: %v", gas) + } + // CallContract + if _, err := ec.CallContract(context.Background(), msg, big.NewInt(1)); err != nil { + t.Fatalf("unexpected error: %v", err) + } + // PendingCallCOntract + if _, err := ec.PendingCallContract(context.Background(), msg); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func testAtFunctions(t *testing.T, client *rpc.Client) { + ec := NewClient(client) + // send a transaction for some interesting pending status + sendTransaction(ec) + time.Sleep(100 * time.Millisecond) + // Check pending transaction count + pending, err := ec.PendingTransactionCount(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending != 1 { + t.Fatalf("unexpected pending, wanted 1 got: %v", pending) + } + // Query balance + balance, err := ec.BalanceAt(context.Background(), testAddr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + penBalance, err := ec.PendingBalanceAt(context.Background(), testAddr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if balance.Cmp(penBalance) == 0 { + t.Fatalf("unexpected balance: %v %v", balance, penBalance) + } + // NonceAt + nonce, err := ec.NonceAt(context.Background(), testAddr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + penNonce, err := ec.PendingNonceAt(context.Background(), testAddr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if penNonce != nonce+1 { + t.Fatalf("unexpected nonce: %v %v", nonce, penNonce) + } + // StorageAt + storage, err := ec.StorageAt(context.Background(), testAddr, common.Hash{}, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + penStorage, err := ec.PendingStorageAt(context.Background(), testAddr, common.Hash{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !bytes.Equal(storage, penStorage) { + t.Fatalf("unexpected storage: %v %v", storage, penStorage) + } + // CodeAt + code, err := ec.CodeAt(context.Background(), testAddr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + penCode, err := ec.PendingCodeAt(context.Background(), testAddr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !bytes.Equal(code, penCode) { + t.Fatalf("unexpected code: %v %v", code, penCode) + } +} + +func sendTransaction(ec *Client) error { + // Retrieve chainID + chainID, err := ec.ChainID(context.Background()) + if err != nil { + return err + } + // Create transaction + tx := types.NewTransaction(0, common.Address{1}, big.NewInt(1), 22000, big.NewInt(1), nil) + signer := types.NewEIP155Signer(chainID) + signature, err := crypto.Sign(signer.Hash(tx).Bytes(), testKey) + if err != nil { + return err + } + signedTx, err := tx.WithSignature(signer, signature) + if err != nil { + return err + } + // Send transaction + return ec.SendTransaction(context.Background(), signedTx) } From 89030ec0b442a08fb82d4452f3132ab1602b8182 Mon Sep 17 00:00:00 2001 From: Chris Ziogas Date: Sat, 9 Jan 2021 18:29:19 +0200 Subject: [PATCH 055/709] eth/downloader: fix race condition in tests (#22140) * downloader: fix race condition in tests * eth/downloader: fix race condition in tests * Revert "downloader: fix race condition in tests" This reverts commit 108033ebc6985de83791d375b6e6647a77d28d5a. --- eth/downloader/downloader_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 6578275d0c..5de1ef3f8e 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -584,14 +584,15 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { time.Sleep(25 * time.Millisecond) tester.lock.Lock() + tester.downloader.queue.lock.Lock() + tester.downloader.queue.resultCache.lock.Lock() { - tester.downloader.queue.resultCache.lock.Lock() cached = tester.downloader.queue.resultCache.countCompleted() - tester.downloader.queue.resultCache.lock.Unlock() frozen = int(atomic.LoadUint32(&blocked)) retrieved = len(tester.ownBlocks) - } + tester.downloader.queue.resultCache.lock.Unlock() + tester.downloader.queue.lock.Unlock() tester.lock.Unlock() if cached == blockCacheMaxItems || From 5a1b38435270336fd86fe742c9951abad870b84d Mon Sep 17 00:00:00 2001 From: gary rong Date: Sun, 10 Jan 2021 19:54:15 +0800 Subject: [PATCH 056/709] core: persist bad blocks (#21827) * core: persist bad blocks * core, eth, internal: address comments * core/rawdb: add badblocks to inspector * core, eth: update * internal: revert * core, eth: only save 10 bad blocks * core/rawdb: address comments * core/rawdb: fix * core: address comments --- core/blockchain.go | 23 +------ core/rawdb/accessors_chain.go | 97 ++++++++++++++++++++++++++++++ core/rawdb/accessors_chain_test.go | 70 +++++++++++++++++++++ core/rawdb/database.go | 2 +- core/rawdb/schema.go | 8 ++- eth/api.go | 31 ++++++---- eth/api_tracer.go | 6 +- 7 files changed, 196 insertions(+), 41 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d9505dcf69..b8f483b85e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -89,7 +89,6 @@ const ( txLookupCacheLimit = 1024 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 - badBlockLimit = 10 TriesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. @@ -208,7 +207,6 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config - badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. writeLegacyJournal bool // Testing flag used to flush the snapshot journal in legacy format. @@ -227,7 +225,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par blockCache, _ := lru.New(blockCacheLimit) txLookupCache, _ := lru.New(txLookupCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) - badBlocks, _ := lru.New(badBlockLimit) bc := &BlockChain{ chainConfig: chainConfig, @@ -249,7 +246,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par futureBlocks: futureBlocks, engine: engine, vmConfig: vmConfig, - badBlocks: badBlocks, } bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) @@ -2374,26 +2370,9 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) { } } -// BadBlocks returns a list of the last 'bad blocks' that the client has seen on the network -func (bc *BlockChain) BadBlocks() []*types.Block { - blocks := make([]*types.Block, 0, bc.badBlocks.Len()) - for _, hash := range bc.badBlocks.Keys() { - if blk, exist := bc.badBlocks.Peek(hash); exist { - block := blk.(*types.Block) - blocks = append(blocks, block) - } - } - return blocks -} - -// addBadBlock adds a bad block to the bad-block LRU cache -func (bc *BlockChain) addBadBlock(block *types.Block) { - bc.badBlocks.Add(block.Hash(), block) -} - // reportBlock logs a bad block error. func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, err error) { - bc.addBadBlock(block) + rawdb.WriteBadBlock(bc.db, block) var receiptString string for i, receipt := range receipts { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index c948cdc7c6..461e1cbb17 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/binary" "math/big" + "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -702,6 +703,102 @@ func DeleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number DeleteTd(db, hash, number) } +const badBlockToKeep = 10 + +type badBlock struct { + Header *types.Header + Body *types.Body +} + +// badBlockList implements the sort interface to allow sorting a list of +// bad blocks by their number in the reverse order. +type badBlockList []*badBlock + +func (s badBlockList) Len() int { return len(s) } +func (s badBlockList) Less(i, j int) bool { + return s[i].Header.Number.Uint64() < s[j].Header.Number.Uint64() +} +func (s badBlockList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// ReadBadBlock retrieves the bad block with the corresponding block hash. +func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block { + blob, err := db.Get(badBlockKey) + if err != nil { + return nil + } + var badBlocks badBlockList + if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { + return nil + } + for _, bad := range badBlocks { + if bad.Header.Hash() == hash { + return types.NewBlockWithHeader(bad.Header).WithBody(bad.Body.Transactions, bad.Body.Uncles) + } + } + return nil +} + +// ReadAllBadBlocks retrieves all the bad blocks in the database. +// All returned blocks are sorted in reverse order by number. +func ReadAllBadBlocks(db ethdb.Reader) []*types.Block { + blob, err := db.Get(badBlockKey) + if err != nil { + return nil + } + var badBlocks badBlockList + if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { + return nil + } + var blocks []*types.Block + for _, bad := range badBlocks { + blocks = append(blocks, types.NewBlockWithHeader(bad.Header).WithBody(bad.Body.Transactions, bad.Body.Uncles)) + } + return blocks +} + +// WriteBadBlock serializes the bad block into the database. If the cumulated +// bad blocks exceeds the limitation, the oldest will be dropped. +func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { + blob, err := db.Get(badBlockKey) + if err != nil { + log.Warn("Failed to load old bad blocks", "error", err) + } + var badBlocks badBlockList + if len(blob) > 0 { + if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { + log.Crit("Failed to decode old bad blocks", "error", err) + } + } + for _, b := range badBlocks { + if b.Header.Number.Uint64() == block.NumberU64() && b.Header.Hash() == block.Hash() { + log.Info("Skip duplicated bad block", "number", block.NumberU64(), "hash", block.Hash()) + return + } + } + badBlocks = append(badBlocks, &badBlock{ + Header: block.Header(), + Body: block.Body(), + }) + sort.Sort(sort.Reverse(badBlocks)) + if len(badBlocks) > badBlockToKeep { + badBlocks = badBlocks[:badBlockToKeep] + } + data, err := rlp.EncodeToBytes(badBlocks) + if err != nil { + log.Crit("Failed to encode bad blocks", "err", err) + } + if err := db.Put(badBlockKey, data); err != nil { + log.Crit("Failed to write bad blocks", "err", err) + } +} + +// DeleteBadBlocks deletes all the bad blocks from the database +func DeleteBadBlocks(db ethdb.KeyValueWriter) { + if err := db.Delete(badBlockKey); err != nil { + log.Crit("Failed to delete bad blocks", "err", err) + } +} + // FindCommonAncestor returns the last common ancestor of two block headers func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header { for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 074c24d8fe..a5804cd309 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "math/big" + "math/rand" "os" "reflect" "testing" @@ -188,6 +189,75 @@ func TestPartialBlockStorage(t *testing.T) { } } +// Tests block storage and retrieval operations. +func TestBadBlockStorage(t *testing.T) { + db := NewMemoryDatabase() + + // Create a test block to move around the database and make sure it's really new + block := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(1), + Extra: []byte("bad block"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + }) + if entry := ReadBadBlock(db, block.Hash()); entry != nil { + t.Fatalf("Non existent block returned: %v", entry) + } + // Write and verify the block in the database + WriteBadBlock(db, block) + if entry := ReadBadBlock(db, block.Hash()); entry == nil { + t.Fatalf("Stored block not found") + } else if entry.Hash() != block.Hash() { + t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) + } + // Write one more bad block + blockTwo := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(2), + Extra: []byte("bad block two"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + }) + WriteBadBlock(db, blockTwo) + + // Write the block one again, should be filtered out. + WriteBadBlock(db, block) + badBlocks := ReadAllBadBlocks(db) + if len(badBlocks) != 2 { + t.Fatalf("Failed to load all bad blocks") + } + + // Write a bunch of bad blocks, all the blocks are should sorted + // in reverse order. The extra blocks should be truncated. + for _, n := range rand.Perm(100) { + block := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(int64(n)), + Extra: []byte("bad block"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + }) + WriteBadBlock(db, block) + } + badBlocks = ReadAllBadBlocks(db) + if len(badBlocks) != badBlockToKeep { + t.Fatalf("The number of persised bad blocks in incorrect %d", len(badBlocks)) + } + for i := 0; i < len(badBlocks)-1; i++ { + if badBlocks[i].NumberU64() < badBlocks[i+1].NumberU64() { + t.Fatalf("The bad blocks are not sorted #[%d](%d) < #[%d](%d)", i, i+1, badBlocks[i].NumberU64(), badBlocks[i+1].NumberU64()) + } + } + + // Delete all bad blocks + DeleteBadBlocks(db) + badBlocks = ReadAllBadBlocks(db) + if len(badBlocks) != 0 { + t.Fatalf("Failed to delete bad blocks") + } +} + // Tests block total difficulty storage and retrieval operations. func TestTdStorage(t *testing.T) { db := NewMemoryDatabase() diff --git a/core/rawdb/database.go b/core/rawdb/database.go index b01a31ebcd..a2cc9c7be9 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -355,7 +355,7 @@ func InspectDatabase(db ethdb.Database) error { bloomTrieNodes.Add(size) default: var accounted bool - for _, meta := range [][]byte{databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey, uncleanShutdownKey} { + for _, meta := range [][]byte{databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey, uncleanShutdownKey, badBlockKey} { if bytes.Equal(key, meta) { metadata.Add(size) accounted = true diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 2aabfd3baa..9749a30d8a 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -66,6 +66,12 @@ var ( // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") + // badBlockKey tracks the list of bad blocks seen by local + badBlockKey = []byte("InvalidBlock") + + // uncleanShutdownKey tracks the list of local crashes + uncleanShutdownKey = []byte("unclean-shutdown") // config prefix for the db + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td @@ -84,8 +90,6 @@ var ( preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage configPrefix = []byte("ethereum-config-") // config prefix for the db - uncleanShutdownKey = []byte("unclean-shutdown") // config prefix for the db - // Chain index prefixes (use `i` + single byte to avoid mixing data types). BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress diff --git a/eth/api.go b/eth/api.go index fd35656476..6cd0fd7005 100644 --- a/eth/api.go +++ b/eth/api.go @@ -331,22 +331,29 @@ type BadBlockArgs struct { // GetBadBlocks returns a list of the last 'bad blocks' that the client has seen on the network // and returns them as a JSON list of block-hashes func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { - blocks := api.eth.BlockChain().BadBlocks() - results := make([]*BadBlockArgs, len(blocks)) - - var err error - for i, block := range blocks { - results[i] = &BadBlockArgs{ - Hash: block.Hash(), - } + var ( + err error + blocks = rawdb.ReadAllBadBlocks(api.eth.chainDb) + results = make([]*BadBlockArgs, 0, len(blocks)) + ) + for _, block := range blocks { + var ( + blockRlp string + blockJSON map[string]interface{} + ) if rlpBytes, err := rlp.EncodeToBytes(block); err != nil { - results[i].RLP = err.Error() // Hacky, but hey, it works + blockRlp = err.Error() // Hacky, but hey, it works } else { - results[i].RLP = fmt.Sprintf("0x%x", rlpBytes) + blockRlp = fmt.Sprintf("0x%x", rlpBytes) } - if results[i].Block, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { - results[i].Block = map[string]interface{}{"error": err.Error()} + if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { + blockJSON = map[string]interface{}{"error": err.Error()} } + results = append(results, &BadBlockArgs{ + Hash: block.Hash(), + RLP: blockRlp, + Block: blockJSON, + }) } return results, nil } diff --git a/eth/api_tracer.go b/eth/api_tracer.go index c68b762255..8e71945ee2 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -404,8 +404,7 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, // EVM against a block pulled from the pool of bad ones and returns them as a JSON // object. func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { - blocks := api.eth.blockchain.BadBlocks() - for _, block := range blocks { + for _, block := range rawdb.ReadAllBadBlocks(api.eth.chainDb) { if block.Hash() == hash { return api.traceBlock(ctx, block, config) } @@ -428,8 +427,7 @@ func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, hash c // execution of EVM against a block pulled from the pool of bad ones to the // local file system and returns a list of files to the caller. func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { - blocks := api.eth.blockchain.BadBlocks() - for _, block := range blocks { + for _, block := range rawdb.ReadAllBadBlocks(api.eth.chainDb) { if block.Hash() == hash { return api.standardTraceBlockToFile(ctx, block, config) } From ab5e3f400f9dd3832508208ecef8b75c681728ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 11 Jan 2021 10:31:03 +0200 Subject: [PATCH 057/709] common/prque: pull in tests and benchmarks from upstream --- common/prque/prque_test.go | 130 ++++++++++++++++++++++++++++++++++++ common/prque/sstack_test.go | 100 +++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 common/prque/prque_test.go create mode 100644 common/prque/sstack_test.go diff --git a/common/prque/prque_test.go b/common/prque/prque_test.go new file mode 100644 index 0000000000..1cffcebad4 --- /dev/null +++ b/common/prque/prque_test.go @@ -0,0 +1,130 @@ +// CookieJar - A contestant's algorithm toolbox +// Copyright (c) 2013 Peter Szilagyi. All rights reserved. +// +// CookieJar is dual licensed: use of this source code is governed by a BSD +// license that can be found in the LICENSE file. Alternatively, the CookieJar +// toolbox may be used in accordance with the terms and conditions contained +// in a signed written agreement between you and the author(s). + +package prque + +import ( + "math/rand" + "testing" +) + +func TestPrque(t *testing.T) { + // Generate a batch of random data and a specific priority order + size := 16 * blockSize + prio := rand.Perm(size) + data := make([]int, size) + for i := 0; i < size; i++ { + data[i] = rand.Int() + } + queue := New(nil) + for rep := 0; rep < 2; rep++ { + // Fill a priority queue with the above data + for i := 0; i < size; i++ { + queue.Push(data[i], int64(prio[i])) + if queue.Size() != i+1 { + t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1) + } + } + // Create a map the values to the priorities for easier verification + dict := make(map[int64]int) + for i := 0; i < size; i++ { + dict[int64(prio[i])] = data[i] + } + // Pop out the elements in priority order and verify them + prevPrio := int64(size + 1) + for !queue.Empty() { + val, prio := queue.Pop() + if prio > prevPrio { + t.Errorf("invalid priority order: %v after %v.", prio, prevPrio) + } + prevPrio = prio + if val != dict[prio] { + t.Errorf("push/pop mismatch: have %v, want %v.", val, dict[prio]) + } + delete(dict, prio) + } + } +} + +func TestReset(t *testing.T) { + // Generate a batch of random data and a specific priority order + size := 16 * blockSize + prio := rand.Perm(size) + data := make([]int, size) + for i := 0; i < size; i++ { + data[i] = rand.Int() + } + queue := New(nil) + for rep := 0; rep < 2; rep++ { + // Fill a priority queue with the above data + for i := 0; i < size; i++ { + queue.Push(data[i], int64(prio[i])) + if queue.Size() != i+1 { + t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1) + } + } + // Create a map the values to the priorities for easier verification + dict := make(map[int64]int) + for i := 0; i < size; i++ { + dict[int64(prio[i])] = data[i] + } + // Pop out half the elements in priority order and verify them + prevPrio := int64(size + 1) + for i := 0; i < size/2; i++ { + val, prio := queue.Pop() + if prio > prevPrio { + t.Errorf("invalid priority order: %v after %v.", prio, prevPrio) + } + prevPrio = prio + if val != dict[prio] { + t.Errorf("push/pop mismatch: have %v, want %v.", val, dict[prio]) + } + delete(dict, prio) + } + // Reset and ensure it's empty + queue.Reset() + if !queue.Empty() { + t.Errorf("priority queue not empty after reset: %v", queue) + } + } +} + +func BenchmarkPush(b *testing.B) { + // Create some initial data + data := make([]int, b.N) + prio := make([]int64, b.N) + for i := 0; i < len(data); i++ { + data[i] = rand.Int() + prio[i] = rand.Int63() + } + // Execute the benchmark + b.ResetTimer() + queue := New(nil) + for i := 0; i < len(data); i++ { + queue.Push(data[i], prio[i]) + } +} + +func BenchmarkPop(b *testing.B) { + // Create some initial data + data := make([]int, b.N) + prio := make([]int64, b.N) + for i := 0; i < len(data); i++ { + data[i] = rand.Int() + prio[i] = rand.Int63() + } + queue := New(nil) + for i := 0; i < len(data); i++ { + queue.Push(data[i], prio[i]) + } + // Execute the benchmark + b.ResetTimer() + for !queue.Empty() { + queue.Pop() + } +} diff --git a/common/prque/sstack_test.go b/common/prque/sstack_test.go new file mode 100644 index 0000000000..2ff093579d --- /dev/null +++ b/common/prque/sstack_test.go @@ -0,0 +1,100 @@ +// CookieJar - A contestant's algorithm toolbox +// Copyright (c) 2013 Peter Szilagyi. All rights reserved. +// +// CookieJar is dual licensed: use of this source code is governed by a BSD +// license that can be found in the LICENSE file. Alternatively, the CookieJar +// toolbox may be used in accordance with the terms and conditions contained +// in a signed written agreement between you and the author(s). + +package prque + +import ( + "math/rand" + "sort" + "testing" +) + +func TestSstack(t *testing.T) { + // Create some initial data + size := 16 * blockSize + data := make([]*item, size) + for i := 0; i < size; i++ { + data[i] = &item{rand.Int(), rand.Int63()} + } + stack := newSstack(nil) + for rep := 0; rep < 2; rep++ { + // Push all the data into the stack, pop out every second + secs := []*item{} + for i := 0; i < size; i++ { + stack.Push(data[i]) + if i%2 == 0 { + secs = append(secs, stack.Pop().(*item)) + } + } + rest := []*item{} + for stack.Len() > 0 { + rest = append(rest, stack.Pop().(*item)) + } + // Make sure the contents of the resulting slices are ok + for i := 0; i < size; i++ { + if i%2 == 0 && data[i] != secs[i/2] { + t.Errorf("push/pop mismatch: have %v, want %v.", secs[i/2], data[i]) + } + if i%2 == 1 && data[i] != rest[len(rest)-i/2-1] { + t.Errorf("push/pop mismatch: have %v, want %v.", rest[len(rest)-i/2-1], data[i]) + } + } + } +} + +func TestSstackSort(t *testing.T) { + // Create some initial data + size := 16 * blockSize + data := make([]*item, size) + for i := 0; i < size; i++ { + data[i] = &item{rand.Int(), int64(i)} + } + // Push all the data into the stack + stack := newSstack(nil) + for _, val := range data { + stack.Push(val) + } + // Sort and pop the stack contents (should reverse the order) + sort.Sort(stack) + for _, val := range data { + out := stack.Pop() + if out != val { + t.Errorf("push/pop mismatch after sort: have %v, want %v.", out, val) + } + } +} + +func TestSstackReset(t *testing.T) { + // Create some initial data + size := 16 * blockSize + data := make([]*item, size) + for i := 0; i < size; i++ { + data[i] = &item{rand.Int(), rand.Int63()} + } + stack := newSstack(nil) + for rep := 0; rep < 2; rep++ { + // Push all the data into the stack, pop out every second + secs := []*item{} + for i := 0; i < size; i++ { + stack.Push(data[i]) + if i%2 == 0 { + secs = append(secs, stack.Pop().(*item)) + } + } + // Reset and verify both pulled and stack contents + stack.Reset() + if stack.Len() != 0 { + t.Errorf("stack not empty after reset: %v", stack) + } + for i := 0; i < size; i++ { + if i%2 == 0 && data[i] != secs[i/2] { + t.Errorf("push/pop mismatch: have %v, want %v.", secs[i/2], data[i]) + } + } + } +} From 49c2816d5464ed208389d52ed7cf47f35630b1c2 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 11 Jan 2021 12:53:13 +0100 Subject: [PATCH 058/709] eth: improve log message (#22146) * eth: fixed typos * eth: fixed log message --- eth/handler.go | 6 +++--- eth/peerset.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 76a429f6d3..a9506c4995 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -437,14 +437,14 @@ func (h *handler) BroadcastTransactions(txs types.Transactions, propagate bool) // Broadcast transactions to a batch of peers not knowing about it if propagate { for _, tx := range txs { - peers := h.peers.ethPeersWithoutTransacion(tx.Hash()) + peers := h.peers.ethPeersWithoutTransaction(tx.Hash()) // Send the block to a subset of our peers transfer := peers[:int(math.Sqrt(float64(len(peers))))] for _, peer := range transfer { txset[peer] = append(txset[peer], tx.Hash()) } - log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) + log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(transfer)) } for peer, hashes := range txset { peer.AsyncSendTransactions(hashes) @@ -453,7 +453,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions, propagate bool) } // Otherwise only broadcast the announcement to peers for _, tx := range txs { - peers := h.peers.ethPeersWithoutTransacion(tx.Hash()) + peers := h.peers.ethPeersWithoutTransaction(tx.Hash()) for _, peer := range peers { annos[peer] = append(annos[peer], tx.Hash()) } diff --git a/eth/peerset.go b/eth/peerset.go index 9b584ec320..bf5785ff3f 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -243,9 +243,9 @@ func (ps *peerSet) ethPeersWithoutBlock(hash common.Hash) []*ethPeer { return list } -// ethPeersWithoutTransacion retrieves a list of `eth` peers that do not have a +// ethPeersWithoutTransaction retrieves a list of `eth` peers that do not have a // given transaction in their set of known hashes. -func (ps *peerSet) ethPeersWithoutTransacion(hash common.Hash) []*ethPeer { +func (ps *peerSet) ethPeersWithoutTransaction(hash common.Hash) []*ethPeer { ps.lock.RLock() defer ps.lock.RUnlock() From 39b3b8ffb44983a260031d5077226d952ddfb9bf Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 11 Jan 2021 14:55:42 +0100 Subject: [PATCH 059/709] graphql: fix issue with unmarshalling int32 into `Long` type #22153 --- graphql/schema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphql/schema.go b/graphql/schema.go index 1fdc370040..6ea63db636 100644 --- a/graphql/schema.go +++ b/graphql/schema.go @@ -300,7 +300,7 @@ const schema string = ` block(number: Long, hash: Bytes32): Block # Blocks returns all the blocks between two numbers, inclusive. If # to is not supplied, it defaults to the most recent known block. - blocks(from: Long!, to: Long): [Block!]! + blocks(from: Long, to: Long): [Block!]! # Pending returns the current pending state. pending: Pending! # Transaction returns a transaction specified by its hash. From 984e752ce52b09142ed936c337456df27128e3bc Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 12 Jan 2021 10:52:13 +0100 Subject: [PATCH 060/709] eth: return error from eth_chainID during sync before EIP-155 activates (#21686) This changes the chainID RPC method to return an error when EIP-155 is not yet active at the current block height. It used to simply return zero in this case, but that's confusing. --- eth/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/api.go b/eth/api.go index 6cd0fd7005..be1dcbb52a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -67,12 +67,12 @@ func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { } // ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. -func (api *PublicEthereumAPI) ChainId() hexutil.Uint64 { - chainID := new(big.Int) +func (api *PublicEthereumAPI) ChainId() (hexutil.Uint64, error) { + // if current block is at or past the EIP-155 replay-protection fork block, return chainID from config if config := api.e.blockchain.Config(); config.IsEIP155(api.e.blockchain.CurrentBlock().Number()) { - chainID = config.ChainID + return (hexutil.Uint64)(config.ChainID.Uint64()), nil } - return (hexutil.Uint64)(chainID.Uint64()) + return hexutil.Uint64(0), fmt.Errorf("chain not synced beyond EIP-155 replay-protection fork block") } // PublicMinerAPI provides an API to control the miner. From 23f837c38827d7c1ea67b71eb6f79934562f0d98 Mon Sep 17 00:00:00 2001 From: meowsbits Date: Tue, 12 Jan 2021 08:50:11 -0600 Subject: [PATCH 061/709] cmd/utils: avoid making console preloads absolute (#22109) Resolves https://github.com/etclabscore/core-geth/issues/273 jsre.JSRE already handles establishing preload file paths relative to the 'assets' path (aka docroot), where it joins the assets dir and the file path if relative, or uses the file path only if absolute. The duplication of this logic by MakeConsolePreloads caused preloaded files to have paths which contained duplicate references to the assets dir path. Date: 2020-12-30 08:25:01-06:00 Signed-off-by: meows --- cmd/utils/flags.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 684e3428ba..fc64539a7c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1908,9 +1908,8 @@ func MakeConsolePreloads(ctx *cli.Context) []string { // Otherwise resolve absolute paths and return them var preloads []string - assets := ctx.GlobalString(JSpathFlag.Name) for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") { - preloads = append(preloads, common.AbsolutePath(assets, strings.TrimSpace(file))) + preloads = append(preloads, strings.TrimSpace(file)) } return preloads } From 93a89b26819c9b21ff32ab650b8916076b53b471 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 12 Jan 2021 17:39:31 +0100 Subject: [PATCH 062/709] go.mod: use github.com/holiman/bloomfilter/v2 (#22044) * deps: use improved bloom filter implementation * eth/handler, trie: use 4 keys for syncbloom + minor fixes * eth/protocols, trie: revert change on syncbloom method signature --- core/state/snapshot/difflayer.go | 2 +- go.mod | 7 ++--- go.sum | 34 ++------------------- trie/sync.go | 2 +- trie/sync_bloom.go | 51 ++++++++------------------------ 5 files changed, 18 insertions(+), 78 deletions(-) diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 3be78cae88..cc82df9a54 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" - "github.com/steakknife/bloomfilter" + bloomfilter "github.com/holiman/bloomfilter/v2" ) var ( diff --git a/go.mod b/go.mod index 0f47809ef6..57c15a34f6 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 - github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 // indirect github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c github.com/fatih/color v1.3.0 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc @@ -32,6 +31,7 @@ require ( github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 github.com/hashicorp/golang-lru v0.5.4 + github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 github.com/huin/goupnp v1.0.0 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 @@ -54,14 +54,11 @@ require ( github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect github.com/shirou/gopsutil v2.20.5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 - github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 - github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect github.com/stretchr/testify v1.4.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/mobile v0.0.0-20200801112145-973feb4309de // indirect golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 golang.org/x/text v0.3.3 @@ -69,5 +66,5 @@ require ( gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 - gotest.tools v2.2.0+incompatible + gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index d1bd608585..d063cd8f08 100644 --- a/go.sum +++ b/go.sum @@ -20,7 +20,6 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -60,8 +59,6 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmak github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 h1:NgO45/5mBLRVfiXerEFzH6ikcZ7DNRPS639xFg3ENzU= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= @@ -95,8 +92,6 @@ github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= -github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -104,8 +99,6 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= @@ -114,6 +107,8 @@ github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKx github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -200,10 +195,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -217,26 +208,12 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de h1:OVJ6QQUBAesB8CZijKDSsXX7xYVtUhrkY0gwMfbi4p4= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayERXBdfZjUYoXEf5BTfDfh8= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= @@ -244,7 +221,6 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -266,12 +242,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= diff --git a/trie/sync.go b/trie/sync.go index bc93ddd3fb..05b9f55d33 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -410,7 +410,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // Bloom filter says this might be a duplicate, double check. // If database says yes, then at least the trie node is present // and we hold the assumption that it's NOT legacy contract code. - if blob := rawdb.ReadTrieNode(s.database, common.BytesToHash(node)); len(blob) > 0 { + if blob := rawdb.ReadTrieNode(s.database, hash); len(blob) > 0 { continue } // False positive, bump fault meter diff --git a/trie/sync_bloom.go b/trie/sync_bloom.go index 979f4748f3..1afcce21da 100644 --- a/trie/sync_bloom.go +++ b/trie/sync_bloom.go @@ -19,7 +19,6 @@ package trie import ( "encoding/binary" "fmt" - "math" "sync" "sync/atomic" "time" @@ -29,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/steakknife/bloomfilter" + bloomfilter "github.com/holiman/bloomfilter/v2" ) var ( @@ -41,18 +40,6 @@ var ( bloomErrorGauge = metrics.NewRegisteredGauge("trie/bloom/error", nil) ) -// syncBloomHasher is a wrapper around a byte blob to satisfy the interface API -// requirements of the bloom library used. It's used to convert a trie hash or -// contract code hash into a 64 bit mini hash. -type syncBloomHasher []byte - -func (f syncBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (f syncBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (f syncBloomHasher) Reset() { panic("not implemented") } -func (f syncBloomHasher) BlockSize() int { panic("not implemented") } -func (f syncBloomHasher) Size() int { return 8 } -func (f syncBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } - // SyncBloom is a bloom filter used during fast sync to quickly decide if a trie // node or contract code already exists on disk or not. It self populates from the // provided disk database on creation in a background thread and will only start @@ -69,7 +56,7 @@ type SyncBloom struct { // initializes it from the database. The bloom is hard coded to use 3 filters. func NewSyncBloom(memory uint64, database ethdb.Iteratee) *SyncBloom { // Create the bloom filter to track known trie nodes - bloom, err := bloomfilter.New(memory*1024*1024*8, 3) + bloom, err := bloomfilter.New(memory*1024*1024*8, 4) if err != nil { panic(fmt.Sprintf("failed to create bloom: %v", err)) } @@ -110,12 +97,11 @@ func (b *SyncBloom) init(database ethdb.Iteratee) { // If the database entry is a trie node, add it to the bloom key := it.Key() if len(key) == common.HashLength { - b.bloom.Add(syncBloomHasher(key)) + b.bloom.AddHash(binary.BigEndian.Uint64(key)) bloomLoadMeter.Mark(1) - } - // If the database entry is a contract code, add it to the bloom - if ok, hash := rawdb.IsCodeKey(key); ok { - b.bloom.Add(syncBloomHasher(hash)) + } else if ok, hash := rawdb.IsCodeKey(key); ok { + // If the database entry is a contract code, add it to the bloom + b.bloom.AddHash(binary.BigEndian.Uint64(hash)) bloomLoadMeter.Mark(1) } // If enough time elapsed since the last iterator swap, restart @@ -125,14 +111,14 @@ func (b *SyncBloom) init(database ethdb.Iteratee) { it.Release() it = database.NewIterator(nil, key) - log.Info("Initializing state bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Initializing state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability(), "elapsed", common.PrettyDuration(time.Since(start))) swap = time.Now() } } it.Release() // Mark the bloom filter inited and return - log.Info("Initialized state bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Initialized state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability(), "elapsed", common.PrettyDuration(time.Since(start))) atomic.StoreUint32(&b.inited, 1) } @@ -141,7 +127,7 @@ func (b *SyncBloom) init(database ethdb.Iteratee) { func (b *SyncBloom) meter() { for { // Report the current error ration. No floats, lame, scale it up. - bloomErrorGauge.Update(int64(b.errorRate() * 100000)) + bloomErrorGauge.Update(int64(b.bloom.FalsePosititveProbability() * 100000)) // Wait one second, but check termination more frequently for i := 0; i < 10; i++ { @@ -162,7 +148,7 @@ func (b *SyncBloom) Close() error { b.pend.Wait() // Wipe the bloom, but mark it "uninited" just in case someone attempts an access - log.Info("Deallocated state bloom", "items", b.bloom.N(), "errorrate", b.errorRate()) + log.Info("Deallocated state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability()) atomic.StoreUint32(&b.inited, 0) b.bloom = nil @@ -175,7 +161,7 @@ func (b *SyncBloom) Add(hash []byte) { if atomic.LoadUint32(&b.closed) == 1 { return } - b.bloom.Add(syncBloomHasher(hash)) + b.bloom.AddHash(binary.BigEndian.Uint64(hash)) bloomAddMeter.Mark(1) } @@ -193,22 +179,9 @@ func (b *SyncBloom) Contains(hash []byte) bool { return true } // Bloom initialized, check the real one and report any successful misses - maybe := b.bloom.Contains(syncBloomHasher(hash)) + maybe := b.bloom.ContainsHash(binary.BigEndian.Uint64(hash)) if !maybe { bloomMissMeter.Mark(1) } return maybe } - -// errorRate calculates the probability of a random containment test returning a -// false positive. -// -// We're calculating it ourselves because the bloom library we used missed a -// parentheses in the formula and calculates it wrong. And it's discontinued... -func (b *SyncBloom) errorRate() float64 { - k := float64(b.bloom.K()) - n := float64(b.bloom.N()) - m := float64(b.bloom.M()) - - return math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k) -} From c7a6be163f07c967172dfcb3fe2ef25ff08dd4de Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Wed, 13 Jan 2021 10:14:36 +0000 Subject: [PATCH 063/709] cmd/utils: don't enumerate USB unless --usb is set (#22130) USB enumeration still occured. Make sure it will only occur if --usb is set. This also deprecates the 'NoUSB' config file option in favor of a new option 'USB'. --- cmd/geth/accountcmd_test.go | 10 +++++----- cmd/geth/consolecmd_test.go | 2 +- cmd/geth/dao_test.go | 4 ++-- cmd/geth/genesis_test.go | 4 ++-- cmd/geth/les_test.go | 6 +++--- cmd/utils/flags.go | 8 ++++---- miner/stress_clique.go | 1 - miner/stress_ethash.go | 1 - node/api_test.go | 1 - node/config.go | 5 ++++- p2p/simulations/adapters/exec.go | 1 - p2p/simulations/adapters/inproc.go | 1 - 12 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cmd/geth/accountcmd_test.go b/cmd/geth/accountcmd_test.go index 04f55e9e7a..9455eeda36 100644 --- a/cmd/geth/accountcmd_test.go +++ b/cmd/geth/accountcmd_test.go @@ -43,13 +43,13 @@ func tmpDatadirWithKeystore(t *testing.T) string { } func TestAccountListEmpty(t *testing.T) { - geth := runGeth(t, "--nousb", "account", "list") + geth := runGeth(t, "account", "list") geth.ExpectExit() } func TestAccountList(t *testing.T) { datadir := tmpDatadirWithKeystore(t) - geth := runGeth(t, "--nousb", "account", "list", "--datadir", datadir) + geth := runGeth(t, "account", "list", "--datadir", datadir) defer geth.ExpectExit() if runtime.GOOS == "windows" { geth.Expect(` @@ -139,7 +139,7 @@ Fatal: Passwords do not match func TestAccountUpdate(t *testing.T) { datadir := tmpDatadirWithKeystore(t) - geth := runGeth(t, "--nousb", "account", "update", + geth := runGeth(t, "account", "update", "--datadir", datadir, "--lightkdf", "f466859ead1932d743d622cb74fc058882e8648a") defer geth.ExpectExit() @@ -154,7 +154,7 @@ Repeat password: {{.InputLine "foobar2"}} } func TestWalletImport(t *testing.T) { - geth := runGeth(t, "--nousb", "wallet", "import", "--lightkdf", "testdata/guswallet.json") + geth := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json") defer geth.ExpectExit() geth.Expect(` !! Unsupported terminal, password will be echoed. @@ -169,7 +169,7 @@ Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f} } func TestWalletImportBadPassword(t *testing.T) { - geth := runGeth(t, "--nousb", "wallet", "import", "--lightkdf", "testdata/guswallet.json") + geth := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json") defer geth.ExpectExit() geth.Expect(` !! Unsupported terminal, password will be echoed. diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index b0555c45d7..93f9e3d0d9 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -42,7 +42,7 @@ func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --ropsten to make the 'writing genesis to disk' faster (no accounts) // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--ropsten", "--nousb", "--networkid", "1337", "--syncmode=full", "--port", "0", + allArgs := []string{"--ropsten", "--networkid", "1337", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64"} return runGeth(t, append(allArgs, args...)...) } diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index df7f14fdb8..29b1a7f474 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -115,10 +115,10 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc if err := ioutil.WriteFile(json, []byte(genesis), 0600); err != nil { t.Fatalf("test %d: failed to write genesis file: %v", test, err) } - runGeth(t, "--datadir", datadir, "--nousb", "--networkid", "1337", "init", json).WaitExit() + runGeth(t, "--datadir", datadir, "--networkid", "1337", "init", json).WaitExit() } else { // Force chain initialization - args := []string{"--port", "0", "--nousb", "--networkid", "1337", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir} + args := []string{"--port", "0", "--networkid", "1337", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir} runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...).WaitExit() } // Retrieve the DAO config flag from the database diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index 0651c32cad..cbc1b38374 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -81,10 +81,10 @@ func TestCustomGenesis(t *testing.T) { if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil { t.Fatalf("test %d: failed to write genesis file: %v", i, err) } - runGeth(t, "--nousb", "--datadir", datadir, "init", json).WaitExit() + runGeth(t, "--datadir", datadir, "init", json).WaitExit() // Query the custom genesis block - geth := runGeth(t, "--nousb", "--networkid", "1337", "--syncmode=full", + geth := runGeth(t, "--networkid", "1337", "--syncmode=full", "--datadir", datadir, "--maxpeers", "0", "--port", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--exec", tt.query, "console") diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index d2f63ac7bd..053ce96aa3 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -130,7 +130,7 @@ var nextIPC = uint32(0) func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc { ipcName := fmt.Sprintf("geth-%d.ipc", atomic.AddUint32(&nextIPC, 1)) - args = append([]string{"--networkid=42", "--port=0", "--nousb", "--ipcpath", ipcName}, args...) + args = append([]string{"--networkid=42", "--port=0", "--ipcpath", ipcName}, args...) t.Logf("Starting %v with rpc: %v", name, args) g := &gethrpc{ @@ -148,7 +148,7 @@ func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc { } func initGeth(t *testing.T) string { - args := []string{"--nousb", "--networkid=42", "init", "./testdata/clique.json"} + args := []string{"--networkid=42", "init", "./testdata/clique.json"} t.Logf("Initializing geth: %v ", args) g := runGeth(t, args...) datadir := g.Datadir @@ -159,7 +159,7 @@ func initGeth(t *testing.T) string { func startLightServer(t *testing.T) *gethrpc { datadir := initGeth(t) t.Logf("Importing keys to geth") - runGeth(t, "--nousb", "--datadir", datadir, "--password", "./testdata/password.txt", "account", "import", "./testdata/key.prv", "--lightkdf").WaitExit() + runGeth(t, "--datadir", datadir, "--password", "./testdata/password.txt", "account", "import", "./testdata/key.prv", "--lightkdf").WaitExit() account := "0x02f0d131f1f97aef08aec6e3291b957d9efe7105" server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--mine", "--light.serve=100", "--light.maxpeers=1", "--nodiscover", "--nat=extip:127.0.0.1", "--verbosity=4") return server diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fc64539a7c..6f4da58320 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1233,12 +1233,12 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LightKDFFlag.Name) { cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name) } - if ctx.GlobalIsSet(USBFlag.Name) { - cfg.NoUSB = !ctx.GlobalBool(USBFlag.Name) - } - if ctx.GlobalIsSet(NoUSBFlag.Name) { + if ctx.GlobalIsSet(NoUSBFlag.Name) || cfg.NoUSB { log.Warn("Option nousb is deprecated and USB is deactivated by default. Use --usb to enable") } + if ctx.GlobalIsSet(USBFlag.Name) { + cfg.USB = ctx.GlobalBool(USBFlag.Name) + } if ctx.GlobalIsSet(InsecureUnlockAllowedFlag.Name) { cfg.InsecureUnlockAllowed = ctx.GlobalBool(InsecureUnlockAllowedFlag.Name) } diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 21538aaaed..a0fd596098 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -178,7 +178,6 @@ func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { NoDiscovery: true, MaxPeers: 25, }, - NoUSB: true, } // Start the node and configure a full Ethereum node on it stack, err := node.New(config) diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 5a7e7685a6..1713af9d23 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -155,7 +155,6 @@ func makeMiner(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { NoDiscovery: true, MaxPeers: 25, }, - NoUSB: true, UseLightweightKDF: true, } // Create the node and configure a full Ethereum node on it diff --git a/node/api_test.go b/node/api_test.go index e4c08962c3..a07ce833c4 100644 --- a/node/api_test.go +++ b/node/api_test.go @@ -248,7 +248,6 @@ func TestStartRPC(t *testing.T) { // Apply some sane defaults. config := test.cfg // config.Logger = testlog.Logger(t, log.LvlDebug) - config.NoUSB = true config.P2P.NoDiscovery = true // Create Node. diff --git a/node/config.go b/node/config.go index 55532632cd..61e41cd7d0 100644 --- a/node/config.go +++ b/node/config.go @@ -95,6 +95,9 @@ type Config struct { // NoUSB disables hardware wallet monitoring and connectivity. NoUSB bool `toml:",omitempty"` + // USB enables hardware wallet monitoring and connectivity. + USB bool `toml:",omitempty"` + // SmartCardDaemonPath is the path to the smartcard daemon's socket SmartCardDaemonPath string `toml:",omitempty"` @@ -476,7 +479,7 @@ func makeAccountManager(conf *Config) (*accounts.Manager, string, error) { // we can have both, but it's very confusing for the user to see the same // accounts in both externally and locally, plus very racey. backends = append(backends, keystore.NewKeyStore(keydir, scryptN, scryptP)) - if !conf.NoUSB { + if conf.USB { // Start a USB hub for Ledger hardware wallets if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 0ed3deab38..35ccdfb068 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -115,7 +115,6 @@ func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) { conf.Stack.P2P.EnableMsgEvents = config.EnableMsgEvents conf.Stack.P2P.NoDiscovery = true conf.Stack.P2P.NAT = nil - conf.Stack.NoUSB = true // Listen on a localhost port, which we set when we // initialise NodeConfig (usually a random port) diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 4fc7abc06a..1cb26a8ea0 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -100,7 +100,6 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { EnableMsgEvents: config.EnableMsgEvents, }, ExternalSigner: config.ExternalSigner, - NoUSB: true, Logger: log.New("node.id", id.String()), }) if err != nil { From c94081774f6c8c43e10deb5735c3f9a09a7bcd04 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 13 Jan 2021 11:29:28 +0100 Subject: [PATCH 064/709] tests: update the reference tests (#22009) --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index 7497b116a0..6c863f03be 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 7497b116a019beb26215cbea4028df068dea06be +Subproject commit 6c863f03bee8d7a66bb7a028a9f880a86a5f4975 From 6296211a3ee2502bdec73256eadaf9185fb4d946 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 13 Jan 2021 11:42:26 +0100 Subject: [PATCH 065/709] graphql: fix spurious error in test (#22164) This solves an issue in graphql tests: graphql_test.go:38: could not create new node: datadir already used by another process --- graphql/graphql_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index fc62da1813..4a1644a61b 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -33,7 +33,14 @@ import ( ) func TestBuildSchema(t *testing.T) { - stack, err := node.New(&node.DefaultConfig) + ddir, err := ioutil.TempDir("", "graphql-buildschema") + if err != nil { + t.Fatalf("failed to create temporary datadir: %v", err) + } + // Copy config + conf := node.DefaultConfig + conf.DataDir = ddir + stack, err := node.New(&conf) if err != nil { t.Fatalf("could not create new node: %v", err) } @@ -157,6 +164,7 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { if err != nil { t.Fatalf("could not read from response body: %v", err) } + resp.Body.Close() // make sure the request is not handled successfully if want, have := "404 page not found\n", string(bodyBytes); have != want { t.Errorf("have:\n%v\nwant:\n%v", have, want) From 2aaff0ad76991be8851ae30454d2e2e967704102 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 13 Jan 2021 11:44:20 +0100 Subject: [PATCH 066/709] consensus/ethash: increase seal timeout for tests (#22162) It seems that the 2 second timeout is not enough for Travis CI: --- FAIL: TestTestMode (2.00s) ethash_test.go:53: sealing result timeout --- consensus/ethash/ethash_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index fdfd81320f..adbf1ccfeb 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -49,7 +49,7 @@ func TestTestMode(t *testing.T) { if err := ethash.VerifySeal(nil, header); err != nil { t.Fatalf("unexpected verification error: %v", err) } - case <-time.NewTimer(2 * time.Second).C: + case <-time.NewTimer(4 * time.Second).C: t.Error("sealing result timeout") } } From 96157a897be2032be5fdb87f947fbe5df8a53bd4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 13 Jan 2021 22:43:07 +0100 Subject: [PATCH 067/709] graphql: fix spurious travis failure (#22166) The tests sometimes failed with certain go versions because the behavior of http.Server.Shutdown changed over time. A bug that was fixed in Go 1.15 could cause active connections on unrelated servers to close unexpectedly. This is fixed by avoiding use of the same port number in all tests. --- graphql/graphql_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 4a1644a61b..e9c129c44c 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -131,7 +131,7 @@ func TestGraphQLBlockSerialization(t *testing.T) { code: 200, }, } { - resp, err := http.Post(fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), "application/json", strings.NewReader(tt.body)) + resp, err := http.Post(fmt.Sprintf("%s/graphql", stack.HTTPEndpoint()), "application/json", strings.NewReader(tt.body)) if err != nil { t.Fatalf("could not post: %v", err) } @@ -156,7 +156,7 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { t.Fatalf("could not start node: %v", err) } body := strings.NewReader(`{"query": "{block{number}}","variables": null}`) - resp, err := http.Post(fmt.Sprintf("http://%s/graphql", "127.0.0.1:9393"), "application/json", body) + resp, err := http.Post(fmt.Sprintf("%s/graphql", stack.HTTPEndpoint()), "application/json", body) if err != nil { t.Fatalf("could not post: %v", err) } @@ -177,9 +177,9 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { func createNode(t *testing.T, gqlEnabled bool) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", - HTTPPort: 9393, + HTTPPort: 0, WSHost: "127.0.0.1", - WSPort: 9393, + WSPort: 0, }) if err != nil { t.Fatalf("could not create node: %v", err) @@ -187,11 +187,11 @@ func createNode(t *testing.T, gqlEnabled bool) *node.Node { if !gqlEnabled { return stack } - createGQLService(t, stack, "127.0.0.1:9393") + createGQLService(t, stack) return stack } -func createGQLService(t *testing.T, stack *node.Node, endpoint string) { +func createGQLService(t *testing.T, stack *node.Node) { // create backend ethConf := ð.Config{ Genesis: &core.Genesis{ From 12969084d17878ccb7978f24574bafacfe99f4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Jan 2021 12:10:52 +0200 Subject: [PATCH 068/709] cmd/faucet: update the embedded website asset --- cmd/faucet/website.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/faucet/website.go b/cmd/faucet/website.go index a091d24919..aed067893a 100644 --- a/cmd/faucet/website.go +++ b/cmd/faucet/website.go @@ -1,6 +1,6 @@ // Code generated by go-bindata. DO NOT EDIT. // sources: -// faucet.html (11.27kB) +// faucet.html (11.276kB) package main @@ -69,7 +69,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _faucetHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x5a\x6d\x93\xdb\x36\x92\xfe\x3c\xfe\x15\x1d\x9e\xbd\x92\xce\x43\x52\x33\x63\x7b\x7d\x12\xa9\x94\xd7\x9b\xdd\xf3\xd5\x5d\x92\x4a\x9c\xba\xdb\xca\xa6\xae\x40\xb2\x25\xc2\x03\x02\x0c\x00\x4a\xa3\x4c\xe9\xbf\x5f\x35\x40\x52\xd4\xcb\x4c\xec\xb5\xaf\x6a\xfd\x61\x4c\x02\x8d\x46\xa3\xfb\x69\xf4\x0b\x95\x7c\xf5\xe7\xef\xde\xbe\xff\xdb\xf7\xdf\x40\x69\x2b\xb1\x78\x92\xd0\x7f\x20\x98\x5c\xa5\x01\xca\x60\xf1\xe4\x22\x29\x91\x15\x8b\x27\x17\x17\x49\x85\x96\x41\x5e\x32\x6d\xd0\xa6\x41\x63\x97\xe1\xeb\x60\x3f\x51\x5a\x5b\x87\xf8\x6b\xc3\xd7\x69\xf0\x3f\xe1\x4f\x6f\xc2\xb7\xaa\xaa\x99\xe5\x99\xc0\x00\x72\x25\x2d\x4a\x9b\x06\xef\xbe\x49\xb1\x58\xe1\x60\x9d\x64\x15\xa6\xc1\x9a\xe3\xa6\x56\xda\x0e\x48\x37\xbc\xb0\x65\x5a\xe0\x9a\xe7\x18\xba\x97\x4b\xe0\x92\x5b\xce\x44\x68\x72\x26\x30\xbd\x0a\x16\x4f\x88\x8f\xe5\x56\xe0\xe2\xfe\x3e\xfa\x16\xed\x46\xe9\xdb\xdd\x6e\x06\x6f\x1a\x5b\xa2\xb4\x3c\x67\x16\x0b\xf8\x0b\x6b\x72\xb4\x49\xec\x29\xdd\x22\xc1\xe5\x2d\x94\x1a\x97\x69\x40\xa2\x9b\x59\x1c\xe7\x85\xfc\x60\xa2\x5c\xa8\xa6\x58\x0a\xa6\x31\xca\x55\x15\xb3\x0f\xec\x2e\x16\x3c\x33\xb1\xdd\x70\x6b\x51\x87\x99\x52\xd6\x58\xcd\xea\xf8\x26\xba\x89\xfe\x18\xe7\xc6\xc4\xfd\x58\x54\x71\x19\xe5\xc6\x04\xa0\x51\xa4\x81\xb1\x5b\x81\xa6\x44\xb4\x01\xc4\x8b\x7f\x6c\xdf\xa5\x92\x36\x64\x1b\x34\xaa\xc2\xf8\x45\xf4\xc7\x68\xea\xb6\x1c\x0e\x3f\xbe\x2b\x6d\x6b\x72\xcd\x6b\x0b\x46\xe7\x1f\xbd\xef\x87\x5f\x1b\xd4\xdb\xf8\x26\xba\x8a\xae\xda\x17\xb7\xcf\x07\x13\x2c\x92\xd8\x33\x5c\x7c\x16\xef\x50\x2a\xbb\x8d\xaf\xa3\x17\xd1\x55\x5c\xb3\xfc\x96\xad\xb0\xe8\x76\xa2\xa9\xa8\x1b\xfc\x62\xfb\x3e\x64\xc3\x0f\xc7\x26\xfc\x12\x9b\x55\xaa\x42\x69\xa3\x0f\x26\xbe\x8e\xae\x5e\x47\xd3\x6e\xe0\x94\xbf\xdb\x80\x8c\x46\x5b\x5d\x44\x6b\xd4\x84\x5c\x11\xe6\x28\x2d\x6a\xb8\xa7\xd1\x8b\x8a\xcb\xb0\x44\xbe\x2a\xed\x0c\xae\xa6\xd3\x67\xf3\x73\xa3\xeb\xd2\x0f\x17\xdc\xd4\x82\x6d\x67\xb0\x14\x78\xe7\x87\x98\xe0\x2b\x19\x72\x8b\x95\x99\x81\xe7\xec\x26\x76\x6e\xcf\x5a\xab\x95\x46\x63\xda\xcd\x6a\x65\xb8\xe5\x4a\xce\x08\x51\xcc\xf2\x35\x9e\xa3\x35\x35\x93\x27\x0b\x58\x66\x94\x68\x2c\x1e\x09\x92\x09\x95\xdf\xfa\x31\xe7\xcd\xc3\x43\xe4\x4a\x28\x3d\x83\x4d\xc9\xdb\x65\xe0\x36\x82\x5a\x63\xcb\x1e\x6a\x56\x14\x5c\xae\x66\xf0\xaa\x6e\xcf\x03\x15\xd3\x2b\x2e\x67\x30\xdd\x2f\x49\xe2\x4e\x8d\x49\xec\x2f\xae\x27\x17\x49\xa6\x8a\xad\xb3\x61\xc1\xd7\x90\x0b\x66\x4c\x1a\x1c\xa9\xd8\x5d\x48\x07\x04\x74\x0f\x31\x2e\xbb\xa9\x83\x39\xad\x36\x01\xb8\x8d\xd2\xc0\x0b\x11\x66\xca\x5a\x55\xcd\xe0\x8a\xc4\x6b\x97\x1c\xf1\x13\xa1\x58\x85\x57\xd7\xdd\xe4\x45\x52\x5e\x75\x4c\x2c\xde\xd9\xd0\xd9\xa7\xb7\x4c\xb0\x48\x78\xb7\x76\xc9\x60\xc9\xc2\x8c\xd9\x32\x00\xa6\x39\x0b\x4b\x5e\x14\x28\xd3\xc0\xea\x06\x09\x47\x7c\x01\xc3\xeb\xef\x81\xdb\xaf\xbc\xea\xe4\x8a\x0b\xbe\x6e\x8f\x35\x78\x3c\x3a\xe1\xc3\x87\x78\x0d\xed\x83\x5a\x2e\x0d\xda\x70\x70\xa6\x01\x31\x97\x75\x63\xc3\x95\x56\x4d\xdd\xcf\x5f\x24\x6e\x14\x78\x91\x06\x8d\x16\x41\x7b\xfd\xbb\x47\xbb\xad\x5b\x55\x04\xfd\xc1\x95\xae\x42\xb2\x84\x56\x22\x80\x5a\xb0\x1c\x4b\x25\x0a\xd4\x69\xf0\xa3\xca\x39\x13\x20\xfd\x99\xe1\xa7\x1f\xfe\x13\x5a\x93\x71\xb9\x82\xad\x6a\x34\x7c\x63\x4b\xd4\xd8\x54\xc0\x8a\x82\xe0\x1a\x45\xd1\x40\x10\x87\xdd\x53\x51\xc3\xcc\xca\x3d\xd5\x45\x92\x35\xd6\xaa\x9e\x30\xb3\x12\x32\x2b\xc3\x02\x97\xac\x11\x16\x0a\xad\xea\x42\x6d\x64\x68\xd5\x6a\x45\x91\xce\x1f\xc2\x2f\x0a\xa0\x60\x96\xb5\x53\x69\xd0\xd1\x76\x36\x64\xa6\x56\x75\x53\xb7\x56\xf4\x83\x78\x57\x33\x59\x60\x41\x36\x17\x06\x83\xc5\x5f\xf9\x1a\xa1\x42\x7f\x96\x8b\x63\x48\xe4\x4c\xa3\x0d\x87\x4c\x4f\x80\x91\xc4\x5e\x18\x7f\x24\x68\xff\x25\x8d\xe8\x38\xf5\x47\xa8\x50\x36\x70\xf0\x16\x6a\xba\x57\x82\xc5\xfd\xbd\x66\x72\x85\xf0\x94\x17\x77\x97\xf0\x94\x55\xaa\x91\x16\x66\x29\x44\x6f\xdc\xa3\xd9\xed\x0e\xb8\x03\x24\x82\x2f\x12\xf6\x18\xbc\x41\xc9\x5c\xf0\xfc\x36\x0d\x2c\x47\x9d\xde\xdf\x13\xf3\xdd\x6e\x0e\xf7\xf7\x7c\x09\x4f\xa3\x1f\x30\x67\xb5\xcd\x4b\xb6\xdb\xad\x74\xf7\x1c\xe1\x1d\xe6\x8d\xc5\xf1\xe4\xfe\x1e\x85\xc1\xdd\xce\x34\x59\xc5\xed\xb8\x5b\x4e\xe3\xb2\xd8\xed\x48\xe6\x56\xce\xdd\x0e\x62\x62\x2a\x0b\xbc\x83\xa7\xd1\xf7\xa8\xb9\x2a\x0c\x78\xfa\x24\x66\x8b\x24\x16\x7c\xd1\xae\x3b\x54\x52\xdc\x88\x3d\x5e\x62\x02\x4c\x8f\x73\xe7\x36\x4e\xd4\xa1\xa4\x67\xbc\x60\x15\xf6\xd2\xb7\x78\x30\xdc\xe2\x2d\x6e\xd3\xe0\xfe\x7e\xb8\xb6\x9d\xcd\x99\x10\x19\x23\xbd\xf8\xa3\xf5\x8b\x7e\x43\xc2\xe9\x9a\x1b\x97\x52\x2d\x3a\x09\xf6\x62\x7f\xa4\x5b\x1f\x5d\x5c\x56\xd5\x33\xb8\xb9\x1e\xdc\x5a\xe7\x3c\xfe\xd5\x91\xc7\xdf\x9c\x25\xae\x99\x44\x01\xee\x6f\x68\x2a\x26\xba\xe7\xd6\x5b\x06\xce\x77\xbc\x28\xa4\x3b\xba\x17\xad\xbf\xeb\xa7\x73\x50\x6b\xd4\x4b\xa1\x36\x33\x60\x8d\x55\x73\xa8\xd8\x5d\x1f\xef\x6e\xa6\xd3\xa1\xdc\x94\x0a\xb2\x4c\xa0\xbb\x5d\x34\xfe\xda\xa0\xb1\xa6\xbf\x4b\xfc\x94\xfb\x4b\x57\x4a\x81\xd2\x60\x71\xa4\x0d\xda\x91\x54\xeb\xa8\x06\xa6\xef\x95\x79\x56\xf6\xa5\x52\x7d\x08\x19\x8a\xd1\xb2\x1e\x44\xbb\x60\x91\x58\xbd\xa7\xbb\x48\x6c\xf1\x49\x21\x40\x53\x8a\xf7\x50\x04\xf0\x37\x1a\x9d\xbd\x46\xd4\x3e\xbf\x20\xc8\x82\x7b\x4d\x62\x5b\x7c\xc6\xce\x04\xc2\x8c\x19\xfc\x98\xed\x5d\xa4\xdf\x6f\xef\x5e\x3f\x77\xff\x12\x99\xb6\x19\x32\xfb\x31\x02\x2c\x1b\x59\x0c\xce\xef\xee\xce\xcf\x15\xa0\x91\x7c\x8d\xda\x70\xbb\xfd\x58\x09\xb0\xd8\x8b\xe0\xdf\x0f\x45\x48\x62\xab\x1f\xc7\xda\xf0\xe5\x0b\x39\xf7\xef\xa5\x24\x37\x8b\x7f\x57\x1b\x28\x14\x1a\xb0\x25\x37\x40\xc1\xf5\xeb\x24\x2e\x6f\x7a\x92\x7a\xf1\x9e\x26\x9c\x52\x61\xe9\x52\x0b\xe0\x06\x74\x23\x5d\xe4\x55\x12\x6c\x89\x87\xe9\x48\x1b\xa4\x23\x78\xaf\x28\xa5\x5b\xa3\xb4\x50\x31\xc1\x73\xae\x1a\x03\x2c\xb7\x4a\x1b\x58\x6a\x55\x01\xde\x95\xac\x31\x96\x18\xd1\xf5\xc1\xd6\x8c\x0b\xe7\x4b\xce\xa4\xa0\x34\xb0\x3c\x6f\xaa\x86\x52\x52\xb9\x02\x94\xaa\x59\x95\xad\x2c\x56\x81\x0f\x4c\x42\xc9\x55\x2f\x8f\xa9\x59\x05\xcc\x5a\x96\xdf\x9a\x4b\xe8\x6e\x05\x60\x1a\xc1\x72\x2c\x68\x55\xae\xaa\x4a\x49\xb8\xd1\x05\xd4\x4c\xdb\x2d\x98\xc3\xdc\x82\xe5\xb9\x8b\x72\x11\xbc\x91\x5b\x25\x11\x4a\xb6\x76\x12\xc2\x7b\x5f\x4e\x90\x5c\x7f\x61\x39\x66\x4a\xf5\xd4\x50\xb1\x6d\xb7\x5d\x2b\xfd\x86\xdb\x92\x7b\xf5\xd4\xa8\x2b\x5a\x5a\x80\xe0\x15\xb7\x26\x4a\xe2\x7a\x7f\xa3\xee\x63\xb3\x08\x4b\xa5\xf9\x6f\x94\xd8\x88\xe1\xf5\x69\x8f\x2e\x97\xee\x6e\x74\x56\x17\xb8\xb4\x33\x78\xe1\xef\xc6\x63\x1c\xb7\x15\xd0\x39\x10\x77\x3c\x5d\x65\x49\x01\x67\x06\x37\x3e\x9d\xf5\x89\x44\x61\x07\x12\x14\x47\x50\xf3\x9b\xbe\x7e\x5d\xdf\xf5\x72\xf4\x39\xf1\xb4\x67\x42\x08\x38\x54\xca\x9a\xf7\x6a\xbc\x84\x8a\xdd\x22\x30\x48\xd8\x51\x85\xdc\x0a\xed\xea\x2b\xee\xfa\x03\xb1\xdd\x20\xda\xaf\xc9\x75\xd3\x1f\x3c\x43\x2e\x57\xcf\xae\xa7\x1e\x91\xf4\x40\xec\x9f\x5d\x4f\xb9\xb4\xea\xd9\xf5\x74\x7a\x37\xfd\xc8\x7f\xcf\xae\xa7\x4a\x3e\xbb\x9e\xda\x12\x9f\x5d\x4f\x9f\x5d\xdf\x0c\xb1\xec\x47\xba\xcc\x92\xa8\xd0\xd0\x6e\x1d\xc4\x03\xb0\x4c\xaf\xd0\xa6\xc1\xff\xb2\x4c\x35\x76\x96\x09\x26\x6f\x83\x85\x13\x97\xb2\x0d\x87\x82\xf3\xf9\x29\xd4\xcc\x10\x24\x48\x62\x87\x92\xb6\x17\x62\x60\x6c\x1a\xad\x55\x23\x29\x2a\x02\x9d\xd9\x79\xa8\x1c\x11\xca\x48\x31\x93\x28\xc9\x74\xbc\x78\xab\xea\x6d\xe8\x98\xb8\xe5\x27\x6a\x34\x4d\x5d\x2b\x6d\xa3\xa1\x3a\x19\xd5\x41\x02\x4d\xfc\x7a\xfa\xf2\xf5\xab\x47\xc5\x37\x94\x65\xbb\x33\xf4\x12\xb2\x4c\xad\x11\x7c\x4e\x9f\xa9\x3b\x60\xb2\x80\x25\xd7\x08\x6c\xc3\xb6\x5f\x25\x71\xe1\x2a\xb0\xcf\x47\xed\xb2\xf5\xae\x7f\x2a\xd8\x76\x2e\x7f\x09\x75\x93\x09\x6e\x4a\x60\x20\x71\x03\x89\xb1\x5a\xc9\xd5\xc2\x8d\xe6\x54\x92\xba\x57\xa8\x95\xb1\x8f\x99\x1f\xab\x0c\x8b\xe2\x0c\x00\xbe\x94\xfd\x37\x9b\x4d\xd4\x69\xd2\x19\xbf\x44\x51\xc7\x74\xfd\x35\x92\xdb\x6d\xec\xdd\x48\xc9\xf8\x6b\x5e\xa4\xd7\xaf\xaf\x5f\xbd\xba\x7e\xf1\x6f\xaf\x5f\xbe\xbc\x7e\xfd\xe2\xe5\x43\xc8\xa0\x43\x7d\x26\x30\x7c\x1a\xfd\xad\xa2\xaa\xb5\xcf\xa1\x3d\x5e\xba\xdc\x8d\x22\x74\x41\x35\x88\x0e\xfe\x61\x0c\x35\x92\x12\x91\x90\x89\xb3\x39\xc4\x27\xa0\xc8\xc1\xe8\x11\xc9\x3e\x13\x5a\x1d\x7c\x08\x29\xaa\xb1\x74\xc2\xae\x98\xe7\x4a\xf6\x70\xba\x04\xc3\xab\x5a\x6c\x21\xdf\x5b\xfd\x3c\xae\x1e\x34\xca\xef\xc2\xea\xd0\x6c\x1e\x64\x2e\xfa\x57\xaa\x40\x8a\xfa\xa6\x31\x39\xd6\xae\xcb\x4b\x91\xf4\x4f\xdb\xdf\x98\xb4\x5c\x62\x17\x71\x23\xf8\x4e\x8a\x2d\x34\x06\x61\xa9\x34\x14\x98\x35\xab\x95\x4b\x13\x34\xd4\x9a\xaf\x99\xc5\x2e\xcc\x9a\x16\x15\x3d\x28\x06\x95\x0d\xa5\x3c\x62\x90\x81\xfc\x4d\x35\x90\x33\x09\x56\xb3\xfc\xd6\x7b\x4a\xa3\x35\x79\x4a\x8d\xfe\x34\x7d\xa0\xcf\x50\xa8\x8d\x23\xf1\xe7\x5e\x72\x14\x2e\xea\x1b\x44\x28\xd5\x06\xaa\x26\x77\x0e\x49\x51\xdd\x1d\x62\xc3\xb8\x85\x46\x5a\x2e\xbc\x3e\x6d\xa3\x25\xe5\x08\x78\x10\xa5\x4f\x6a\xbf\x04\xab\xc5\xfb\x12\xcf\xa4\x44\x7d\xd5\x06\x1a\xdf\x7a\x72\xa8\xb5\xb2\x98\x93\x41\x81\xad\x18\x97\x86\x2c\xe2\xf2\x00\xac\x3e\xa2\xaa\xeb\x9f\xda\x87\x7d\x87\xd2\x4d\xc7\x31\xfc\x55\xa8\x8c\x09\x58\x13\xd2\x33\x41\xe9\x9c\x82\x52\xd1\xd1\x07\xda\x32\x96\xd9\xc6\x80\x5a\xba\x51\x2f\x39\xad\x5f\x33\x4d\x16\xc4\xaa\xb6\x90\xb6\xfd\x35\x1a\x33\xa8\xd7\x6d\xd7\x90\x5e\xa9\x72\x3f\x98\xef\xb5\x9e\xc2\xcf\xbf\xcc\x9f\xb4\xa2\xfc\x19\x97\x0e\x12\x84\x6f\x7f\x64\x5b\x32\x0b\xb9\x46\x66\xd1\x40\x2e\x94\x69\xb4\x97\xb0\xd0\xaa\x06\x92\xb2\xe3\xd4\x71\xa6\x89\xda\xed\xd6\x31\x19\x97\xcc\x94\x93\xb6\x3d\xa8\xd1\x59\xa9\x9f\xeb\xc6\x2f\x08\x75\x63\x62\xc0\xd3\xe9\x1c\x78\xd2\xf1\x8d\x04\xca\x95\x2d\xe7\xc0\x9f\x3f\xef\x89\x2f\xf8\x12\xc6\x1d\xc5\xcf\xfc\x97\xc8\xde\x45\xb4\x0b\xa4\x29\x0c\x77\x73\x1b\xb6\x7c\x4c\x2d\x78\x8e\x63\x7e\x09\x57\x93\x79\x37\x9b\x69\x64\xb7\xdd\x5b\x6b\x47\xff\x9f\xfb\xbb\x9b\x1f\x6a\xc6\x29\xff\x40\x37\xbe\xf6\x37\xc0\x60\xc5\x8d\x85\x46\x0b\x68\x7d\xd8\x9b\xa0\x37\x88\xa3\x1b\x6a\xe5\x04\x97\xed\x43\x8b\xa9\xee\x08\x9e\x4d\x64\x50\x16\xe3\xff\xf8\xf1\xbb\x6f\x23\x63\x35\x97\x2b\xbe\xdc\x8e\xef\x1b\x2d\x66\xf0\x74\x1c\xfc\x4b\xa3\x45\x30\xf9\x79\xfa\x4b\xb4\x66\xa2\xc1\x4b\x67\xef\x99\xfb\x7b\xb2\xcb\x25\xb4\x8f\x33\x38\xdc\x70\x37\x99\xcc\xcf\xf7\x49\x06\x6d\x1d\x8d\x06\xed\x98\x08\x7b\xe0\x1f\xeb\x88\x41\x85\xb6\x54\xce\x75\x35\xe6\x4a\x4a\xcc\x2d\x34\xb5\x92\xad\x4a\x40\x28\x63\xf6\x40\xec\x28\xd2\x53\x50\xb4\xf4\xa9\x0b\xd6\xff\x8d\xd9\x8f\x2a\xbf\x45\x3b\x1e\x8f\x37\x5c\x16\x6a\x13\x09\xe5\xaf\xda\x88\x9c\x54\xe5\x4a\x40\x9a\xa6\xd0\x46\xd1\x60\x02\x5f\x43\xb0\x31\x14\x4f\x03\x98\xd1\x23\x3d\x4d\xe0\x39\x1c\x2f\x2f\x29\xde\x3f\x87\x20\x66\x35\x0f\x26\xde\x1d\x3a\xc5\x2b\x59\xa1\x31\x6c\x85\x43\x01\x5d\x65\xd4\x83\x8c\xce\x51\x99\x15\xa4\xe0\x0c\x54\x33\x6d\xd0\x93\x44\x54\x8d\x77\x68\x23\xcc\x3a\xb2\x34\x05\xd9\x08\xb1\x07\xa9\x77\x8a\x79\x07\xbf\x03\xf2\xc8\xc7\x9a\xaf\xd2\x14\xa8\x34\x25\x15\x17\xfb\x95\x64\x7c\x5f\x44\x4f\x22\x8a\x0b\xfb\x15\x93\xf9\x10\xcd\x07\xdc\xb0\xf8\x3d\x76\x58\x1c\xf3\xc3\xe2\x01\x86\xae\x67\xf1\x18\x3f\xdf\xe3\x18\xb0\x73\x03\x0f\x70\x93\x4d\x95\xa1\x7e\x8c\x9d\xef\x59\xb4\xec\x9c\xaa\xdf\x49\x3b\x58\x7b\x09\x57\xaf\x26\x0f\x70\x47\xad\xd5\x83\xcc\xa5\xb2\xdb\xf1\xbd\x60\x5b\xca\x99\x60\x64\x55\xfd\xd6\xb5\x18\x46\x97\x2e\xe2\xce\xa0\xe7\x70\xe9\x9a\xc7\x33\x18\xb9\x37\x9a\xe7\x15\xba\x55\x2f\xa7\xd3\xe9\x25\x74\x5f\x5d\xfe\xc4\xc8\x09\x75\x83\xbb\x07\xe4\x31\x4d\x9e\x53\xdc\xff\x1c\x89\x5a\x1e\xbd\x4c\xed\xfb\x67\x48\xd5\xc7\x86\x03\xb1\xe0\x0f\x7f\x80\x93\xd9\x43\x18\xc7\x31\xfc\x17\xa3\x32\x5c\x08\xd7\x3d\x70\x4d\x83\x9e\xbe\xe2\xc6\xb8\x62\xdc\x40\xa1\x24\xb6\x6b\x3e\xed\xda\x3f\x91\xb1\x25\x83\x05\x4c\x8f\x05\xa4\xeb\x70\x10\x16\xce\x44\x8b\x01\xdf\xc3\x40\x70\xb1\x1b\xee\x77\xb0\x92\x57\x08\x5f\xa5\x10\x04\xc3\xc5\x27\x14\x44\xd0\x33\xbb\x30\x68\xdf\x7b\x5b\x8c\xdb\xe8\x78\x2e\x76\x4d\x2e\xe1\x66\x3a\x9d\x4e\x4e\x84\xd8\xed\xd5\xfb\xa6\xa6\xb4\x09\x98\xdc\xba\x2b\xb1\xd7\xad\x4b\x1c\x29\x05\xa2\x2b\x4d\x40\xae\x84\xf0\x39\x4b\xbb\x94\x14\xdc\x36\x4f\x52\x08\xaf\xe6\x67\xa2\xe8\x40\x93\x83\xa3\x1d\x9b\xe7\x8c\xee\x8f\x4d\x74\xa8\xb3\x23\xe2\xf0\xea\xc0\x28\x07\xf6\x3a\x6f\x98\x8b\x5e\x6e\xbe\xd7\xe8\x91\xb9\xf6\xf6\x3a\xd6\xd9\x40\x7e\xcf\xe7\xf9\xd5\x47\x1e\xa3\x9f\xae\x1b\x53\x8e\x8f\x04\x9d\xcc\x4f\x6d\xf3\xce\xa2\xa6\x2c\x59\x51\xc8\x22\x5b\x50\x29\xa0\xf1\xc4\x24\x2e\x55\xd7\x18\x6a\x94\x05\xea\x2e\xa5\xf0\x99\x3d\x25\x80\x07\x26\xf3\x55\xe5\x10\x4e\x9f\xe8\x30\x2e\x25\x53\x12\x01\x00\x8e\x9c\xc0\x01\xf5\x00\xa9\x44\x8c\x82\xd5\x06\x0b\x48\xc1\x7f\x04\x1f\x4f\xa2\x46\xf2\xbb\xf1\x24\x6c\xdf\x8f\x79\x74\xf3\xf3\xbe\x4c\xec\xc4\x7e\x9e\x42\x90\x58\x0d\xbc\x48\x47\x01\x3c\x3f\xe7\x82\x14\x75\x47\x8b\xbd\x04\xc3\xa5\x00\x89\x2d\x16\xae\x0f\xea\xeb\xb5\xbf\x07\x19\xcb\x6f\x57\xae\x10\x9a\x51\xaa\x35\x3e\x61\xcb\xd6\xcc\x32\xed\xb8\x4e\xe6\xb0\x27\x6f\x0b\xc5\x9c\x8c\x33\x07\x5f\x91\xba\x76\x2b\xf4\x9f\x28\xdc\x5b\xa6\x74\x81\x3a\xd4\xac\xe0\x8d\x99\xc1\x8b\xfa\x6e\xfe\xf7\xee\x13\x8e\x6b\x0a\x3f\x2a\x6a\xad\x71\x71\x22\x51\xdb\x65\x7c\x0e\x41\x12\x13\xc1\xef\xb1\xe9\x0f\x3b\xfc\xf8\x0e\x67\x5a\xdf\xd0\x7f\x1a\x6f\xc7\x2b\x5e\x14\x02\x49\xe0\x3d\x7b\x72\x46\xb2\xff\xd0\xa5\x0e\xb7\x84\xb6\xe7\xbd\x5f\xb3\x03\x14\x06\x1f\x59\xd0\xb7\xcf\x47\x04\x80\x90\x8e\xcc\x9d\xce\xdb\x62\xdb\x0d\xeb\x91\xd3\x45\xfb\x53\x8a\xa2\xd1\x2e\xd7\x1a\x87\x2d\xc0\x2e\x61\x64\x28\xf7\x2b\xcc\x68\x12\x95\x4d\xc5\x24\xff\x0d\xc7\x14\x97\x26\x5e\x57\xae\x1f\x1f\x9c\x5e\xc9\x27\xc2\xec\x1b\xe5\xa3\x2e\xc6\x8d\x5a\x25\x8e\x3a\xeb\xbe\xd8\xd7\xf6\x33\x98\xce\x47\x9f\xa8\xa1\xf3\xbb\x84\x19\xd3\x30\x7c\x09\xbb\xe0\x0b\x5a\xd1\xee\xdd\x5c\xc6\xf4\xc8\x77\x32\x5c\x7e\x2e\xd5\x26\x1d\xdd\x4c\x7b\x21\xbd\xa1\x9d\x9d\x47\x2d\xd6\x4e\x8c\x41\x52\x76\xae\xb9\x80\x9b\xe9\x97\x90\xd6\x77\x43\x8e\x4e\x60\x35\xaf\xb1\x00\x96\x5b\xbe\xc6\xff\x87\x83\x7c\x01\x25\x7f\xb2\x88\x84\xc3\x4e\x79\x0e\xa6\x07\xf2\xd2\x6c\xaf\xdb\x7f\x25\x7f\x83\xd8\x69\xf8\x39\x04\x67\x0f\xf2\x20\x12\x8f\x08\x8f\x5c\xfb\x61\xbf\x77\x1f\x98\x82\xe3\x98\x42\xd9\x6e\xff\x71\x74\x12\x95\xb6\x12\xe3\x20\xb1\xee\x47\x32\x24\x73\xcf\xc1\x31\xf0\xc3\x87\x29\xdd\xee\xb0\x90\xa1\xfa\x1d\x8f\xea\x2c\x18\x24\x27\x7d\x2d\xd6\x65\x22\xb0\xdb\xff\x96\x28\x8e\xe1\x47\xcb\xb4\x05\x06\x3f\xbd\x83\xa6\x2e\x98\xf5\x9f\x72\x28\x3e\xfa\x4f\x25\xdd\x8f\x8d\x32\xa6\x0d\x2c\x95\xde\x30\x5d\xb4\xfd\x19\x5b\xe2\xd6\x7d\xca\xe9\x52\x3f\x83\xf6\x1d\xdd\x62\x6b\x26\xc6\x27\x75\xdf\xd3\xf1\x28\x1a\x9a\x7c\x34\x89\x90\xe5\xe5\x29\xa1\x8b\x58\xfd\xbe\x29\x7c\xeb\x4a\x80\xf1\xd3\xb1\x2d\xb9\x99\x44\xcc\x5a\x3d\x1e\x1d\x80\x61\x34\x21\xbb\x5e\x0d\x4a\xb2\x7e\x79\x72\xe0\x56\x8f\xf1\xd8\x27\xd3\x7d\x22\xd0\x91\xe7\xc6\x8c\x3d\xae\x46\x97\x03\xde\x87\xb0\x1a\x3d\x1b\xf5\x86\xda\xbb\xf7\xfe\x1c\xe9\x59\x49\x0e\x58\x8f\xc8\xcb\x46\x27\xdb\xb3\xa2\x78\x4b\xfe\x33\x0e\xce\x78\xfa\x31\x3a\x26\xbd\xb2\xfd\x7d\xfd\xa8\x96\xfd\xcf\x32\x1e\x50\x31\x2f\x46\x93\xc8\x34\x99\xef\x4d\x8c\x5f\xf6\x05\x58\x47\xe6\xc0\x7b\x1c\x0a\x4e\x12\x0a\xda\xe2\x30\xa9\x08\x8f\x92\x90\x47\xa2\x46\xbb\xa5\x3f\xd5\xee\x92\x14\x3e\x9d\xf4\xad\xad\x6f\x0c\x25\x57\xbe\xf5\xbf\xc1\xcc\xb8\x4e\x02\xb4\x78\x77\xdd\x1c\xdf\xb5\x79\xf3\xfd\xbb\x41\xe7\xa6\xf7\x88\xb1\xe3\xde\xff\x0e\xf0\x5c\x9f\xe4\xec\x0f\x0f\x37\x9b\x4d\xb4\x52\x6a\x25\xfc\x4f\x0e\xfb\x46\x4a\xcc\x6a\x1e\x7d\x30\x01\x30\xb3\x95\x39\x14\xb8\x44\xbd\x18\xb0\x6f\xbb\x2b\x49\xec\x7f\x12\x97\xc4\xfe\x57\xbf\xff\x17\x00\x00\xff\xff\x31\x9f\x54\x5e\x06\x2c\x00\x00") +var _faucetHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x5a\x7b\x93\xdb\x36\x92\xff\x7b\xfc\x29\x3a\x3c\x7b\x25\x9d\x87\xa4\x66\xc6\xf6\xfa\x24\x52\x29\xaf\x37\xbb\xe7\xab\xbb\x24\x95\x38\x75\xb7\x95\x4d\x5d\x81\x64\x4b\x84\x07\x04\x18\x00\x94\x46\x99\xd2\x77\xbf\x6a\x80\xa4\xa8\xc7\x4c\xec\xb5\xaf\x6a\xfd\xc7\x98\xc4\xa3\xd1\x8f\x5f\xa3\x1f\x54\xf2\xd5\x9f\xbf\x7b\xfb\xfe\x6f\xdf\x7f\x03\xa5\xad\xc4\xe2\x49\x42\xff\x81\x60\x72\x95\x06\x28\x83\xc5\x93\x8b\xa4\x44\x56\x2c\x9e\x5c\x5c\x24\x15\x5a\x06\x79\xc9\xb4\x41\x9b\x06\x8d\x5d\x86\xaf\x83\xfd\x44\x69\x6d\x1d\xe2\xaf\x0d\x5f\xa7\xc1\xff\x84\x3f\xbd\x09\xdf\xaa\xaa\x66\x96\x67\x02\x03\xc8\x95\xb4\x28\x6d\x1a\xbc\xfb\x26\xc5\x62\x85\x83\x7d\x92\x55\x98\x06\x6b\x8e\x9b\x5a\x69\x3b\x58\xba\xe1\x85\x2d\xd3\x02\xd7\x3c\xc7\xd0\xbd\x5c\x02\x97\xdc\x72\x26\x42\x93\x33\x81\xe9\x55\xb0\x78\x42\x74\x2c\xb7\x02\x17\xf7\xf7\xd1\xb7\x68\x37\x4a\xdf\xee\x76\x33\x78\xd3\xd8\x12\xa5\xe5\x39\xb3\x58\xc0\x5f\x58\x93\xa3\x4d\x62\xbf\xd2\x6d\x12\x5c\xde\x42\xa9\x71\x99\x06\xc4\xba\x99\xc5\x71\x5e\xc8\x0f\x26\xca\x85\x6a\x8a\xa5\x60\x1a\xa3\x5c\x55\x31\xfb\xc0\xee\x62\xc1\x33\x13\xdb\x0d\xb7\x16\x75\x98\x29\x65\x8d\xd5\xac\x8e\x6f\xa2\x9b\xe8\x8f\x71\x6e\x4c\xdc\x8f\x45\x15\x97\x51\x6e\x4c\x00\x1a\x45\x1a\x18\xbb\x15\x68\x4a\x44\x1b\x40\xbc\xf8\xc7\xce\x5d\x2a\x69\x43\xb6\x41\xa3\x2a\x8c\x5f\x44\x7f\x8c\xa6\xee\xc8\xe1\xf0\xe3\xa7\xd2\xb1\x26\xd7\xbc\xb6\x60\x74\xfe\xd1\xe7\x7e\xf8\xb5\x41\xbd\x8d\x6f\xa2\xab\xe8\xaa\x7d\x71\xe7\x7c\x30\xc1\x22\x89\x3d\xc1\xc5\x67\xd1\x0e\xa5\xb2\xdb\xf8\x3a\x7a\x11\x5d\xc5\x35\xcb\x6f\xd9\x0a\x8b\xee\x24\x9a\x8a\xba\xc1\x2f\x76\xee\x43\x36\xfc\x70\x6c\xc2\x2f\x71\x58\xa5\x2a\x94\x36\xfa\x60\xe2\xeb\xe8\xea\x75\x34\xed\x06\x4e\xe9\xbb\x03\xc8\x68\x74\xd4\x45\xb4\x46\x4d\xc8\x15\x61\x8e\xd2\xa2\x86\x7b\x1a\xbd\xa8\xb8\x0c\x4b\xe4\xab\xd2\xce\xe0\x6a\x3a\x7d\x36\x3f\x37\xba\x2e\xfd\x70\xc1\x4d\x2d\xd8\x76\x06\x4b\x81\x77\x7e\x88\x09\xbe\x92\x21\xb7\x58\x99\x19\x78\xca\x6e\x62\xe7\xce\xac\xb5\x5a\x69\x34\xa6\x3d\xac\x56\x86\x5b\xae\xe4\x8c\x10\xc5\x2c\x5f\xe3\xb9\xb5\xa6\x66\xf2\x64\x03\xcb\x8c\x12\x8d\xc5\x23\x46\x32\xa1\xf2\x5b\x3f\xe6\xbc\x79\x28\x44\xae\x84\xd2\x33\xd8\x94\xbc\xdd\x06\xee\x20\xa8\x35\xb6\xe4\xa1\x66\x45\xc1\xe5\x6a\x06\xaf\xea\x56\x1e\xa8\x98\x5e\x71\x39\x83\xe9\x7e\x4b\x12\x77\x6a\x4c\x62\x7f\x71\x3d\xb9\x48\x32\x55\x6c\x9d\x0d\x0b\xbe\x86\x5c\x30\x63\xd2\xe0\x48\xc5\xee\x42\x3a\x58\x40\xf7\x10\xe3\xb2\x9b\x3a\x98\xd3\x6a\x13\x80\x3b\x28\x0d\x3c\x13\x61\xa6\xac\x55\xd5\x0c\xae\x88\xbd\x76\xcb\x11\x3d\x11\x8a\x55\x78\x75\xdd\x4d\x5e\x24\xe5\x55\x47\xc4\xe2\x9d\x0d\x9d\x7d\x7a\xcb\x04\x8b\x84\x77\x7b\x97\x0c\x96\x2c\xcc\x98\x2d\x03\x60\x9a\xb3\xb0\xe4\x45\x81\x32\x0d\xac\x6e\x90\x70\xc4\x17\x30\xbc\xfe\x1e\xb8\xfd\xca\xab\x8e\xaf\xb8\xe0\xeb\x56\xac\xc1\xe3\x91\x84\x0f\x0b\xf1\x1a\xda\x07\xb5\x5c\x1a\xb4\xe1\x40\xa6\xc1\x62\x2e\xeb\xc6\x86\x2b\xad\x9a\xba\x9f\xbf\x48\xdc\x28\xf0\x22\x0d\x1a\x2d\x82\xf6\xfa\x77\x8f\x76\x5b\xb7\xaa\x08\x7a\xc1\x95\xae\x42\xb2\x84\x56\x22\x80\x5a\xb0\x1c\x4b\x25\x0a\xd4\x69\xf0\xa3\xca\x39\x13\x20\xbd\xcc\xf0\xd3\x0f\xff\x09\xad\xc9\xb8\x5c\xc1\x56\x35\x1a\xbe\xb1\x25\x6a\x6c\x2a\x60\x45\x41\x70\x8d\xa2\x28\x88\xf7\x9c\x38\xf0\x9e\xf2\x1a\x66\x56\xee\xf9\xbd\x48\xb2\xc6\x5a\xd5\x2f\xcc\xac\x84\xcc\xca\xb0\xc0\x25\x6b\x84\x85\x42\xab\xba\x50\x1b\x19\x5a\xb5\x5a\x51\xa8\xf3\x52\xf8\x4d\x01\x14\xcc\xb2\x76\x2a\x0d\xba\xb5\x9d\x11\x99\xa9\x55\xdd\xd4\xad\x19\xfd\x20\xde\xd5\x4c\x16\x58\x90\xd1\x85\xc1\x60\xf1\x57\xbe\x46\xa8\xd0\x0b\x73\x71\x8c\x89\x9c\x69\xb4\xe1\x90\xe8\x09\x32\x92\xd8\x33\xe3\x45\x82\xf6\x5f\xd2\x88\x8e\x52\x2f\x42\x85\xb2\x81\x83\xb7\x50\xd3\xc5\x12\x2c\xee\xef\x35\x93\x2b\x84\xa7\xbc\xb8\xbb\x84\xa7\xac\x52\x8d\xb4\x30\x4b\x21\x7a\xe3\x1e\xcd\x6e\x77\x40\x1d\x20\x11\x7c\x91\xb0\xc7\xf0\x0d\x4a\xe6\x82\xe7\xb7\x69\x60\x39\xea\xf4\xfe\x9e\x88\xef\x76\x73\xb8\xbf\xe7\x4b\x78\x1a\xfd\x80\x39\xab\x6d\x5e\xb2\xdd\x6e\xa5\xbb\xe7\x08\xef\x30\x6f\x2c\x8e\x27\xf7\xf7\x28\x0c\xee\x76\xa6\xc9\x2a\x6e\xc7\xdd\x76\x1a\x97\xc5\x6e\x47\x3c\xb7\x7c\xee\x76\x10\x13\x51\x59\xe0\x1d\x3c\x8d\xbe\x47\xcd\x55\x61\xc0\xaf\x4f\x62\xb6\x48\x62\xc1\x17\xed\xbe\x43\x25\xc5\x8d\xd8\xe3\x25\x26\xc0\xf4\x40\x77\x7e\xe3\x58\x1d\x72\x7a\xc6\x0d\x56\x61\xcf\x7d\x8b\x07\xc3\x2d\xde\xe2\x36\x0d\xee\xef\x87\x7b\xdb\xd9\x9c\x09\x91\x31\xd2\x8b\x17\xad\xdf\xf4\x1b\x12\x4e\xd7\xdc\xb8\x9c\x6a\xd1\x71\xb0\x67\xfb\x23\xfd\xfa\xe8\xe6\xb2\xaa\x9e\xc1\xcd\xf5\xe0\xda\x3a\xe7\xf2\xaf\x8e\x5c\xfe\xe6\xec\xe2\x9a\x49\x14\xe0\xfe\x86\xa6\x62\xa2\x7b\x6e\xbd\x65\x70\x0d\x1c\x6f\x0a\xe9\x92\xee\x59\xeb\x2f\xfb\xe9\x1c\xd4\x1a\xf5\x52\xa8\xcd\x0c\x58\x63\xd5\x1c\x2a\x76\xd7\x07\xbc\x9b\xe9\x74\xc8\x37\xe5\x82\x2c\x13\xe8\xae\x17\x8d\xbf\x36\x68\xac\xe9\x2f\x13\x3f\xe5\xfe\xd2\x9d\x52\xa0\x34\x58\x1c\x69\x83\x4e\x24\xd5\xba\x55\x03\xd3\xf7\xca\x3c\xcb\xfb\x52\xa9\x3e\x86\x0c\xd9\x68\x49\x0f\xc2\x5d\xb0\x48\xac\xde\xaf\xbb\x48\x6c\xf1\x49\x31\x40\x53\x8e\xf7\x50\x08\xf0\x37\x1a\xc9\x5e\x23\x6a\x9f\x60\x10\x64\xc1\xbd\x26\xb1\x2d\x3e\xe3\x64\x02\x61\xc6\x0c\x7e\xcc\xf1\x2e\xd4\xef\x8f\x77\xaf\x9f\x7b\x7e\x89\x4c\xdb\x0c\x99\xfd\x18\x06\x96\x8d\x2c\x06\xf2\xbb\xbb\xf3\x73\x19\x68\x24\x5f\xa3\x36\xdc\x6e\x3f\x96\x03\x2c\xf6\x2c\xf8\xf7\x43\x16\x92\xd8\xea\xc7\xb1\x36\x7c\xf9\x42\xce\xfd\x7b\x39\xc9\xcd\xe2\xdf\xd5\x06\x0a\x85\x06\x6c\xc9\x0d\x50\x74\xfd\x3a\x89\xcb\x9b\x7e\x49\xbd\x78\x4f\x13\x4e\xa9\xb0\x74\xb9\x05\x70\x03\xba\x91\x2e\xf4\x2a\x09\xb6\xc4\xc3\x7c\xa4\x8d\xd2\x11\xbc\x57\x94\xd3\xad\x51\x5a\xa8\x98\xe0\x39\x57\x8d\x01\x96\x5b\xa5\x0d\x2c\xb5\xaa\x00\xef\x4a\xd6\x18\x4b\x84\xe8\xfa\x60\x6b\xc6\x85\xf3\x25\x67\x52\x50\x1a\x58\x9e\x37\x55\x43\x39\xa9\x5c\x01\x4a\xd5\xac\xca\x96\x17\xab\xc0\x07\x26\xa1\xe4\xaa\xe7\xc7\xd4\xac\x02\x66\x2d\xcb\x6f\xcd\x25\x74\xb7\x02\x30\x8d\x60\x39\x16\xb4\x2b\x57\x55\xa5\x24\xdc\xe8\x02\x6a\xa6\xed\x16\xcc\x61\x72\xc1\xf2\xdc\x45\xb9\x08\xde\xc8\xad\x92\x08\x25\x5b\x3b\x0e\xe1\xbd\xaf\x27\x88\xaf\xbf\xb0\x1c\x33\xa5\xfa\xd5\x50\xb1\x6d\x77\x5c\xcb\xfd\x86\xdb\x92\x7b\xf5\xd4\xa8\x2b\xda\x5a\x80\xe0\x15\xb7\x26\x4a\xe2\x7a\x7f\xa3\xee\x63\xb3\x08\x4b\xa5\xf9\x6f\x94\xd9\x88\xe1\xf5\x69\x8f\x2e\x97\xee\x6e\x74\x56\x17\xb8\xb4\x33\x78\xe1\xef\xc6\x63\x1c\xb7\x25\xd0\x39\x10\x77\x34\x5d\x69\x49\x01\x67\x06\x37\x3e\x9f\xf5\x89\x44\x61\x07\x1c\x14\x47\x50\xf3\x87\xbe\x7e\x5d\xdf\xf5\x7c\xf4\x49\xf1\xb4\x27\x42\x08\x38\x54\xca\x9a\xf7\x6a\xbc\x84\x8a\xdd\x22\x30\x48\xd8\x51\x89\xdc\x32\xed\x0a\x2c\xee\x1a\x04\xb1\xdd\x20\xda\xaf\xc9\x75\xd3\x1f\x3c\x41\x2e\x57\xcf\xae\xa7\x1e\x91\xf4\x40\xe4\x9f\x5d\x4f\xb9\xb4\xea\xd9\xf5\x74\x7a\x37\xfd\xc8\x7f\xcf\xae\xa7\x4a\x3e\xbb\x9e\xda\x12\x9f\x5d\x4f\x9f\x5d\xdf\x0c\xb1\xec\x47\xba\xd4\x92\x56\xa1\xa1\xd3\x3a\x88\x07\x60\x99\x5e\xa1\x4d\x83\xff\x65\x99\x6a\xec\x2c\x13\x4c\xde\x06\x0b\xc7\x2e\x65\x1b\x0e\x05\xe7\x13\x54\xa8\x99\x21\x48\x10\xc7\x0e\x25\x6d\x33\xc4\xc0\xd8\x34\x5a\xab\x46\x52\x54\x04\x92\xd9\x79\xa8\x1c\x11\xca\x48\x31\x93\x28\xc9\x74\xbc\x78\xab\xea\x6d\xe8\x88\xb8\xed\x27\x6a\x34\x4d\x5d\x2b\x6d\xa3\xa1\x3a\x19\x15\x42\x02\x4d\xfc\x7a\xfa\xf2\xf5\xab\x47\xd9\x37\x94\x66\x3b\x19\x7a\x0e\x59\xa6\xd6\x08\x3e\xa9\xcf\xd4\x1d\x30\x59\xc0\x92\x6b\x04\xb6\x61\xdb\xaf\x92\xb8\x70\x25\xd8\xe7\xa3\x76\xd9\x7a\xd7\x3f\x15\x6c\x3b\x97\xbf\x84\xba\xc9\x04\x37\x25\x30\x90\xb8\x81\xc4\x58\xad\xe4\x6a\xe1\x46\x73\xaa\x49\xdd\x2b\xd4\xca\xd8\xc7\xcc\x8f\x55\x86\x45\x71\x06\x00\x5f\xca\xfe\x9b\xcd\x26\xea\x34\xe9\x8c\x5f\xa2\xa8\x63\xba\xfe\x1a\xc9\xed\x36\xf6\x6e\xa4\x64\xfc\x35\x2f\xd2\xeb\xd7\xd7\xaf\x5e\x5d\xbf\xf8\xb7\xd7\x2f\x5f\x5e\xbf\x7e\xf1\xf2\x21\x64\x90\x50\x9f\x09\x0c\x9f\x46\x7f\xab\xa8\x6c\xed\x73\x68\x8f\x97\x2e\x77\xa3\x08\x5d\x50\x0d\xa2\x83\x7f\x18\x43\x8d\xa4\x44\x24\x64\xe2\x6c\x0e\xf1\x09\x28\x72\x30\x7a\x84\xb3\xcf\x84\x56\x07\x1f\x42\x8a\x6a\x2c\x49\xd8\x55\xf3\x5c\xc9\x1e\x4e\x97\x60\x78\x55\x8b\x2d\xe4\x7b\xab\x9f\xc7\xd5\x83\x46\xf9\x5d\x58\x1d\x9a\xcd\x83\xcc\x45\xff\x4a\x15\x48\x51\xdf\x34\x26\xc7\xda\xb5\x79\x29\x92\xfe\x69\xfb\x1b\x93\x96\x4b\xec\x22\x6e\x04\xdf\x49\xb1\x85\xc6\x20\x2c\x95\x86\x02\xb3\x66\xb5\x72\x69\x82\x86\x5a\xf3\x35\xb3\xd8\x85\x59\xd3\xa2\xa2\x07\xc5\xa0\xb2\xa1\x94\x47\x0c\x32\x90\xbf\xa9\x06\x72\x26\xc1\x6a\x96\xdf\x7a\x4f\x69\xb4\x26\x4f\xa9\xd1\x4b\xd3\x07\xfa\x0c\x85\xda\xb8\x25\x5e\xee\x25\x47\xe1\xa2\xbe\x41\x84\x52\x6d\xa0\x6a\x72\xe7\x90\x14\xd5\x9d\x10\x1b\xc6\x2d\x34\xd2\x72\xe1\xf5\x69\x1b\x2d\x29\x47\xc0\x83\x28\x7d\x52\xfb\x25\x58\x2d\xde\x97\x78\x26\x25\xea\xab\x36\xd0\xf8\xd6\x2f\x87\x5a\x2b\x8b\x39\x19\x14\xd8\x8a\x71\x69\xc8\x22\x2e\x0f\xc0\xea\x23\xaa\xba\xfe\xa9\x7d\xd8\xb7\x28\xdd\x74\x1c\xc3\x5f\x85\xca\x98\x80\x35\x21\x3d\x13\x94\xce\x29\x28\x15\x89\x3e\xd0\x96\xb1\xcc\x36\x06\xd4\xd2\x8d\x7a\xce\x69\xff\x9a\x69\xb2\x20\x56\xb5\x85\xb4\x6d\xb0\xd1\x98\x41\xbd\x6e\xdb\x86\xf4\x4a\x95\xfb\xc1\x7c\xaf\xf5\x14\x7e\xfe\x65\xfe\xa4\x65\xe5\xcf\xb8\x74\x90\x20\x7c\x7b\x91\x6d\xc9\x2c\xe4\x1a\x99\x45\x03\xb9\x50\xa6\xd1\x9e\xc3\x42\xab\x1a\x88\xcb\x8e\x52\x47\x99\x26\x6a\x77\x5a\x47\x64\x5c\x32\x53\x4e\xda\xfe\xa0\x46\x67\xa5\x7e\xae\x1b\xbf\x20\xd4\x8d\x89\x00\x4f\xa7\x73\xe0\x49\x47\x37\x12\x28\x57\xb6\x9c\x03\x7f\xfe\xbc\x5f\x7c\xc1\x97\x30\xee\x56\xfc\xcc\x7f\x89\xec\x5d\x44\xa7\x40\x9a\xc2\xf0\x34\x77\x60\x4b\xc7\xd4\x82\xe7\x38\xe6\x97\x70\x35\x99\x77\xb3\x99\x46\x76\xdb\xbd\xb5\x76\xf4\xff\xb9\xbf\xbb\xf9\xa1\x66\x9c\xf2\x0f\x74\xe3\x6b\x7f\x03\x0c\x56\xdc\x58\x68\xb4\x80\xd6\x87\xbd\x09\x7a\x83\xb8\x75\x43\xad\x9c\xe0\xb2\x7d\x68\x31\xd5\x89\xe0\xc9\x44\x06\x65\x31\xfe\x8f\x1f\xbf\xfb\x36\x32\x56\x73\xb9\xe2\xcb\xed\xf8\xbe\xd1\x62\x06\x4f\xc7\xc1\xbf\x34\x5a\x04\x93\x9f\xa7\xbf\x44\x6b\x26\x1a\xbc\x74\xf6\x9e\xb9\xbf\x27\xa7\x5c\x42\xfb\x38\x83\xc3\x03\x77\x93\xc9\xfc\x7c\x9f\x64\xd0\xd6\xd1\x68\xd0\x8e\x69\x61\x0f\xfc\x63\x1d\x31\xa8\xd0\x96\xca\xb9\xae\xc6\x5c\x49\x89\xb9\x85\xa6\x56\xb2\x55\x09\x08\x65\xcc\x1e\x88\xdd\x8a\xf4\x14\x14\xed\xfa\xd4\x05\xeb\xff\xc6\xec\x47\x95\xdf\xa2\x1d\x8f\xc7\x1b\x2e\x0b\xb5\x89\x84\xf2\x57\x6d\x44\x4e\xaa\x72\x25\x20\x4d\x53\x68\xa3\x68\x30\x81\xaf\x21\xd8\x18\x8a\xa7\x01\xcc\xe8\x91\x9e\x26\xf0\x1c\x8e\xb7\x97\x14\xef\x9f\x43\x10\xb3\x9a\x07\x13\xef\x0e\x9d\xe2\x95\xac\xd0\x18\xb6\xc2\x21\x83\xae\x32\xea\x41\x46\x72\x54\x66\x05\x29\x38\x03\xd5\x4c\x1b\xf4\x4b\x22\xaa\xc6\x3b\xb4\x11\x66\xdd\xb2\x34\x05\xd9\x08\xb1\x07\xa9\x77\x8a\x79\x07\xbf\x83\xe5\x91\x8f\x35\x5f\xa5\x29\x50\x69\x4a\x2a\x2e\xf6\x3b\xc9\xf8\xbe\x88\x9e\x44\x14\x17\xf6\x3b\x26\xf3\x21\x9a\x0f\xa8\x61\xf1\x7b\xe4\xb0\x38\xa6\x87\xc5\x03\x04\x5d\xcf\xe2\x31\x7a\xbe\xc7\x31\x20\xe7\x06\x1e\xa0\x26\x9b\x2a\x43\xfd\x18\x39\xdf\xb3\x68\xc9\x39\x55\xbf\x93\x76\xb0\xf7\x12\xae\x5e\x4d\x1e\xa0\x8e\x5a\xab\x07\x89\x4b\x65\xb7\xe3\x7b\xc1\xb6\x94\x33\xc1\xc8\xaa\xfa\xad\x6b\x31\x8c\x2e\x5d\xc4\x9d\x41\x4f\xe1\xd2\x35\x8f\x67\x30\x72\x6f\x34\xcf\x2b\x74\xbb\x5e\x4e\xa7\xd3\x4b\xe8\x3e\xbb\xfc\x89\x91\x13\xea\x06\x77\x0f\xf0\x63\x9a\x3c\xa7\xb8\xff\x39\x1c\xb5\x34\x7a\x9e\xda\xf7\xcf\xe0\xaa\x8f\x0d\x07\x6c\xc1\x1f\xfe\x00\x27\xb3\x87\x30\x8e\x63\xf8\x2f\x46\x65\xb8\x10\xae\x7b\xe0\x9a\x06\xfd\xfa\x8a\x1b\xe3\x8a\x71\x03\x85\x92\xd8\xee\xf9\xb4\x6b\xff\x84\xc7\x76\x19\x2c\x60\x7a\xcc\x20\x5d\x87\x83\xb0\x70\x26\x5a\x0c\xe8\x1e\x06\x82\x8b\xdd\xf0\xbc\x83\x9d\xbc\x42\xf8\x2a\x85\x20\x18\x6e\x3e\x59\x41\x0b\x7a\x62\x17\x06\xed\x7b\x6f\x8b\x71\x1b\x1d\xcf\xc5\xae\xc9\x25\xdc\x4c\xa7\xd3\xc9\x09\x13\xbb\xbd\x7a\xdf\xd4\x94\x36\x01\x93\x5b\x77\x25\xf6\xba\x75\x89\x23\xa5\x40\x74\xa5\x09\xc8\x95\x10\x3e\x67\x69\xb7\x92\x82\xdb\xe6\x49\x0a\xe1\xd5\xfc\x4c\x14\x1d\x68\x72\x20\xda\xb1\x79\xce\xe8\xfe\xd8\x44\x87\x3a\x3b\x5a\x1c\x5e\x1d\x18\xe5\xc0\x5e\xe7\x0d\x73\xd1\xf3\xcd\xf7\x1a\x3d\x32\xd7\xde\x5e\xc7\x3a\x1b\xf0\xef\xe9\x3c\xbf\xfa\x48\x31\xfa\xe9\xba\x31\xe5\xf8\x88\xd1\xc9\xfc\xd4\x36\xef\x2c\x6a\xca\x92\x15\x85\x2c\xb2\x05\x95\x02\x1a\x4f\x4c\xe2\x52\x75\x8d\xa1\x46\x59\xa0\xee\x52\x0a\x9f\xd9\x53\x02\x78\x60\x32\x5f\x55\x0e\xe1\x34\x90\xe8\x44\xb7\x73\xe0\xb0\xa0\x34\x0f\x78\x18\x0e\x64\x71\x79\x99\x92\x08\x00\x70\xe4\x09\x0e\xad\x07\x70\xa5\xc5\x28\x58\x6d\xb0\x80\x14\xfc\xa7\xf0\xf1\x24\x6a\x24\xbf\x1b\x4f\xc2\xf6\xfd\x98\x46\x37\x3f\xef\x6b\xc5\x8e\xf7\xe7\x29\x04\x89\xd5\xc0\x8b\x74\x14\xc0\xf3\x73\x7e\x48\xa1\x77\xb4\xd8\x73\x30\xdc\x0a\x90\xd8\x62\xe1\x9a\xa1\xbe\x68\xfb\x7b\x90\xb1\xfc\x76\xe5\xaa\xa1\x19\xe5\x5b\xe3\x13\xb2\x6c\xcd\x2c\xd3\x8e\xea\x64\x0e\xfb\xe5\x6d\xb5\x98\x93\x85\xe6\xe0\xcb\x52\xd7\x73\x85\xfe\x3b\x85\x7b\xcb\x94\x2e\x50\x87\x9a\x15\xbc\x31\x33\x78\x51\xdf\xcd\xff\xde\x7d\xc7\x71\x9d\xe1\x47\x59\xad\x35\x2e\x4e\x38\x6a\x5b\x8d\xcf\x21\x48\x62\x5a\xf0\x7b\x64\x7a\x61\x87\x9f\xe0\xe1\x4c\xff\x1b\xfa\x0f\xe4\xed\x78\xc5\x8b\x42\x20\x31\xbc\x27\x4f\x1e\x49\xf6\x1f\xfa\xd5\xe1\x91\xd0\x36\xbe\xf7\x7b\x76\x80\xc2\xe0\x23\x1b\xfa\x1e\xfa\x88\x00\x10\x92\xc8\xdc\xe9\xbc\xad\xb8\xdd\xb0\x1e\x39\x5d\xb4\x3f\xa8\x28\x1a\xed\x12\xae\x71\xd8\x02\xec\x12\x46\x86\x12\xc0\xc2\x8c\x26\x51\xd9\x54\x4c\xf2\xdf\x70\x4c\xc1\x69\xe2\x75\xe5\x9a\xf2\xc1\xe9\xbd\x7c\xc2\xcc\xbe\x5b\x3e\xea\x02\xdd\xa8\x55\xe2\xa8\xb3\xee\x8b\x7d\x81\x3f\x83\xe9\x7c\xf4\x89\x1a\x3a\x7f\x4a\x98\x31\x0d\xc3\x97\xb0\x8b\xc0\xa0\x15\x9d\xde\xcd\x65\x4c\x8f\x7c\x3b\xc3\x25\xe9\x52\x6d\xd2\xd1\xcd\xb4\x67\xd2\x1b\xda\xd9\x79\xd4\x62\xed\xc4\x18\xc4\x65\xe7\x9a\x0b\xb8\x99\x7e\x09\x6e\x7d\x4b\xe4\x48\x02\xab\x79\x8d\x05\xb0\xdc\xf2\x35\xfe\x3f\x08\xf2\x05\x94\xfc\xc9\x2c\x12\x0e\x3b\xe5\x39\x98\x1e\xf0\x4b\xb3\xbd\x6e\xff\x95\xfc\x0d\x62\xa7\xe1\xe7\x10\x9c\x15\xe4\x41\x24\x1e\x2d\x3c\x72\xed\x87\xfd\xde\x7d\x65\x0a\x8e\x03\x0b\xa5\xbc\xfd\x17\xd2\x49\x54\xda\x4a\x8c\x83\xc4\xba\x9f\xca\x10\xcf\x3d\x05\x47\xc0\x0f\x1f\xe6\x75\xbb\xc3\x6a\x86\x8a\x78\x3c\x2a\xb6\x60\x90\xa1\xf4\x05\x59\x97\x8e\xc0\x6e\xff\x8b\xa2\x38\x86\x1f\x2d\xd3\x16\x18\xfc\xf4\x0e\x9a\xba\x60\xd6\x7f\xcf\xa1\x20\xe9\xbf\x97\x74\x3f\x39\xca\x98\x36\xb0\x54\x7a\xc3\x74\xd1\x36\x69\x6c\x89\x5b\xf7\x3d\xa7\xcb\xff\x0c\xda\x77\x74\x8b\xad\x99\x18\x9f\x14\x7f\x4f\xc7\xa3\x68\x68\xf2\xd1\x24\x42\x96\x97\xa7\x0b\x5d\xc4\xea\xcf\x4d\xe1\x5b\x57\x07\x8c\x9f\x8e\x6d\xc9\xcd\x24\x62\xd6\xea\xf1\xe8\x00\x0c\xa3\x09\xd9\xf5\x6a\x50\x97\xf5\xdb\x93\x03\xb7\x7a\x8c\xc6\x3e\xa3\xee\xb3\x81\x6e\x79\x6e\xcc\xd8\xe3\x6a\x74\x39\xa0\x7d\x08\xab\xd1\xb3\x51\x6f\xa8\xbd\x7b\xef\xe5\x48\xcf\x72\x72\x40\x7a\x44\x5e\x36\x3a\x39\x9e\x15\xc5\x5b\xf2\x9f\x71\x70\xc6\xd3\x8f\xd1\x31\xe9\x95\xed\xef\xeb\x47\xb5\xec\x7f\x9b\xf1\x80\x8a\x79\x31\x9a\x44\xa6\xc9\x7c\x83\x62\xfc\xb2\xaf\xc2\xba\x65\x0e\xbc\xc7\xa1\xe0\x24\xa1\xa0\x23\x0e\x93\x8a\xf0\x28\x09\x79\x24\x6a\xb4\x47\x7a\xa9\x76\x97\xa4\xf0\xe9\xa4\xef\x6f\x7d\x63\x28\xc3\xf2\xfd\xff\x0d\x66\xc6\xb5\x13\xa0\xc5\xbb\x6b\xe9\xf8\xd6\xcd\x9b\xef\xdf\x0d\xda\x37\xbd\x47\x8c\x1d\xf5\xfe\xd7\x80\xe7\x9a\x25\x67\x7f\x7e\xb8\xd9\x6c\xa2\x95\x52\x2b\xe1\x7f\x78\xd8\x77\x53\x62\x56\xf3\xe8\x83\x09\x80\x99\xad\xcc\xa1\xc0\x25\xea\xc5\x80\x7c\xdb\x62\x49\x62\xff\xc3\xb8\x24\xf6\xbf\xfd\xfd\xbf\x00\x00\x00\xff\xff\xb2\x1e\x6f\x68\x0c\x2c\x00\x00") func faucetHtmlBytes() ([]byte, error) { return bindataRead( @@ -85,7 +85,7 @@ func faucetHtml() (*asset, error) { } info := bindataFileInfo{name: "faucet.html", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xdb, 0xa2, 0x98, 0x44, 0x4b, 0x50, 0xf8, 0xa1, 0xac, 0x4a, 0x76, 0x2e, 0xcc, 0x3d, 0xcb, 0x81, 0x9e, 0x2a, 0xaa, 0x87, 0xf5, 0x9d, 0x53, 0x4, 0x8a, 0xdd, 0x5a, 0xfe, 0xd3, 0xc3, 0xf, 0x11}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xc5, 0x8d, 0xb, 0x7a, 0xfd, 0x70, 0x68, 0x68, 0xd2, 0xd8, 0xf3, 0xf6, 0xac, 0x72, 0xed, 0xc2, 0x76, 0x18, 0x2d, 0x1, 0xe5, 0x3b, 0x55, 0xb, 0xce, 0xfc, 0xb6, 0xd5, 0x59, 0xc3, 0x94, 0x5b}} return a, nil } From c4deebbf1e186e3b7e96c4e0ab395d3207cec55e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 15 Jan 2021 12:26:46 +0200 Subject: [PATCH 069/709] core/state/snapshot: add generation logs to storage too --- core/state/snapshot/generate.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 17f1ca6078..fcc6b44cb6 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -281,6 +281,10 @@ func (dl *diskLayer) generate(stats *generatorStats) { abort <- stats return } + if time.Since(logged) > 8*time.Second { + stats.Log("Generating state snapshot", dl.root, append(accountHash[:], storeIt.Key...)) + logged = time.Now() + } } } if err := storeIt.Err; err != nil { From 8d62ee65b2d3100da0292232f8169282237f5487 Mon Sep 17 00:00:00 2001 From: gary rong Date: Sat, 16 Jan 2021 06:04:38 +0800 Subject: [PATCH 070/709] les: don't drop sentTo for normal cases (#22048) --- les/retrieve.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/les/retrieve.go b/les/retrieve.go index ca4f867ea8..3174d49878 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -337,7 +337,6 @@ func (r *sentReq) tryRequest() { } defer func() { - // send feedback to server pool and remove peer if hard timeout happened pp, ok := p.(*serverPeer) if hrto && ok { pp.Log().Debug("Request timed out hard") @@ -345,10 +344,6 @@ func (r *sentReq) tryRequest() { r.rm.peers.unregister(pp.id) } } - - r.lock.Lock() - delete(r.sentTo, p) - r.lock.Unlock() }() select { From c76573a97b15e28c0d5c783cab3a62e9203db1c9 Mon Sep 17 00:00:00 2001 From: Dan DeGreef Date: Sat, 16 Jan 2021 11:15:18 -0600 Subject: [PATCH 071/709] eth/protocols/eth: fix slice resize flaw (#22181) --- eth/protocols/eth/broadcast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 2349398fae..74ec2f0654 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -179,7 +179,7 @@ func (p *Peer) announceTransactions() { queue = append(queue, hashes...) if len(queue) > maxQueuedTxAnns { // Fancy copy and resize to ensure buffer doesn't grow indefinitely - queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxAnns:])] } case <-done: From 034ecc3210623d1585f0ffd6a2c8a64d4b7ee0c1 Mon Sep 17 00:00:00 2001 From: gary rong Date: Sun, 17 Jan 2021 02:06:18 +0800 Subject: [PATCH 072/709] les: remove useless protocol defines (#22115) This PR has two changes in the les protocol: - the auxRoot is not supported. See ethereum/devp2p#171 for more information - the empty response will be returned in GetHelperTrieProofsMsg request if the merkle proving is failed. note, for backward compatibility, the empty merkle proof as well as the request auxiliary data will still be returned in les2/3 protocol no matter the proving is successful or not. the proving failure can happen e.g. request the proving for a non-included entry in helper trie (unstable header). --- les/benchmark.go | 2 +- les/handler_test.go | 2 +- les/odr_requests.go | 9 ++++----- les/server_handler.go | 32 +++++++++++++++++--------------- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/les/benchmark.go b/les/benchmark.go index 312d4533df..6255c1049e 100644 --- a/les/benchmark.go +++ b/les/benchmark.go @@ -156,7 +156,7 @@ func (b *benchmarkHelperTrie) request(peer *serverPeer, index int) error { for i := range reqs { key := make([]byte, 8) binary.BigEndian.PutUint64(key[:], uint64(rand.Int63n(int64(b.headNum)))) - reqs[i] = HelperTrieReq{Type: htCanonical, TrieIdx: b.sectionCount - 1, Key: key, AuxReq: auxHeader} + reqs[i] = HelperTrieReq{Type: htCanonical, TrieIdx: b.sectionCount - 1, Key: key, AuxReq: htAuxHeader} } } diff --git a/les/handler_test.go b/les/handler_test.go index 04277f661b..f5cbeb8efc 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -443,7 +443,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { Type: htCanonical, TrieIdx: 0, Key: key, - AuxReq: auxHeader, + AuxReq: htAuxHeader, }} // Send the proof request and verify the response sendRequest(server.peer.app, GetHelperTrieProofsMsg, 42, requestsV2) diff --git a/les/odr_requests.go b/les/odr_requests.go index a8cf8f50a9..962b88a322 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -295,10 +295,9 @@ const ( htCanonical = iota // Canonical hash trie htBloomBits // BloomBits trie - // applicable for all helper trie requests - auxRoot = 1 - // applicable for htCanonical - auxHeader = 2 + // helper trie auxiliary types + // htAuxNone = 1 ; deprecated number, used in les2/3 previously. + htAuxHeader = 2 // applicable for htCanonical, requests for relevant headers ) type HelperTrieReq struct { @@ -339,7 +338,7 @@ func (r *ChtRequest) Request(reqID uint64, peer *serverPeer) error { Type: htCanonical, TrieIdx: r.ChtNum, Key: encNum[:], - AuxReq: auxHeader, + AuxReq: htAuxHeader, } return peer.requestHelperTrieProofs(reqID, []HelperTrieReq{req}) } diff --git a/les/server_handler.go b/les/server_handler.go index 2316c9c5a4..bec4206e2b 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -741,22 +741,24 @@ func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error { auxTrie, _ = trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) } } - if request.AuxReq == auxRoot { - var data []byte - if root != (common.Hash{}) { - data = root[:] - } + if auxTrie == nil { + sendResponse(req.ReqID, 0, nil, task.servingTime) + return + } + // TODO(rjl493456442) short circuit if the proving is failed. + // The original client side code has a dirty hack to retrieve + // the headers with no valid proof. Keep the compatibility for + // legacy les protocol and drop this hack when the les2/3 are + // not supported. + err := auxTrie.Prove(request.Key, request.FromLevel, nodes) + if p.version >= lpv4 && err != nil { + sendResponse(req.ReqID, 0, nil, task.servingTime) + return + } + if request.AuxReq == htAuxHeader { + data := h.getAuxiliaryHeaders(request) auxData = append(auxData, data) auxBytes += len(data) - } else { - if auxTrie != nil { - auxTrie.Prove(request.Key, request.FromLevel, nodes) - } - if request.AuxReq != 0 { - data := h.getAuxiliaryHeaders(request) - auxData = append(auxData, data) - auxBytes += len(data) - } } if nodes.DataSize()+auxBytes >= softResponseLimit { break @@ -904,7 +906,7 @@ func (h *serverHandler) getHelperTrie(typ uint, index uint64) (common.Hash, stri // getAuxiliaryHeaders returns requested auxiliary headers for the CHT request. func (h *serverHandler) getAuxiliaryHeaders(req HelperTrieReq) []byte { - if req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8 { + if req.Type == htCanonical && req.AuxReq == htAuxHeader && len(req.Key) == 8 { blockNum := binary.BigEndian.Uint64(req.Key) hash := rawdb.ReadCanonicalHash(h.chainDb, blockNum) return rawdb.ReadHeaderRLP(h.chainDb, hash, blockNum) From 398182284cb1635be833017e87d484795a5e5c56 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 18 Jan 2021 14:33:15 +0100 Subject: [PATCH 073/709] tests/fuzzers/abi: better test generation (#22158) * tests/fuzzers/abi: better test generation * tests/fuzzers/abi: fixed packing issue * oss-fuzz: enable abi fuzzer --- oss-fuzz.sh | 6 +- tests/fuzzers/abi/abifuzzer.go | 115 ++++++++++++---------------- tests/fuzzers/abi/abifuzzer_test.go | 9 +-- 3 files changed, 51 insertions(+), 79 deletions(-) diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 5919b2077f..7ae4d77b29 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -100,6 +100,7 @@ compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty +compile_fuzzertests/fuzzers/abi Fuzz fuzzAbi compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul @@ -113,8 +114,3 @@ compile_fuzzer tests/fuzzers/bls12381 FuzzMapG2 fuzz_map_g2 #TODO: move this to tests/fuzzers, if possible compile_fuzzer crypto/blake2b Fuzz fuzzBlake2b - - -# This doesn't work very well @TODO -#compile_fuzzertests/fuzzers/abi Fuzz fuzzAbi - diff --git a/tests/fuzzers/abi/abifuzzer.go b/tests/fuzzers/abi/abifuzzer.go index 76d3c800f7..8c083b371e 100644 --- a/tests/fuzzers/abi/abifuzzer.go +++ b/tests/fuzzers/abi/abifuzzer.go @@ -17,38 +17,53 @@ package abi import ( - "bytes" "fmt" - "math/rand" "reflect" "strings" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/crypto" fuzz "github.com/google/gofuzz" ) -func unpackPack(abi abi.ABI, method string, inputType []interface{}, input []byte) bool { - outptr := reflect.New(reflect.TypeOf(inputType)) - if err := abi.UnpackIntoInterface(outptr.Interface(), method, input); err == nil { - output, err := abi.Pack(method, input) +var ( + names = []string{"_name", "name", "NAME", "name_", "__", "_name_", "n"} + stateMut = []string{"", "pure", "view", "payable"} + stateMutabilites = []*string{&stateMut[0], &stateMut[1], &stateMut[2], &stateMut[3]} + pays = []string{"", "true", "false"} + payables = []*string{&pays[0], &pays[1]} + vNames = []string{"a", "b", "c", "d", "e", "f", "g"} + varNames = append(vNames, names...) + varTypes = []string{"bool", "address", "bytes", "string", + "uint8", "int8", "uint8", "int8", "uint16", "int16", + "uint24", "int24", "uint32", "int32", "uint40", "int40", "uint48", "int48", "uint56", "int56", + "uint64", "int64", "uint72", "int72", "uint80", "int80", "uint88", "int88", "uint96", "int96", + "uint104", "int104", "uint112", "int112", "uint120", "int120", "uint128", "int128", "uint136", "int136", + "uint144", "int144", "uint152", "int152", "uint160", "int160", "uint168", "int168", "uint176", "int176", + "uint184", "int184", "uint192", "int192", "uint200", "int200", "uint208", "int208", "uint216", "int216", + "uint224", "int224", "uint232", "int232", "uint240", "int240", "uint248", "int248", "uint256", "int256", + "bytes1", "bytes2", "bytes3", "bytes4", "bytes5", "bytes6", "bytes7", "bytes8", "bytes9", "bytes10", "bytes11", + "bytes12", "bytes13", "bytes14", "bytes15", "bytes16", "bytes17", "bytes18", "bytes19", "bytes20", "bytes21", + "bytes22", "bytes23", "bytes24", "bytes25", "bytes26", "bytes27", "bytes28", "bytes29", "bytes30", "bytes31", + "bytes32", "bytes"} +) + +func unpackPack(abi abi.ABI, method string, input []byte) ([]interface{}, bool) { + if out, err := abi.Unpack(method, input); err == nil { + _, err := abi.Pack(method, out...) if err != nil { // We have some false positives as we can unpack these type successfully, but not pack them if err.Error() == "abi: cannot use []uint8 as type [0]int8 as argument" || err.Error() == "abi: cannot use uint8 as type int8 as argument" { - return false + return out, false } panic(err) } - if !bytes.Equal(input, output[4:]) { - panic(fmt.Sprintf("unpackPack is not equal, \ninput : %x\noutput: %x", input, output[4:])) - } - return true + return out, true } - return false + return nil, false } -func packUnpack(abi abi.ABI, method string, input []interface{}) bool { +func packUnpack(abi abi.ABI, method string, input *[]interface{}) bool { if packed, err := abi.Pack(method, input); err == nil { outptr := reflect.New(reflect.TypeOf(input)) err := abi.UnpackIntoInterface(outptr.Interface(), method, packed) @@ -100,64 +115,23 @@ func createABI(name string, stateMutability, payable *string, inputs []args) (ab return abi.JSON(strings.NewReader(sig)) } -func fillStruct(structs []interface{}, data []byte) { - if structs != nil && len(data) != 0 { - fuzz.NewFromGoFuzz(data).Fuzz(&structs) - } -} - -func createStructs(args []args) []interface{} { - structs := make([]interface{}, len(args)) - for i, arg := range args { - t, err := abi.NewType(arg.typ, "", nil) - if err != nil { - panic(err) - } - structs[i] = reflect.New(t.GetType()).Elem() - } - return structs -} - func runFuzzer(input []byte) int { good := false + fuzzer := fuzz.NewFromGoFuzz(input) - names := []string{"_name", "name", "NAME", "name_", "__", "_name_", "n"} - stateMut := []string{"", "pure", "view", "payable"} - stateMutabilites := []*string{nil, &stateMut[0], &stateMut[1], &stateMut[2], &stateMut[3]} - pays := []string{"true", "false"} - payables := []*string{nil, &pays[0], &pays[1]} - varNames := []string{"a", "b", "c", "d", "e", "f", "g"} - varNames = append(varNames, names...) - varTypes := []string{"bool", "address", "bytes", "string", - "uint8", "int8", "uint8", "int8", "uint16", "int16", - "uint24", "int24", "uint32", "int32", "uint40", "int40", "uint48", "int48", "uint56", "int56", - "uint64", "int64", "uint72", "int72", "uint80", "int80", "uint88", "int88", "uint96", "int96", - "uint104", "int104", "uint112", "int112", "uint120", "int120", "uint128", "int128", "uint136", "int136", - "uint144", "int144", "uint152", "int152", "uint160", "int160", "uint168", "int168", "uint176", "int176", - "uint184", "int184", "uint192", "int192", "uint200", "int200", "uint208", "int208", "uint216", "int216", - "uint224", "int224", "uint232", "int232", "uint240", "int240", "uint248", "int248", "uint256", "int256", - "bytes1", "bytes2", "bytes3", "bytes4", "bytes5", "bytes6", "bytes7", "bytes8", "bytes9", "bytes10", "bytes11", - "bytes12", "bytes13", "bytes14", "bytes15", "bytes16", "bytes17", "bytes18", "bytes19", "bytes20", "bytes21", - "bytes22", "bytes23", "bytes24", "bytes25", "bytes26", "bytes27", "bytes28", "bytes29", "bytes30", "bytes31", - "bytes32", "bytes"} - rnd := rand.New(rand.NewSource(123456)) - if len(input) > 0 { - kec := crypto.Keccak256(input) - rnd = rand.New(rand.NewSource(int64(kec[0]))) - } - name := names[rnd.Intn(len(names))] - stateM := stateMutabilites[rnd.Intn(len(stateMutabilites))] - payable := payables[rnd.Intn(len(payables))] + name := names[getUInt(fuzzer)%len(names)] + stateM := stateMutabilites[getUInt(fuzzer)%len(stateMutabilites)] + payable := payables[getUInt(fuzzer)%len(payables)] maxLen := 5 for k := 1; k < maxLen; k++ { var arg []args for i := k; i > 0; i-- { argName := varNames[i] - argTyp := varTypes[rnd.Int31n(int32(len(varTypes)))] - if rnd.Int31n(10) == 0 { + argTyp := varTypes[getUInt(fuzzer)%len(varTypes)] + if getUInt(fuzzer)%10 == 0 { argTyp += "[]" - } else if rnd.Int31n(10) == 0 { - arrayArgs := rnd.Int31n(30) + 1 + } else if getUInt(fuzzer)%10 == 0 { + arrayArgs := getUInt(fuzzer)%30 + 1 argTyp += fmt.Sprintf("[%d]", arrayArgs) } arg = append(arg, args{ @@ -169,10 +143,8 @@ func runFuzzer(input []byte) int { if err != nil { continue } - structs := createStructs(arg) - b := unpackPack(abi, name, structs, input) - fillStruct(structs, input) - c := packUnpack(abi, name, structs) + structs, b := unpackPack(abi, name, input) + c := packUnpack(abi, name, &structs) good = good || b || c } if good { @@ -184,3 +156,12 @@ func runFuzzer(input []byte) int { func Fuzz(input []byte) int { return runFuzzer(input) } + +func getUInt(fuzzer *fuzz.Fuzzer) int { + var i int + fuzzer.Fuzz(&i) + if i < 0 { + i *= -1 + } + return i +} diff --git a/tests/fuzzers/abi/abifuzzer_test.go b/tests/fuzzers/abi/abifuzzer_test.go index c72577e9ee..c59c45ab1a 100644 --- a/tests/fuzzers/abi/abifuzzer_test.go +++ b/tests/fuzzers/abi/abifuzzer_test.go @@ -23,13 +23,8 @@ import ( // TestReplicate can be used to replicate crashers from the fuzzing tests. // Just replace testString with the data in .quoted func TestReplicate(t *testing.T) { - testString := "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00" + - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + - "\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00000000000" + - "00000000000000000000" + - "00000000000000000000" + - "00000001" + testString := "N\xef\xbf0\xef\xbf99000000000000" + + "000000000000" data := []byte(testString) runFuzzer(data) From 10555d46849fc805aa28921fed4d46e4bdaf0c4c Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 18 Jan 2021 14:36:05 +0100 Subject: [PATCH 074/709] cmd/geth: dump config for metrics (#22083) * cmd/geth: dump config * cmd/geth: dump config * cmd/geth: properly read config again * cmd/geth: override metrics if flags are set * cmd/geth: write metrics regardless if enabled * cmd/geth: renamed to metricsfromcliargs * metrics: add default configuration --- cmd/geth/config.go | 43 ++++++++++++++++++++++++++++++++++++++++--- cmd/utils/flags.go | 14 +++++++------- metrics/config.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 metrics/config.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index bf1fc55b17..c5b330b2d8 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/naoina/toml" @@ -88,6 +89,7 @@ type gethConfig struct { Shh whisperDeprecatedConfig Node node.Config Ethstats ethstatsConfig + Metrics metrics.Config } func loadConfig(file string, cfg *gethConfig) error { @@ -119,8 +121,9 @@ func defaultNodeConfig() node.Config { func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // Load defaults. cfg := gethConfig{ - Eth: eth.DefaultConfig, - Node: defaultNodeConfig(), + Eth: eth.DefaultConfig, + Node: defaultNodeConfig(), + Metrics: metrics.DefaultConfig, } // Load config file. @@ -133,7 +136,6 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { log.Warn("Deprecated whisper config detected. Whisper has been moved to github.com/ethereum/whisper") } } - // Apply flags. utils.SetNodeConfig(ctx, &cfg.Node) stack, err := node.New(&cfg.Node) @@ -146,6 +148,8 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { } utils.SetShhConfig(ctx, stack) + applyMetricConfig(ctx, &cfg) + return stack, cfg } @@ -204,3 +208,36 @@ func dumpConfig(ctx *cli.Context) error { return nil } + +func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { + if ctx.GlobalIsSet(utils.MetricsEnabledFlag.Name) { + cfg.Metrics.Enabled = ctx.GlobalBool(utils.MetricsEnabledFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsEnabledExpensiveFlag.Name) { + cfg.Metrics.EnabledExpensive = ctx.GlobalBool(utils.MetricsEnabledExpensiveFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsHTTPFlag.Name) { + cfg.Metrics.HTTP = ctx.GlobalString(utils.MetricsHTTPFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsPortFlag.Name) { + cfg.Metrics.Port = ctx.GlobalInt(utils.MetricsPortFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBFlag.Name) { + cfg.Metrics.EnableInfluxDB = ctx.GlobalBool(utils.MetricsEnableInfluxDBFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBEndpointFlag.Name) { + cfg.Metrics.InfluxDBEndpoint = ctx.GlobalString(utils.MetricsInfluxDBEndpointFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBDatabaseFlag.Name) { + cfg.Metrics.InfluxDBDatabase = ctx.GlobalString(utils.MetricsInfluxDBDatabaseFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBUsernameFlag.Name) { + cfg.Metrics.InfluxDBUsername = ctx.GlobalString(utils.MetricsInfluxDBUsernameFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBPasswordFlag.Name) { + cfg.Metrics.InfluxDBPassword = ctx.GlobalString(utils.MetricsInfluxDBPasswordFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) { + cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name) + } +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 6f4da58320..688e25618d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -684,12 +684,12 @@ var ( MetricsHTTPFlag = cli.StringFlag{ Name: "metrics.addr", Usage: "Enable stand-alone metrics HTTP server listening interface", - Value: "127.0.0.1", + Value: metrics.DefaultConfig.HTTP, } MetricsPortFlag = cli.IntFlag{ Name: "metrics.port", Usage: "Metrics HTTP server listening port", - Value: 6060, + Value: metrics.DefaultConfig.Port, } MetricsEnableInfluxDBFlag = cli.BoolFlag{ Name: "metrics.influxdb", @@ -698,22 +698,22 @@ var ( MetricsInfluxDBEndpointFlag = cli.StringFlag{ Name: "metrics.influxdb.endpoint", Usage: "InfluxDB API endpoint to report metrics to", - Value: "http://localhost:8086", + Value: metrics.DefaultConfig.InfluxDBEndpoint, } MetricsInfluxDBDatabaseFlag = cli.StringFlag{ Name: "metrics.influxdb.database", Usage: "InfluxDB database name to push reported metrics to", - Value: "geth", + Value: metrics.DefaultConfig.InfluxDBDatabase, } MetricsInfluxDBUsernameFlag = cli.StringFlag{ Name: "metrics.influxdb.username", Usage: "Username to authorize access to the database", - Value: "test", + Value: metrics.DefaultConfig.InfluxDBUsername, } MetricsInfluxDBPasswordFlag = cli.StringFlag{ Name: "metrics.influxdb.password", Usage: "Password to authorize access to the database", - Value: "test", + Value: metrics.DefaultConfig.InfluxDBPassword, } // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. // For example `host` tag could be used so that we can group all nodes and average a measurement @@ -722,7 +722,7 @@ var ( MetricsInfluxDBTagsFlag = cli.StringFlag{ Name: "metrics.influxdb.tags", Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", - Value: "host=localhost", + Value: metrics.DefaultConfig.InfluxDBTags, } EWASMInterpreterFlag = cli.StringFlag{ Name: "vm.ewasm", diff --git a/metrics/config.go b/metrics/config.go new file mode 100644 index 0000000000..d05d664265 --- /dev/null +++ b/metrics/config.go @@ -0,0 +1,45 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of go-ethereum. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +// Config contains the configuration for the metric collection. +type Config struct { + Enabled bool `toml:",omitempty"` + EnabledExpensive bool `toml:",omitempty"` + HTTP string `toml:",omitempty"` + Port int `toml:",omitempty"` + EnableInfluxDB bool `toml:",omitempty"` + InfluxDBEndpoint string `toml:",omitempty"` + InfluxDBDatabase string `toml:",omitempty"` + InfluxDBUsername string `toml:",omitempty"` + InfluxDBPassword string `toml:",omitempty"` + InfluxDBTags string `toml:",omitempty"` +} + +// DefaultConfig is the default config for metrics used in go-ethereum. +var DefaultConfig = Config{ + Enabled: false, + EnabledExpensive: false, + HTTP: "127.0.0.1", + Port: 6060, + EnableInfluxDB: false, + InfluxDBEndpoint: "http://localhost:8086", + InfluxDBDatabase: "geth", + InfluxDBUsername: "test", + InfluxDBPassword: "test", + InfluxDBTags: "host=localhost", +} From 5e9f5ca5d302298b933668af539ad1e213bdfa6e Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 18 Jan 2021 21:39:43 +0800 Subject: [PATCH 075/709] core/state/snapshot: write snapshot generator in batch (#22163) * core/state/snapshot: write snapshot generator in batch * core: refactor the tests * core: update tests * core: update tests --- core/blockchain_snapshot_test.go | 1033 ++++++++++++++++++------------ core/state/snapshot/generate.go | 35 +- core/state/snapshot/journal.go | 2 +- 3 files changed, 653 insertions(+), 417 deletions(-) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index f35dae1678..cb634a451d 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -28,27 +28,19 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) -// snapshotTest is a test case for snapshot recovery. It can be used for -// simulating these scenarios: -// (i) Geth restarts normally with valid legacy snapshot -// (ii) Geth restarts normally with valid new-format snapshot -// (iii) Geth restarts after the crash, with broken legacy snapshot -// (iv) Geth restarts after the crash, with broken new-format snapshot -// (v) Geth restarts normally, but it's requested to be rewound to a lower point via SetHead -// (vi) Geth restarts normally with a stale snapshot -type snapshotTest struct { - legacy bool // Flag whether the loaded snapshot is in legacy format - crash bool // Flag whether the Geth restarts from the previous crash - restartCrash int // Number of blocks to insert after the normal stop, then the crash happens - gapped int // Number of blocks to insert without enabling snapshot - setHead uint64 // Block number to set head back to - +// snapshotTestBasic wraps the common testing fields in the snapshot tests. +type snapshotTestBasic struct { + legacy bool // Wether write the snapshot journal in legacy format chainBlocks int // Number of blocks to generate for the canonical chain snapshotBlock uint64 // Block number of the relevant snapshot disk layer commitBlock uint64 // Block number for which to commit the state to disk @@ -58,56 +50,418 @@ type snapshotTest struct { expHeadFastBlock uint64 // Block number of the expected head fast sync block expHeadBlock uint64 // Block number of the expected head full block expSnapshotBottom uint64 // The block height corresponding to the snapshot disk layer + + // share fields, set in runtime + datadir string + db ethdb.Database + gendb ethdb.Database + engine consensus.Engine +} + +func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { + // Create a temporary persistent database + datadir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create temporary datadir: %v", err) + } + os.RemoveAll(datadir) + + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") + if err != nil { + t.Fatalf("Failed to create persistent database: %v", err) + } + // Initialize a fresh chain + var ( + genesis = new(Genesis).MustCommit(db) + engine = ethash.NewFullFaker() + gendb = rawdb.NewMemoryDatabase() + + // Snapshot is enabled, the first snapshot is created from the Genesis. + // The snapshot memory allowance is 256MB, it means no snapshot flush + // will happen during the block insertion. + cacheConfig = defaultCacheConfig + ) + chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to create chain: %v", err) + } + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, gendb, basic.chainBlocks, func(i int, b *BlockGen) {}) + + // Insert the blocks with configured settings. + var breakpoints []uint64 + if basic.commitBlock > basic.snapshotBlock { + breakpoints = append(breakpoints, basic.snapshotBlock, basic.commitBlock) + } else { + breakpoints = append(breakpoints, basic.commitBlock, basic.snapshotBlock) + } + var startPoint uint64 + for _, point := range breakpoints { + if _, err := chain.InsertChain(blocks[startPoint:point]); err != nil { + t.Fatalf("Failed to import canonical chain start: %v", err) + } + startPoint = point + + if basic.commitBlock > 0 && basic.commitBlock == point { + chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil) + } + if basic.snapshotBlock > 0 && basic.snapshotBlock == point { + if basic.legacy { + // Here we commit the snapshot disk root to simulate + // committing the legacy snapshot. + rawdb.WriteSnapshotRoot(db, blocks[point-1].Root()) + } else { + // Flushing the entire snap tree into the disk, the + // relavant (a) snapshot root and (b) snapshot generator + // will be persisted atomically. + chain.snaps.Cap(blocks[point-1].Root(), 0) + diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root() + if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) { + t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot) + } + } + } + } + if _, err := chain.InsertChain(blocks[startPoint:]); err != nil { + t.Fatalf("Failed to import canonical chain tail: %v", err) + } + + // Set runtime fields + basic.datadir = datadir + basic.db = db + basic.gendb = gendb + basic.engine = engine + + // Ugly hack, notify the chain to flush the journal in legacy format + // if it's requested. + if basic.legacy { + chain.writeLegacyJournal = true + } + return chain, blocks } -func (tt *snapshotTest) dump() string { +func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks []*types.Block) { + // Iterate over all the remaining blocks and ensure there are no gaps + verifyNoGaps(t, chain, true, blocks) + verifyCutoff(t, chain, true, blocks, basic.expCanonicalBlocks) + + if head := chain.CurrentHeader(); head.Number.Uint64() != basic.expHeadHeader { + t.Errorf("Head header mismatch: have %d, want %d", head.Number, basic.expHeadHeader) + } + if head := chain.CurrentFastBlock(); head.NumberU64() != basic.expHeadFastBlock { + t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadFastBlock) + } + if head := chain.CurrentBlock(); head.NumberU64() != basic.expHeadBlock { + t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadBlock) + } + + // Check the disk layer, ensure they are matched + block := chain.GetBlockByNumber(basic.expSnapshotBottom) + if block == nil { + t.Errorf("The correspnding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom) + } else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) { + t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot()) + } + + // Check the snapshot, ensure it's integrated + if err := snapshot.VerifyState(chain.snaps, block.Root()); err != nil { + t.Errorf("The disk layer is not integrated %v", err) + } +} + +func (basic *snapshotTestBasic) dump() string { buffer := new(strings.Builder) fmt.Fprint(buffer, "Chain:\n G") - for i := 0; i < tt.chainBlocks; i++ { + for i := 0; i < basic.chainBlocks; i++ { fmt.Fprintf(buffer, "->C%d", i+1) } fmt.Fprint(buffer, " (HEAD)\n\n") fmt.Fprintf(buffer, "Commit: G") - if tt.commitBlock > 0 { - fmt.Fprintf(buffer, ", C%d", tt.commitBlock) + if basic.commitBlock > 0 { + fmt.Fprintf(buffer, ", C%d", basic.commitBlock) } fmt.Fprint(buffer, "\n") fmt.Fprintf(buffer, "Snapshot: G") - if tt.snapshotBlock > 0 { - fmt.Fprintf(buffer, ", C%d", tt.snapshotBlock) + if basic.snapshotBlock > 0 { + fmt.Fprintf(buffer, ", C%d", basic.snapshotBlock) } fmt.Fprint(buffer, "\n") - if tt.crash { - fmt.Fprintf(buffer, "\nCRASH\n\n") - } else { - fmt.Fprintf(buffer, "\nSetHead(%d)\n\n", tt.setHead) - } + //if crash { + // fmt.Fprintf(buffer, "\nCRASH\n\n") + //} else { + // fmt.Fprintf(buffer, "\nSetHead(%d)\n\n", basic.setHead) + //} fmt.Fprintf(buffer, "------------------------------\n\n") fmt.Fprint(buffer, "Expected in leveldb:\n G") - for i := 0; i < tt.expCanonicalBlocks; i++ { + for i := 0; i < basic.expCanonicalBlocks; i++ { fmt.Fprintf(buffer, "->C%d", i+1) } fmt.Fprintf(buffer, "\n\n") - fmt.Fprintf(buffer, "Expected head header : C%d\n", tt.expHeadHeader) - fmt.Fprintf(buffer, "Expected head fast block: C%d\n", tt.expHeadFastBlock) - if tt.expHeadBlock == 0 { + fmt.Fprintf(buffer, "Expected head header : C%d\n", basic.expHeadHeader) + fmt.Fprintf(buffer, "Expected head fast block: C%d\n", basic.expHeadFastBlock) + if basic.expHeadBlock == 0 { fmt.Fprintf(buffer, "Expected head block : G\n") } else { - fmt.Fprintf(buffer, "Expected head block : C%d\n", tt.expHeadBlock) + fmt.Fprintf(buffer, "Expected head block : C%d\n", basic.expHeadBlock) } - if tt.expSnapshotBottom == 0 { + if basic.expSnapshotBottom == 0 { fmt.Fprintf(buffer, "Expected snapshot disk : G\n") } else { - fmt.Fprintf(buffer, "Expected snapshot disk : C%d\n", tt.expSnapshotBottom) + fmt.Fprintf(buffer, "Expected snapshot disk : C%d\n", basic.expSnapshotBottom) } return buffer.String() } +func (basic *snapshotTestBasic) teardown() { + basic.db.Close() + basic.gendb.Close() + os.RemoveAll(basic.datadir) +} + +// snapshotTest is a test case type for normal snapshot recovery. +// It can be used for testing that restart Geth normally. +type snapshotTest struct { + snapshotTestBasic +} + +func (snaptest *snapshotTest) test(t *testing.T) { + // It's hard to follow the test case, visualize the input + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // fmt.Println(tt.dump()) + chain, blocks := snaptest.prepare(t) + + // Restart the chain normally + chain.Stop() + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + defer newchain.Stop() + + snaptest.verify(t, newchain, blocks) +} + +// crashSnapshotTest is a test case type for innormal snapshot recovery. +// It can be used for testing that restart Geth after the crash. +type crashSnapshotTest struct { + snapshotTestBasic +} + +func (snaptest *crashSnapshotTest) test(t *testing.T) { + // It's hard to follow the test case, visualize the input + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // fmt.Println(tt.dump()) + chain, blocks := snaptest.prepare(t) + + // Pull the plug on the database, simulating a hard crash + db := chain.db + db.Close() + + // Start a new blockchain back up and see where the repair leads us + newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "") + if err != nil { + t.Fatalf("Failed to reopen persistent database: %v", err) + } + defer newdb.Close() + + // The interesting thing is: instead of starting the blockchain after + // the crash, we do restart twice here: one after the crash and one + // after the normal stop. It's used to ensure the broken snapshot + // can be detected all the time. + newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + newchain.Stop() + + newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + defer newchain.Stop() + + snaptest.verify(t, newchain, blocks) +} + +// gappedSnapshotTest is a test type used to test this scenario: +// - have a complete snapshot +// - restart without enabling the snapshot +// - insert a few blocks +// - restart with enabling the snapshot again +type gappedSnapshotTest struct { + snapshotTestBasic + gapped int // Number of blocks to insert without enabling snapshot +} + +func (snaptest *gappedSnapshotTest) test(t *testing.T) { + // It's hard to follow the test case, visualize the input + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // fmt.Println(tt.dump()) + chain, blocks := snaptest.prepare(t) + + // Insert blocks without enabling snapshot if gapping is required. + chain.Stop() + gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.gapped, func(i int, b *BlockGen) {}) + + // Insert a few more blocks without enabling snapshot + var cacheConfig = &CacheConfig{ + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 0, + } + newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + newchain.InsertChain(gappedBlocks) + newchain.Stop() + + // Restart the chain with enabling the snapshot + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + defer newchain.Stop() + + snaptest.verify(t, newchain, blocks) +} + +// setHeadSnapshotTest is the test type used to test this scenario: +// - have a complete snapshot +// - set the head to a lower point +// - restart +type setHeadSnapshotTest struct { + snapshotTestBasic + setHead uint64 // Block number to set head back to +} + +func (snaptest *setHeadSnapshotTest) test(t *testing.T) { + // It's hard to follow the test case, visualize the input + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // fmt.Println(tt.dump()) + chain, blocks := snaptest.prepare(t) + + // Rewind the chain if setHead operation is required. + chain.SetHead(snaptest.setHead) + chain.Stop() + + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + defer newchain.Stop() + + snaptest.verify(t, newchain, blocks) +} + +// restartCrashSnapshotTest is the test type used to test this scenario: +// - have a complete snapshot +// - restart chain +// - insert more blocks with enabling the snapshot +// - commit the snapshot +// - crash +// - restart again +type restartCrashSnapshotTest struct { + snapshotTestBasic + newBlocks int +} + +func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { + // It's hard to follow the test case, visualize the input + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // fmt.Println(tt.dump()) + chain, blocks := snaptest.prepare(t) + + // Firstly, stop the chain properly, with all snapshot journal + // and state committed. + chain.Stop() + + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}) + newchain.InsertChain(newBlocks) + + // Commit the entire snapshot into the disk if requested. Note only + // (a) snapshot root and (b) snapshot generator will be committed, + // the diff journal is not. + newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) + + // Simulate the blockchain crash + // Don't call chain.Stop here, so that no snapshot + // journal and latest state will be committed + + // Restart the chain after the crash + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + defer newchain.Stop() + + snaptest.verify(t, newchain, blocks) +} + +// wipeCrashSnapshotTest is the test type used to test this scenario: +// - have a complete snapshot +// - restart, insert more blocks without enabling the snapshot +// - restart again with enabling the snapshot +// - crash +type wipeCrashSnapshotTest struct { + snapshotTestBasic + newBlocks int +} + +func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { + // It's hard to follow the test case, visualize the input + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // fmt.Println(tt.dump()) + chain, blocks := snaptest.prepare(t) + + // Firstly, stop the chain properly, with all snapshot journal + // and state committed. + chain.Stop() + + config := &CacheConfig{ + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 0, + } + newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}) + newchain.InsertChain(newBlocks) + newchain.Stop() + + // Restart the chain, the wiper should starts working + config = &CacheConfig{ + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 256, + SnapshotWait: false, // Don't wait rebuild + } + newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + // Simulate the blockchain crash. + + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("Failed to recreate chain: %v", err) + } + snaptest.verify(t, newchain, blocks) +} + // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot // journal will be persisted correctly. In this case no snapshot recovery is // required. @@ -129,20 +483,21 @@ func TestRestartWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C8 // Expected snapshot disk : G - testSnapshot(t, &snapshotTest{ - legacy: false, - crash: false, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 8, - expSnapshotBottom: 0, // Initial disk layer built from genesis - }) + test := &snapshotTest{ + snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 8, + expSnapshotBottom: 0, // Initial disk layer built from genesis + }, + } + test.test(t) + test.teardown() } // Tests a Geth restart with valid but "legacy" snapshot. Before the shutdown, @@ -166,20 +521,22 @@ func TestRestartWithLegacySnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C8 // Expected snapshot disk : G - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: false, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 8, - expSnapshotBottom: 0, // Initial disk layer built from genesis - }) + t.Skip("Legacy format testing is not supported") + test := &snapshotTest{ + snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 8, + expSnapshotBottom: 0, // Initial disk layer built from genesis + }, + } + test.test(t) + test.teardown() } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -205,20 +562,21 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - testSnapshot(t, &snapshotTest{ - legacy: false, - crash: true, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }) + test := &crashSnapshotTest{ + snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 0, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -244,20 +602,21 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C2 // Expected snapshot disk : C4 - testSnapshot(t, &snapshotTest{ - legacy: false, - crash: true, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 2, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 2, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }) + test := &crashSnapshotTest{ + snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 2, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 2, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() } // Tests a Geth was crashed and restarts with a broken snapshot. In this case @@ -283,20 +642,21 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - testSnapshot(t, &snapshotTest{ - legacy: false, - crash: true, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 6, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }) + test := &crashSnapshotTest{ + snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 6, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 0, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() } // Tests a Geth was crashed and restarts with a broken and "legacy format" @@ -321,20 +681,22 @@ func TestNoCommitCrashWithLegacySnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : G - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: true, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) - }) + t.Skip("Legacy format testing is not supported") + test := &crashSnapshotTest{ + snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 0, + expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) + }, + } + test.test(t) + test.teardown() } // Tests a Geth was crashed and restarts with a broken and "legacy format" @@ -359,20 +721,22 @@ func TestLowCommitCrashWithLegacySnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C2 // Expected snapshot disk : C2 - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: true, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 2, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 2, - expSnapshotBottom: 2, // Rebuilt snapshot from the latest HEAD - }) + t.Skip("Legacy format testing is not supported") + test := &crashSnapshotTest{ + snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 2, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 2, + expSnapshotBottom: 2, // Rebuilt snapshot from the latest HEAD + }, + } + test.test(t) + test.teardown() } // Tests a Geth was crashed and restarts with a broken and "legacy format" @@ -402,20 +766,22 @@ func TestHighCommitCrashWithLegacySnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : G - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: true, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 6, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) - }) + t.Skip("Legacy format testing is not supported") + test := &crashSnapshotTest{ + snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 6, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 0, + expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) + }, + } + test.test(t) + test.teardown() } // Tests a Geth was running with snapshot enabled. Then restarts without @@ -439,20 +805,22 @@ func TestGappedNewSnapshot(t *testing.T) { // Expected head fast block: C10 // Expected head block : C10 // Expected snapshot disk : C10 - testSnapshot(t, &snapshotTest{ - legacy: false, - crash: false, - gapped: 2, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD - }) + test := &gappedSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD + }, + gapped: 2, + } + test.test(t) + test.teardown() } // Tests a Geth was running with leagcy snapshot enabled. Then restarts @@ -476,20 +844,23 @@ func TestGappedLegacySnapshot(t *testing.T) { // Expected head fast block: C10 // Expected head block : C10 // Expected snapshot disk : C10 - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: false, - gapped: 2, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD - }) + t.Skip("Legacy format testing is not supported") + test := &gappedSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD + }, + gapped: 2, + } + test.test(t) + test.teardown() } // Tests the Geth was running with snapshot enabled and resetHead is applied. @@ -513,20 +884,22 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { // Expected head fast block: C4 // Expected head block : C4 // Expected snapshot disk : G - testSnapshot(t, &snapshotTest{ - legacy: false, - crash: false, - gapped: 0, - setHead: 4, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 4, - expHeadHeader: 4, - expHeadFastBlock: 4, - expHeadBlock: 4, - expSnapshotBottom: 0, // The initial disk layer is built from the genesis - }) + test := &setHeadSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 4, + expHeadHeader: 4, + expHeadFastBlock: 4, + expHeadBlock: 4, + expSnapshotBottom: 0, // The initial disk layer is built from the genesis + }, + setHead: 4, + } + test.test(t) + test.teardown() } // Tests the Geth was running with snapshot(legacy-format) enabled and resetHead @@ -550,20 +923,23 @@ func TestSetHeadWithLegacySnapshot(t *testing.T) { // Expected head fast block: C4 // Expected head block : C4 // Expected snapshot disk : G - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: false, - gapped: 0, - setHead: 4, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 4, - expHeadHeader: 4, - expHeadFastBlock: 4, - expHeadBlock: 4, - expSnapshotBottom: 0, // The initial disk layer is built from the genesis - }) + t.Skip("Legacy format testing is not supported") + test := &setHeadSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 4, + expHeadHeader: 4, + expHeadFastBlock: 4, + expHeadBlock: 4, + expSnapshotBottom: 0, // The initial disk layer is built from the genesis + }, + setHead: 4, + } + test.test(t) + test.teardown() } // Tests the Geth was running with snapshot(legacy-format) enabled and upgrades @@ -589,209 +965,60 @@ func TestRecoverSnapshotFromCrashWithLegacyDiffJournal(t *testing.T) { // Expected head fast block: C10 // Expected head block : C8 // Expected snapshot disk : C10 - testSnapshot(t, &snapshotTest{ - legacy: true, - crash: false, - restartCrash: 2, - gapped: 0, - setHead: 0, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 8, // The persisted state in the first running - expSnapshotBottom: 10, // The persisted disk layer in the second running - }) -} - -func testSnapshot(t *testing.T, tt *snapshotTest) { - // It's hard to follow the test case, visualize the input - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) - // fmt.Println(tt.dump()) - - // Create a temporary persistent database - datadir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Failed to create temporary datadir: %v", err) - } - os.RemoveAll(datadir) - - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") - if err != nil { - t.Fatalf("Failed to create persistent database: %v", err) - } - defer db.Close() // Might double close, should be fine - - // Initialize a fresh chain - var ( - genesis = new(Genesis).MustCommit(db) - engine = ethash.NewFullFaker() - gendb = rawdb.NewMemoryDatabase() - - // Snapshot is enabled, the first snapshot is created from the Genesis. - // The snapshot memory allowance is 256MB, it means no snapshot flush - // will happen during the block insertion. - cacheConfig = defaultCacheConfig - ) - chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to create chain: %v", err) - } - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, gendb, tt.chainBlocks, func(i int, b *BlockGen) {}) - - // Insert the blocks with configured settings. - var breakpoints []uint64 - if tt.commitBlock > tt.snapshotBlock { - breakpoints = append(breakpoints, tt.snapshotBlock, tt.commitBlock) - } else { - breakpoints = append(breakpoints, tt.commitBlock, tt.snapshotBlock) - } - var startPoint uint64 - for _, point := range breakpoints { - if _, err := chain.InsertChain(blocks[startPoint:point]); err != nil { - t.Fatalf("Failed to import canonical chain start: %v", err) - } - startPoint = point - - if tt.commitBlock > 0 && tt.commitBlock == point { - chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil) - } - if tt.snapshotBlock > 0 && tt.snapshotBlock == point { - if tt.legacy { - // Here we commit the snapshot disk root to simulate - // committing the legacy snapshot. - rawdb.WriteSnapshotRoot(db, blocks[point-1].Root()) - } else { - chain.snaps.Cap(blocks[point-1].Root(), 0) - diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root() - if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) { - t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot) - } - } - } - } - if _, err := chain.InsertChain(blocks[startPoint:]); err != nil { - t.Fatalf("Failed to import canonical chain tail: %v", err) + t.Skip("Legacy format testing is not supported") + test := &restartCrashSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + legacy: true, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 8, // The persisted state in the first running + expSnapshotBottom: 10, // The persisted disk layer in the second running + }, + newBlocks: 2, } - // Set the flag for writing legacy journal if necessary - if tt.legacy { - chain.writeLegacyJournal = true - } - // Pull the plug on the database, simulating a hard crash - if tt.crash { - db.Close() - - // Start a new blockchain back up and see where the repair leads us - db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") - if err != nil { - t.Fatalf("Failed to reopen persistent database: %v", err) - } - defer db.Close() - - // The interesting thing is: instead of start the blockchain after - // the crash, we do restart twice here: one after the crash and one - // after the normal stop. It's used to ensure the broken snapshot - // can be detected all the time. - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - chain.Stop() - - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer chain.Stop() - } else if tt.gapped > 0 { - // Insert blocks without enabling snapshot if gapping is required. - chain.Stop() - gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], engine, gendb, tt.gapped, func(i int, b *BlockGen) {}) - - // Insert a few more blocks without enabling snapshot - var cacheConfig = &CacheConfig{ - TrieCleanLimit: 256, - TrieDirtyLimit: 256, - TrieTimeLimit: 5 * time.Minute, - SnapshotLimit: 0, - } - chain, err = NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - chain.InsertChain(gappedBlocks) - chain.Stop() - - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer chain.Stop() - } else if tt.setHead != 0 { - // Rewind the chain if setHead operation is required. - chain.SetHead(tt.setHead) - chain.Stop() - - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer chain.Stop() - } else if tt.restartCrash != 0 { - // Firstly, stop the chain properly, with all snapshot journal - // and state committed. - chain.Stop() - - // Restart chain, forcibly flush the disk layer journal with new format - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], engine, gendb, tt.restartCrash, func(i int, b *BlockGen) {}) - chain, err = NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - chain.InsertChain(newBlocks) - chain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) - - // Simulate the blockchain crash - // Don't call chain.Stop here, so that no snapshot - // journal and latest state will be committed - - // Restart the chain after the crash - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer chain.Stop() - } else { - chain.Stop() - - // Restart the chain normally - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer chain.Stop() - } - - // Iterate over all the remaining blocks and ensure there are no gaps - verifyNoGaps(t, chain, true, blocks) - verifyCutoff(t, chain, true, blocks, tt.expCanonicalBlocks) + test.test(t) + test.teardown() +} - if head := chain.CurrentHeader(); head.Number.Uint64() != tt.expHeadHeader { - t.Errorf("Head header mismatch: have %d, want %d", head.Number, tt.expHeadHeader) - } - if head := chain.CurrentFastBlock(); head.NumberU64() != tt.expHeadFastBlock { - t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadFastBlock) - } - if head := chain.CurrentBlock(); head.NumberU64() != tt.expHeadBlock { - t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadBlock) - } - // Check the disk layer, ensure they are matched - block := chain.GetBlockByNumber(tt.expSnapshotBottom) - if block == nil { - t.Errorf("The correspnding block[%d] of snapshot disk layer is missing", tt.expSnapshotBottom) - } else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) { - t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot()) +// Tests the Geth was running with a complete snapshot and then imports a few +// more new blocks on top without enabling the snapshot. After the restart, +// crash happens. Check everything is ok after the restart. +func TestRecoverSnapshotFromWipingCrash(t *testing.T) { + // Chain: + // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) + // + // Commit: G + // Snapshot: G + // + // SetHead(0) + // + // ------------------------------ + // + // Expected in leveldb: + // G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 + // + // Expected head header : C10 + // Expected head fast block: C10 + // Expected head block : C8 + // Expected snapshot disk : C10 + test := &wipeCrashSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + legacy: false, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, + }, + newBlocks: 2, } + test.test(t) + test.teardown() } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index fcc6b44cb6..2b41dd5513 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -101,18 +101,26 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache i wiper = wipeSnapshot(diskdb, true) } // Create a new disk layer with an initialized state marker at zero - rawdb.WriteSnapshotRoot(diskdb, root) - + var ( + stats = &generatorStats{wiping: wiper, start: time.Now()} + batch = diskdb.NewBatch() + genMarker = []byte{} // Initialized but empty! + ) + rawdb.WriteSnapshotRoot(batch, root) + journalProgress(batch, genMarker, stats) + if err := batch.Write(); err != nil { + log.Crit("Failed to write initialized state marker", "error", err) + } base := &diskLayer{ diskdb: diskdb, triedb: triedb, root: root, cache: fastcache.New(cache * 1024 * 1024), - genMarker: []byte{}, // Initialized but empty! + genMarker: genMarker, genPending: make(chan struct{}), genAbort: make(chan chan *generatorStats), } - go base.generate(&generatorStats{wiping: wiper, start: time.Now()}) + go base.generate(stats) log.Debug("Start snapshot generation", "root", root) return base } @@ -137,10 +145,12 @@ func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorSta panic(err) // Cannot happen, here to catch dev errors } var logstr string - switch len(marker) { - case 0: + switch { + case marker == nil: logstr = "done" - case common.HashLength: + case bytes.Equal(marker, []byte{}): + logstr = "empty" + case len(marker) == common.HashLength: logstr = fmt.Sprintf("%#x", marker) default: logstr = fmt.Sprintf("%#x:%#x", marker[:common.HashLength], marker[common.HashLength:]) @@ -307,13 +317,12 @@ func (dl *diskLayer) generate(stats *generatorStats) { abort <- stats return } - // Snapshot fully generated, set the marker to nil - if batch.ValueSize() > 0 { - // Ensure the generator entry is in sync with the data - journalProgress(batch, nil, stats) + // Snapshot fully generated, set the marker to nil. + // Note even there is nothing to commit, persist the + // generator anyway to mark the snapshot is complete. + journalProgress(batch, nil, stats) + batch.Write() - batch.Write() - } log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots, "storage", stats.storage, "elapsed", common.PrettyDuration(time.Since(stats.start))) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 178ba08902..d7e454cceb 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -441,6 +441,6 @@ func (dl *diffLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) { if err := rlp.Encode(buffer, storage); err != nil { return common.Hash{}, err } - log.Debug("Legacy journalled disk layer", "root", dl.root, "parent", dl.parent.Root()) + log.Debug("Legacy journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } From 24c1e3053b6767add29bf257c7943dc6aa5fa91d Mon Sep 17 00:00:00 2001 From: Alex Mazalov Date: Tue, 19 Jan 2021 08:26:42 +0000 Subject: [PATCH 076/709] cmd/geth: graceful shutdown if disk is full (#22103) Adding warnings of free disk space left and graceful shutdown when there is not enough space left. This also adds a flag datadir.minfreedisk which can be used to set the trigger for low disk space, and setting it to zero disables the check. Co-authored-by: Martin Holst Swende Co-authored-by: Felix Lange --- cmd/geth/main.go | 3 ++- cmd/geth/usage.go | 1 + cmd/utils/cmd.go | 34 +++++++++++++++++++++++++++++- cmd/utils/diskusage.go | 35 +++++++++++++++++++++++++++++++ cmd/utils/diskusage_windows.go | 38 ++++++++++++++++++++++++++++++++++ cmd/utils/flags.go | 4 ++++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 cmd/utils/diskusage.go create mode 100644 cmd/utils/diskusage_windows.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7af24e6523..e577ab370d 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -65,6 +65,7 @@ var ( utils.LegacyBootnodesV5Flag, utils.DataDirFlag, utils.AncientFlag, + utils.MinFreeDiskSpaceFlag, utils.KeyStoreDirFlag, utils.ExternalSignerFlag, utils.NoUSBFlag, @@ -368,7 +369,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend) { debug.Memsize.Add("node", stack) // Start up the node itself - utils.StartNode(stack) + utils.StartNode(ctx, stack) // Unlock any account specifically requested unlockAccounts(ctx, stack) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 78ebb807e1..25accc9b79 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -36,6 +36,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ configFileFlag, utils.DataDirFlag, utils.AncientFlag, + utils.MinFreeDiskSpaceFlag, utils.KeyStoreDirFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 869cf90ea5..4306216892 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -26,17 +26,20 @@ import ( "runtime" "strings" "syscall" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" + "gopkg.in/urfave/cli.v1" ) const ( @@ -63,7 +66,7 @@ func Fatalf(format string, args ...interface{}) { os.Exit(1) } -func StartNode(stack *node.Node) { +func StartNode(ctx *cli.Context, stack *node.Node) { if err := stack.Start(); err != nil { Fatalf("Error starting protocol stack: %v", err) } @@ -71,6 +74,17 @@ func StartNode(stack *node.Node) { sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sigc) + + minFreeDiskSpace := eth.DefaultConfig.TrieDirtyCache + if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) { + minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name) + } else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { + minFreeDiskSpace = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + } + if minFreeDiskSpace > 0 { + go monitorFreeDiskSpace(sigc, stack.InstanceDir(), uint64(minFreeDiskSpace)*1024*1024) + } + <-sigc log.Info("Got interrupt, shutting down...") go stack.Close() @@ -85,6 +99,24 @@ func StartNode(stack *node.Node) { }() } +func monitorFreeDiskSpace(sigc chan os.Signal, path string, freeDiskSpaceCritical uint64) { + for { + freeSpace, err := getFreeDiskSpace(path) + if err != nil { + log.Warn("Failed to get free disk space", "path", path, "err", err) + break + } + if freeSpace < freeDiskSpaceCritical { + log.Error("Low disk space. Gracefully shutting down Geth to prevent database corruption.", "available", common.StorageSize(freeSpace)) + sigc <- syscall.SIGTERM + break + } else if freeSpace < 2*freeDiskSpaceCritical { + log.Warn("Disk space is running low. Geth will shutdown if disk space runs below critical level.", "available", common.StorageSize(freeSpace), "critical_level", common.StorageSize(freeDiskSpaceCritical)) + } + time.Sleep(60 * time.Second) + } +} + func ImportChain(chain *core.BlockChain, fn string) error { // Watch for Ctrl-C while the import is running. // If a signal is received, the import will stop at the next batch. diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go new file mode 100644 index 0000000000..a822118a39 --- /dev/null +++ b/cmd/utils/diskusage.go @@ -0,0 +1,35 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !windows + +package utils + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func getFreeDiskSpace(path string) (uint64, error) { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return 0, fmt.Errorf("failed to call Statfs: %v", err) + } + + // Available blocks * size per block = available space in bytes + return stat.Bavail * uint64(stat.Bsize), nil +} diff --git a/cmd/utils/diskusage_windows.go b/cmd/utils/diskusage_windows.go new file mode 100644 index 0000000000..9bf7740b99 --- /dev/null +++ b/cmd/utils/diskusage_windows.go @@ -0,0 +1,38 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package utils + +import ( + "fmt" + + "golang.org/x/sys/windows" +) + +func getFreeDiskSpace(path string) (uint64, error) { + + cwd, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, fmt.Errorf("failed to call UTF16PtrFromString: %v", err) + } + + var freeBytesAvailableToCaller, totalNumberOfBytes, totalNumberOfFreeBytes uint64 + if err := windows.GetDiskFreeSpaceEx(cwd, &freeBytesAvailableToCaller, &totalNumberOfBytes, &totalNumberOfFreeBytes); err != nil { + return 0, fmt.Errorf("failed to call GetDiskFreeSpaceEx: %v", err) + } + + return freeBytesAvailableToCaller, nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 688e25618d..df036cbbf0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -113,6 +113,10 @@ var ( Name: "datadir.ancient", Usage: "Data directory for ancient chain segments (default = inside chaindata)", } + MinFreeDiskSpaceFlag = DirectoryFlag{ + Name: "datadir.minfreedisk", + Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", + } KeyStoreDirFlag = DirectoryFlag{ Name: "keystore", Usage: "Directory for the keystore (default = inside the datadir)", From 45cb1a580abad0d4e8caa1c8b7dfacd5ef3d27bc Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 19 Jan 2021 17:52:45 +0800 Subject: [PATCH 077/709] eth, les: add new config field SyncFromCheckpoint (#22123) This PR introduces a new config field SyncFromCheckpoint for light client. In some special scenarios, it's required to start synchronization from some arbitrary checkpoint or even from the scratch. So this PR offers this flexibility to users so that the synchronization start point can be configured. There are two relevant configs: SyncFromCheckpoint and Checkpoint. - If the SyncFromCheckpoint is true, the light client will try to sync from the specified checkpoint. - If the Checkpoint is not configured, then the light client will sync from the scratch(from the latest header if the database is not empty) Additional notes: these two configs are not visible in the CLI flags but only accessable in the config file. Example Usage: [Eth] SyncFromCheckpoint = true [Eth.Checkpoint] SectionIndex = 100 SectionHead = "0xabc" CHTRoot = "0xabc" BloomRoot = "0xabc" PS. Historical checkpoint can be retrieved from the synced full node or light client via les_getCheckpoint API. --- eth/config.go | 11 +-- eth/gen_config.go | 12 ++++ les/client_handler.go | 9 ++- les/commons.go | 6 +- les/sync.go | 52 ++++++++++----- les/sync_test.go | 152 ++++++++++++++++++++++++++++++++++++++++-- les/test_helper.go | 2 +- 7 files changed, 209 insertions(+), 35 deletions(-) diff --git a/eth/config.go b/eth/config.go index 77d03e9569..446467d364 100644 --- a/eth/config.go +++ b/eth/config.go @@ -127,11 +127,12 @@ type Config struct { Whitelist map[uint64]common.Hash `toml:"-"` // Light client options - LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests - LightIngress int `toml:",omitempty"` // Incoming bandwidth limit for light servers - LightEgress int `toml:",omitempty"` // Outgoing bandwidth limit for light servers - LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - LightNoPrune bool `toml:",omitempty"` // Whether to disable light chain pruning + LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests + LightIngress int `toml:",omitempty"` // Incoming bandwidth limit for light servers + LightEgress int `toml:",omitempty"` // Outgoing bandwidth limit for light servers + LightPeers int `toml:",omitempty"` // Maximum number of LES client peers + LightNoPrune bool `toml:",omitempty"` // Whether to disable light chain pruning + SyncFromCheckpoint bool `toml:",omitempty"` // Whether to sync the header chain from the configured checkpoint // Ultra Light client options UltraLightServers []string `toml:",omitempty"` // List of trusted ultra light servers diff --git a/eth/gen_config.go b/eth/gen_config.go index dd04635eee..e68b29ce5e 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -21,6 +21,7 @@ func (c Config) MarshalTOML() (interface{}, error) { NetworkId uint64 SyncMode downloader.SyncMode EthDiscoveryURLs []string + SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` @@ -30,6 +31,7 @@ func (c Config) MarshalTOML() (interface{}, error) { LightEgress int `toml:",omitempty"` LightPeers int `toml:",omitempty"` LightNoPrune bool `toml:",omitempty"` + SyncFromCheckpoint bool `toml:",omitempty"` UltraLightServers []string `toml:",omitempty"` UltraLightFraction int `toml:",omitempty"` UltraLightOnlyAnnounce bool `toml:",omitempty"` @@ -62,6 +64,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode enc.EthDiscoveryURLs = c.EthDiscoveryURLs + enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit @@ -71,6 +74,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.LightEgress = c.LightEgress enc.LightPeers = c.LightPeers enc.LightNoPrune = c.LightNoPrune + enc.SyncFromCheckpoint = c.SyncFromCheckpoint enc.UltraLightServers = c.UltraLightServers enc.UltraLightFraction = c.UltraLightFraction enc.UltraLightOnlyAnnounce = c.UltraLightOnlyAnnounce @@ -107,6 +111,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NetworkId *uint64 SyncMode *downloader.SyncMode EthDiscoveryURLs []string + SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` @@ -116,6 +121,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { LightEgress *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` LightNoPrune *bool `toml:",omitempty"` + SyncFromCheckpoint *bool `toml:",omitempty"` UltraLightServers []string `toml:",omitempty"` UltraLightFraction *int `toml:",omitempty"` UltraLightOnlyAnnounce *bool `toml:",omitempty"` @@ -159,6 +165,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.EthDiscoveryURLs != nil { c.EthDiscoveryURLs = dec.EthDiscoveryURLs } + if dec.SnapDiscoveryURLs != nil { + c.SnapDiscoveryURLs = dec.SnapDiscoveryURLs + } if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } @@ -186,6 +195,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightNoPrune != nil { c.LightNoPrune = *dec.LightNoPrune } + if dec.SyncFromCheckpoint != nil { + c.SyncFromCheckpoint = *dec.SyncFromCheckpoint + } if dec.UltraLightServers != nil { c.UltraLightServers = dec.UltraLightServers } diff --git a/les/client_handler.go b/les/client_handler.go index 6de5766961..6cd786cda0 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -44,9 +44,12 @@ type clientHandler struct { downloader *downloader.Downloader backend *LightEthereum - closeCh chan struct{} - wg sync.WaitGroup // WaitGroup used to track all connected peers. - syncDone func() // Test hooks when syncing is done. + closeCh chan struct{} + wg sync.WaitGroup // WaitGroup used to track all connected peers. + + // Hooks used in the testing + syncStart func(header *types.Header) // Hook called when the syncing is started + syncEnd func(header *types.Header) // Hook called when the syncing is done } func newClientHandler(ulcServers []string, ulcFraction int, checkpoint *params.TrustedCheckpoint, backend *LightEthereum) *clientHandler { diff --git a/les/commons.go b/les/commons.go index 003e196d2b..8de1057d26 100644 --- a/les/commons.go +++ b/les/commons.go @@ -157,17 +157,17 @@ func (c *lesCommons) setupOracle(node *node.Node, genesis common.Hash, ethconfig config = params.CheckpointOracles[genesis] } if config == nil { - log.Info("Checkpoint registrar is not enabled") + log.Info("Checkpoint oracle is not enabled") return nil } if config.Address == (common.Address{}) || uint64(len(config.Signers)) < config.Threshold { - log.Warn("Invalid checkpoint registrar config") + log.Warn("Invalid checkpoint oracle config") return nil } oracle := checkpointoracle.New(config, c.localCheckpoint) rpcClient, _ := node.Attach() client := ethclient.NewClient(rpcClient) oracle.Start(client) - log.Info("Configured checkpoint registrar", "address", config.Address, "signers", len(config.Signers), "threshold", config.Threshold) + log.Info("Configured checkpoint oracle", "address", config.Address, "signers", len(config.Signers), "threshold", config.Threshold) return oracle } diff --git a/les/sync.go b/les/sync.go index ad3a0e0f3c..fa5ef4ff82 100644 --- a/les/sync.go +++ b/les/sync.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" ) var errInvalidCheckpoint = errors.New("invalid advertised checkpoint") @@ -98,22 +99,33 @@ func (h *clientHandler) synchronise(peer *serverPeer) { if currentTd != nil && peer.Td().Cmp(currentTd) < 0 { return } - // Recap the checkpoint. - // - // The light client may be connected to several different versions of the server. - // (1) Old version server which can not provide stable checkpoint in the handshake packet. - // => Use hardcoded checkpoint or empty checkpoint - // (2) New version server but simple checkpoint syncing is not enabled(e.g. mainnet, new testnet or private network) - // => Use hardcoded checkpoint or empty checkpoint - // (3) New version server but the provided stable checkpoint is even lower than the hardcoded one. - // => Use hardcoded checkpoint + // Recap the checkpoint. The light client may be connected to several different + // versions of the server. + // (1) Old version server which can not provide stable checkpoint in the + // handshake packet. + // => Use local checkpoint or empty checkpoint + // (2) New version server but simple checkpoint syncing is not enabled + // (e.g. mainnet, new testnet or private network) + // => Use local checkpoint or empty checkpoint + // (3) New version server but the provided stable checkpoint is even lower + // than the local one. + // => Use local checkpoint // (4) New version server with valid and higher stable checkpoint // => Use provided checkpoint - var checkpoint = &peer.checkpoint - var hardcoded bool + var ( + local bool + checkpoint = &peer.checkpoint + ) if h.checkpoint != nil && h.checkpoint.SectionIndex >= peer.checkpoint.SectionIndex { - checkpoint = h.checkpoint // Use the hardcoded one. - hardcoded = true + local, checkpoint = true, h.checkpoint + } + // Replace the checkpoint with locally configured one If it's required by + // users. Nil checkpoint means synchronization from the scratch. + if h.backend.config.SyncFromCheckpoint { + local, checkpoint = true, h.backend.config.Checkpoint + if h.backend.config.Checkpoint == nil { + checkpoint = ¶ms.TrustedCheckpoint{} + } } // Determine whether we should run checkpoint syncing or normal light syncing. // @@ -121,7 +133,7 @@ func (h *clientHandler) synchronise(peer *serverPeer) { // // 1. The checkpoint is empty // 2. The latest head block of the local chain is above the checkpoint. - // 3. The checkpoint is hardcoded(recap with local hardcoded checkpoint) + // 3. The checkpoint is local(replaced with local checkpoint) // 4. For some networks the checkpoint syncing is not activated. mode := checkpointSync switch { @@ -131,7 +143,7 @@ func (h *clientHandler) synchronise(peer *serverPeer) { case latest.Number.Uint64() >= (checkpoint.SectionIndex+1)*h.backend.iConfig.ChtSize-1: mode = lightSync log.Debug("Disable checkpoint syncing", "reason", "local chain beyond the checkpoint") - case hardcoded: + case local: mode = legacyCheckpointSync log.Debug("Disable checkpoint syncing", "reason", "checkpoint is hardcoded") case h.backend.oracle == nil || !h.backend.oracle.IsRunning(): @@ -143,12 +155,14 @@ func (h *clientHandler) synchronise(peer *serverPeer) { } log.Debug("Disable checkpoint syncing", "reason", "checkpoint syncing is not activated") } + // Notify testing framework if syncing has completed(for testing purpose). defer func() { - if h.syncDone != nil { - h.syncDone() + if h.syncEnd != nil { + h.syncEnd(h.backend.blockchain.CurrentHeader()) } }() + start := time.Now() if mode == checkpointSync || mode == legacyCheckpointSync { // Validate the advertised checkpoint @@ -177,6 +191,10 @@ func (h *clientHandler) synchronise(peer *serverPeer) { return } } + + if h.syncStart != nil { + h.syncStart(h.backend.blockchain.CurrentHeader()) + } // Fetch the remaining block headers based on the current chain header. if err := h.downloader.Synchronise(peer.id, peer.Head(), peer.Td(), downloader.LightSync); err != nil { log.Debug("Synchronise failed", "reason", err) diff --git a/les/sync_test.go b/les/sync_test.go index 2eb0f88bf9..64e7283663 100644 --- a/les/sync_test.go +++ b/les/sync_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/params" @@ -53,7 +54,7 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { time.Sleep(10 * time.Millisecond) } } - // Generate 128+1 blocks (totally 1 CHT sections) + // Generate 128+1 blocks (totally 1 CHT section) server, client, tearDown := newClientServerEnv(t, int(config.ChtSize+config.ChtConfirms), protocol, waitIndexers, nil, 0, false, false, true) defer tearDown() @@ -100,8 +101,7 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { } done := make(chan error) - client.handler.syncDone = func() { - header := client.handler.backend.blockchain.CurrentHeader() + client.handler.syncEnd = func(header *types.Header) { if header.Number.Uint64() == expected { done <- nil } else { @@ -144,7 +144,7 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { time.Sleep(10 * time.Millisecond) } } - // Generate 512+4 blocks (totally 1 CHT sections) + // Generate 128+1 blocks (totally 1 CHT section) server, client, tearDown := newClientServerEnv(t, int(config.ChtSize+config.ChtConfirms), 3, waitIndexers, nil, 0, false, false, true) defer tearDown() @@ -198,8 +198,7 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { } done := make(chan error) - client.handler.syncDone = func() { - header := client.handler.backend.blockchain.CurrentHeader() + client.handler.syncEnd = func(header *types.Header) { if header.Number.Uint64() == expected { done <- nil } else { @@ -220,3 +219,144 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { t.Error("checkpoint syncing timeout") } } + +func TestSyncFromConfiguredCheckpoint(t *testing.T) { + config := light.TestServerIndexerConfig + + waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + cs, _, _ := cIndexer.Sections() + bts, _, _ := btIndexer.Sections() + if cs >= 2 && bts >= 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + } + // Generate 256+1 blocks (totally 2 CHT sections) + server, client, tearDown := newClientServerEnv(t, int(2*config.ChtSize+config.ChtConfirms), 3, waitIndexers, nil, 0, false, false, true) + defer tearDown() + + // Configure the local checkpoint(the first section) + head := server.handler.blockchain.GetHeaderByNumber(config.ChtSize - 1).Hash() + cp := ¶ms.TrustedCheckpoint{ + SectionIndex: 0, + SectionHead: head, + CHTRoot: light.GetChtRoot(server.db, 0, head), + BloomRoot: light.GetBloomTrieRoot(server.db, 0, head), + } + client.handler.backend.config.SyncFromCheckpoint = true + client.handler.backend.config.Checkpoint = cp + client.handler.checkpoint = cp + client.handler.backend.blockchain.AddTrustedCheckpoint(cp) + + var ( + start = make(chan error, 1) + end = make(chan error, 1) + expectStart = config.ChtSize - 1 + expectEnd = 2*config.ChtSize + config.ChtConfirms + ) + client.handler.syncStart = func(header *types.Header) { + if header.Number.Uint64() == expectStart { + start <- nil + } else { + start <- fmt.Errorf("blockchain length mismatch, want %d, got %d", expectStart, header.Number) + } + } + client.handler.syncEnd = func(header *types.Header) { + if header.Number.Uint64() == expectEnd { + end <- nil + } else { + end <- fmt.Errorf("blockchain length mismatch, want %d, got %d", expectEnd, header.Number) + } + } + // Create connected peer pair. + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + t.Fatalf("Failed to connect testing peers %v", err) + } + + select { + case err := <-start: + if err != nil { + t.Error("sync failed", err) + } + return + case <-time.NewTimer(10 * time.Second).C: + t.Error("checkpoint syncing timeout") + } + + select { + case err := <-end: + if err != nil { + t.Error("sync failed", err) + } + return + case <-time.NewTimer(10 * time.Second).C: + t.Error("checkpoint syncing timeout") + } +} + +func TestSyncAll(t *testing.T) { + config := light.TestServerIndexerConfig + + waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + cs, _, _ := cIndexer.Sections() + bts, _, _ := btIndexer.Sections() + if cs >= 2 && bts >= 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + } + // Generate 256+1 blocks (totally 2 CHT sections) + server, client, tearDown := newClientServerEnv(t, int(2*config.ChtSize+config.ChtConfirms), 3, waitIndexers, nil, 0, false, false, true) + defer tearDown() + + client.handler.backend.config.SyncFromCheckpoint = true + + var ( + start = make(chan error, 1) + end = make(chan error, 1) + expectStart = uint64(0) + expectEnd = 2*config.ChtSize + config.ChtConfirms + ) + client.handler.syncStart = func(header *types.Header) { + if header.Number.Uint64() == expectStart { + start <- nil + } else { + start <- fmt.Errorf("blockchain length mismatch, want %d, got %d", expectStart, header.Number) + } + } + client.handler.syncEnd = func(header *types.Header) { + if header.Number.Uint64() == expectEnd { + end <- nil + } else { + end <- fmt.Errorf("blockchain length mismatch, want %d, got %d", expectEnd, header.Number) + } + } + // Create connected peer pair. + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + t.Fatalf("Failed to connect testing peers %v", err) + } + + select { + case err := <-start: + if err != nil { + t.Error("sync failed", err) + } + return + case <-time.NewTimer(10 * time.Second).C: + t.Error("checkpoint syncing timeout") + } + + select { + case err := <-end: + if err != nil { + t.Error("sync failed", err) + } + return + case <-time.NewTimer(10 * time.Second).C: + t.Error("checkpoint syncing timeout") + } +} diff --git a/les/test_helper.go b/les/test_helper.go index d108a8dace..04482ba68e 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -541,7 +541,7 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, callback indexer ) if connect { done := make(chan struct{}) - client.syncDone = func() { close(done) } + client.syncEnd = func(_ *types.Header) { close(done) } cpeer, speer, err = newTestPeerPair("peer", protocol, server, client) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) From d1301eb0df07219d576c56400128c56f4f65beab Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 20 Jan 2021 18:21:13 +0100 Subject: [PATCH 078/709] oss-fuzz: fix abi fuzzer (#22199) --- oss-fuzz.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 7ae4d77b29..860ce0952f 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -100,7 +100,7 @@ compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty -compile_fuzzertests/fuzzers/abi Fuzz fuzzAbi +compile_fuzzer tests/fuzzers/abi Fuzz fuzzAbi compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul From 7da8f75d5bab50ad5477d50bc96d383e5c8359b8 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 21 Jan 2021 02:34:21 +0800 Subject: [PATCH 079/709] go.mod: upgrade golang-lru (#22134) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 57c15a34f6..d630dc6996 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 - github.com/hashicorp/golang-lru v0.5.4 + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 github.com/huin/goupnp v1.0.0 diff --git a/go.sum b/go.sum index d063cd8f08..f8939df2ba 100644 --- a/go.sum +++ b/go.sum @@ -105,8 +105,8 @@ github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDF github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= From 81bf9f97c9edf0169e47b5b700715e2eae58e08a Mon Sep 17 00:00:00 2001 From: meowsbits Date: Wed, 20 Jan 2021 15:45:01 -0600 Subject: [PATCH 080/709] downloader: extract findAncestor search functions (#21744) This is a simple refactoring, extracting common ancestor negotiation logic to named function --- eth/downloader/downloader.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 315354ea99..31c1cb47c3 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -90,6 +90,7 @@ var ( errCanceled = errors.New("syncing canceled (requested)") errNoSyncActive = errors.New("no sync active") errTooOld = errors.New("peer's protocol version too old") + errNoAncestorFound = errors.New("no common ancestor found") ) type Downloader struct { @@ -814,6 +815,26 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) } } + ancestor, err := d.findAncestorSpanSearch(p, mode, remoteHeight, localHeight, floor) + if err == nil { + return ancestor, nil + } + // The returned error was not nil. + // If the error returned does not reflect that a common ancestor was not found, return it. + // If the error reflects that a common ancestor was not found, continue to binary search, + // where the error value will be reassigned. + if !errors.Is(err, errNoAncestorFound) { + return 0, err + } + + ancestor, err = d.findAncestorBinarySearch(p, mode, remoteHeight, floor) + if err != nil { + return 0, err + } + return ancestor, nil +} + +func (d *Downloader) findAncestorSpanSearch(p *peerConnection, mode SyncMode, remoteHeight, localHeight uint64, floor int64) (commonAncestor uint64, err error) { from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip) @@ -894,6 +915,12 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) p.log.Debug("Found common ancestor", "number", number, "hash", hash) return number, nil } + return 0, errNoAncestorFound +} + +func (d *Downloader) findAncestorBinarySearch(p *peerConnection, mode SyncMode, remoteHeight uint64, floor int64) (commonAncestor uint64, err error) { + hash := common.Hash{} + // Ancestor not found, we need to binary search over our chain start, end := uint64(0), remoteHeight if floor > 0 { From 1e1865b73f6b0d2fef656d2f37e20e41d13a5ef0 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 5 Feb 2020 13:12:09 +0100 Subject: [PATCH 081/709] core: implement background trie prefetcher Squashed from the following commits: core/state: lazily init snapshot storage map core/state: fix flawed meter on storage reads core/state: make statedb/stateobjects reuse a hasher core/blockchain, core/state: implement new trie prefetcher core: make trie prefetcher deliver tries to statedb core/state: refactor trie_prefetcher, export storage tries blockchain: re-enable the next-block-prefetcher state: remove panics in trie prefetcher core/state/trie_prefetcher: address some review concerns sq --- core/blockchain.go | 27 +++- core/state/database.go | 12 +- core/state/state_object.go | 72 +++++++--- core/state/statedb.go | 45 +++++- core/state/trie_prefetcher.go | 249 ++++++++++++++++++++++++++++++++++ crypto/crypto.go | 17 ++- crypto/crypto_test.go | 7 + 7 files changed, 395 insertions(+), 34 deletions(-) create mode 100644 core/state/trie_prefetcher.go diff --git a/core/blockchain.go b/core/blockchain.go index b8f483b85e..ccb99bded5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -201,11 +201,12 @@ type BlockChain struct { running int32 // 0 if chain is running, 1 when stopped procInterrupt int32 // interrupt signaler for block processing - engine consensus.Engine - validator Validator // Block and state validator interface - prefetcher Prefetcher // Block state prefetcher interface - processor Processor // Block transaction processor interface - vmConfig vm.Config + engine consensus.Engine + validator Validator // Block and state validator interface + triePrefetcher *state.TriePrefetcher // Trie prefetcher interface + prefetcher Prefetcher + processor Processor // Block transaction processor interface + vmConfig vm.Config shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. @@ -249,6 +250,15 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) + tp := state.NewTriePrefetcher(bc.stateCache) + + bc.wg.Add(1) + go func() { + tp.Loop() + bc.wg.Done() + }() + bc.triePrefetcher = tp + bc.processor = NewStateProcessor(chainConfig, bc, engine) var err error @@ -991,6 +1001,9 @@ func (bc *BlockChain) Stop() { bc.scope.Close() close(bc.quit) bc.StopInsert() + if bc.triePrefetcher != nil { + bc.triePrefetcher.Close() + } bc.wg.Wait() // Ensure that the entirety of the state snapshot is journalled to disk. @@ -1857,6 +1870,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1) } statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) + statedb.UsePrefetcher(bc.triePrefetcher) if err != nil { return it.index, err } @@ -1891,8 +1905,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete, we can mark them snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete, we can mark them - - triehash := statedb.AccountHashes + statedb.StorageHashes // Save to not double count in validation + triehash := statedb.AccountHashes + statedb.StorageHashes // Save to not double count in validation trieproc := statedb.SnapshotAccountReads + statedb.AccountReads + statedb.AccountUpdates trieproc += statedb.SnapshotStorageReads + statedb.StorageReads + statedb.StorageUpdates diff --git a/core/state/database.go b/core/state/database.go index 83f7b2a839..1a06e33409 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -129,12 +129,20 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - return trie.NewSecure(root, db.db) + tr, err := trie.NewSecure(root, db.db) + if err != nil { + return nil, err + } + return tr, nil } // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - return trie.NewSecure(root, db.db) + tr, err := trie.NewSecure(root, db.db) + if err != nil { + return nil, err + } + return tr, nil } // CopyTrie returns an independent copy of the given trie. diff --git a/core/state/state_object.go b/core/state/state_object.go index d0d3b4513e..43c5074d92 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -157,11 +157,20 @@ func (s *stateObject) touch() { func (s *stateObject) getTrie(db Database) Trie { if s.trie == nil { - var err error - s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root) - if err != nil { - s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{}) - s.setError(fmt.Errorf("can't create storage trie: %v", err)) + // Try fetching from prefetcher first + // We don't prefetch empty tries + if s.data.Root != emptyRoot && s.db.prefetcher != nil { + // When the miner is creating the pending state, there is no + // prefetcher + s.trie = s.db.prefetcher.GetTrie(s.data.Root) + } + if s.trie == nil { + var err error + s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root) + if err != nil { + s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{}) + s.setError(fmt.Errorf("can't create storage trie: %v", err)) + } } } return s.trie @@ -197,12 +206,24 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has } // If no live objects are available, attempt to use snapshots var ( - enc []byte - err error + enc []byte + err error + meter *time.Duration ) + readStart := time.Now() + if metrics.EnabledExpensive { + // If the snap is 'under construction', the first lookup may fail. If that + // happens, we don't want to double-count the time elapsed. Thus this + // dance with the metering. + defer func() { + if meter != nil { + *meter += time.Since(readStart) + } + }() + } if s.db.snap != nil { if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.SnapshotStorageReads += time.Since(start) }(time.Now()) + meter = &s.db.SnapshotStorageReads } // If the object was destructed in *this* block (and potentially resurrected), // the storage has been cleared out, and we should *not* consult the previous @@ -217,8 +238,14 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has } // If snapshot unavailable or reading from it failed, load from the database if s.db.snap == nil || err != nil { + if meter != nil { + // If we already spent time checking the snapshot, account for it + // and reset the readStart + *meter += time.Since(readStart) + readStart = time.Now() + } if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageReads += time.Since(start) }(time.Now()) + meter = &s.db.StorageReads } if enc, err = s.getTrie(db).TryGet(key.Bytes()); err != nil { s.setError(err) @@ -283,8 +310,13 @@ func (s *stateObject) setState(key, value common.Hash) { // finalise moves all dirty storage slots into the pending area to be hashed or // committed later. It is invoked at the end of every transaction. func (s *stateObject) finalise() { + trieChanges := make([]common.Hash, 0, len(s.dirtyStorage)) for key, value := range s.dirtyStorage { s.pendingStorage[key] = value + trieChanges = append(trieChanges, key) + } + if len(trieChanges) > 0 && s.db.prefetcher != nil && s.data.Root != emptyRoot { + s.db.prefetcher.PrefetchStorage(s.data.Root, trieChanges) } if len(s.dirtyStorage) > 0 { s.dirtyStorage = make(Storage) @@ -303,18 +335,11 @@ func (s *stateObject) updateTrie(db Database) Trie { if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) } - // Retrieve the snapshot storage map for the object + // The snapshot storage map for the object var storage map[common.Hash][]byte - if s.db.snap != nil { - // Retrieve the old storage map, if available, create a new one otherwise - storage = s.db.snapStorage[s.addrHash] - if storage == nil { - storage = make(map[common.Hash][]byte) - s.db.snapStorage[s.addrHash] = storage - } - } // Insert all the pending updates into the trie tr := s.getTrie(db) + hasher := s.db.hasher for key, value := range s.pendingStorage { // Skip noop changes, persist actual changes if value == s.originStorage[key] { @@ -331,8 +356,15 @@ func (s *stateObject) updateTrie(db Database) Trie { s.setError(tr.TryUpdate(key[:], v)) } // If state snapshotting is active, cache the data til commit - if storage != nil { - storage[crypto.Keccak256Hash(key[:])] = v // v will be nil if value is 0x00 + if s.db.snap != nil { + if storage == nil { + // Retrieve the old storage map, if available, create a new one otherwise + if storage = s.db.snapStorage[s.addrHash]; storage == nil { + storage = make(map[common.Hash][]byte) + s.db.snapStorage[s.addrHash] = storage + } + } + storage[crypto.HashData(hasher, key[:])] = v // v will be nil if value is 0x00 } } if len(s.pendingStorage) > 0 { diff --git a/core/state/statedb.go b/core/state/statedb.go index a9d1de2e06..ce50962e8b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -62,8 +62,11 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - trie Trie + db Database + prefetcher *TriePrefetcher + originalRoot common.Hash // The pre-state root, before any changes were made + trie Trie + hasher crypto.KeccakState snaps *snapshot.Tree snap snapshot.Snapshot @@ -125,6 +128,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) sdb := &StateDB{ db: db, trie: tr, + originalRoot: root, snaps: snaps, stateObjects: make(map[common.Address]*stateObject), stateObjectsPending: make(map[common.Address]struct{}), @@ -133,6 +137,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) preimages: make(map[common.Hash][]byte), journal: newJournal(), accessList: newAccessList(), + hasher: crypto.NewKeccakState(), } if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { @@ -144,6 +149,13 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) return sdb, nil } +func (s *StateDB) UsePrefetcher(prefetcher *TriePrefetcher) { + if prefetcher != nil { + s.prefetcher = prefetcher + s.prefetcher.Resume(s.originalRoot) + } +} + // setError remembers the first non-nil error it is called with. func (s *StateDB) setError(err error) { if s.dbErr == nil { @@ -532,7 +544,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { defer func(start time.Time) { s.SnapshotAccountReads += time.Since(start) }(time.Now()) } var acc *snapshot.Account - if acc, err = s.snap.Account(crypto.Keccak256Hash(addr.Bytes())); err == nil { + if acc, err = s.snap.Account(crypto.HashData(s.hasher, addr.Bytes())); err == nil { if acc == nil { return nil } @@ -675,6 +687,7 @@ func (s *StateDB) Copy() *StateDB { logSize: s.logSize, preimages: make(map[common.Hash][]byte, len(s.preimages)), journal: newJournal(), + hasher: crypto.NewKeccakState(), } // Copy the dirty states, logs, and preimages for addr := range s.journal.dirties { @@ -760,6 +773,7 @@ func (s *StateDB) GetRefund() uint64 { // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { + var addressesToPrefetch []common.Address for addr := range s.journal.dirties { obj, exist := s.stateObjects[addr] if !exist { @@ -788,7 +802,17 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { } s.stateObjectsPending[addr] = struct{}{} s.stateObjectsDirty[addr] = struct{}{} + // At this point, also ship the address off to the precacher. The precacher + // will start loading tries, and when the change is eventually committed, + // the commit-phase will be a lot faster + if s.prefetcher != nil { + addressesToPrefetch = append(addressesToPrefetch, addr) + } + } + if s.prefetcher != nil { + s.prefetcher.PrefetchAddresses(addressesToPrefetch) } + // Invalidate journal because reverting across transactions is not allowed. s.clearJournalAndRefund() } @@ -800,6 +824,21 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Finalise all the dirty storage states and write them into the tries s.Finalise(deleteEmptyObjects) + // Now we're about to start to write changes to the trie. The trie is so + // far _untouched_. We can check with the prefetcher, if it can give us + // a trie which has the same root, but also has some content loaded into it. + // If so, use that one instead. + if s.prefetcher != nil { + s.prefetcher.Pause() + // We only want to do this _once_, if someone calls IntermediateRoot again, + // we shouldn't fetch the trie again + if s.originalRoot != (common.Hash{}) { + if trie := s.prefetcher.GetTrie(s.originalRoot); trie != nil { + s.trie = trie + } + s.originalRoot = common.Hash{} + } + } for addr := range s.stateObjectsPending { obj := s.stateObjects[addr] if obj.deleted { diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go new file mode 100644 index 0000000000..8a1aab3253 --- /dev/null +++ b/core/state/trie_prefetcher.go @@ -0,0 +1,249 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + // trieDeliveryMeter counts how many times the prefetcher was unable to supply + // the statedb with a prefilled trie. This meter should be zero -- if it's not, that + // needs to be investigated + trieDeliveryMissMeter = metrics.NewRegisteredMeter("trie/prefetch/deliverymiss", nil) + + triePrefetchFetchMeter = metrics.NewRegisteredMeter("trie/prefetch/fetch", nil) + triePrefetchSkipMeter = metrics.NewRegisteredMeter("trie/prefetch/skip", nil) + triePrefetchDropMeter = metrics.NewRegisteredMeter("trie/prefetch/drop", nil) +) + +// TriePrefetcher is an active prefetcher, which receives accounts or storage +// items on two channels, and does trie-loading of the items. +// The goal is to get as much useful content into the caches as possible +type TriePrefetcher struct { + requestCh chan (fetchRequest) // Chan to receive requests for data to fetch + cmdCh chan (*cmd) // Chan to control activity, pause/new root + quitCh chan (struct{}) + deliveryCh chan (struct{}) + db Database + + paused bool + + storageTries map[common.Hash]Trie + accountTrie Trie + accountTrieRoot common.Hash +} + +func NewTriePrefetcher(db Database) *TriePrefetcher { + return &TriePrefetcher{ + requestCh: make(chan fetchRequest, 200), + cmdCh: make(chan *cmd), + quitCh: make(chan struct{}), + deliveryCh: make(chan struct{}), + db: db, + } +} + +type cmd struct { + root common.Hash +} + +type fetchRequest struct { + slots []common.Hash + storageRoot *common.Hash + addresses []common.Address +} + +func (p *TriePrefetcher) Loop() { + var ( + accountTrieRoot common.Hash + accountTrie Trie + storageTries map[common.Hash]Trie + + err error + // Some tracking of performance + skipped int64 + fetched int64 + + paused = true + ) + // The prefetcher loop has two distinct phases: + // 1: Paused: when in this state, the accumulated tries are accessible to outside + // callers. + // 2: Active prefetching, awaiting slots and accounts to prefetch + for { + select { + case <-p.quitCh: + return + case cmd := <-p.cmdCh: + // Clear out any old requests + drain: + for { + select { + case req := <-p.requestCh: + if req.slots != nil { + skipped += int64(len(req.slots)) + } else { + skipped += int64(len(req.addresses)) + } + default: + break drain + } + } + if paused { + // Clear old data + p.storageTries = nil + p.accountTrie = nil + p.accountTrieRoot = common.Hash{} + // Resume again + storageTries = make(map[common.Hash]Trie) + accountTrieRoot = cmd.root + accountTrie, err = p.db.OpenTrie(accountTrieRoot) + if err != nil { + log.Error("Trie prefetcher failed opening trie", "root", accountTrieRoot, "err", err) + } + if accountTrieRoot == (common.Hash{}) { + log.Error("Trie prefetcher unpaused with bad root") + } + paused = false + } else { + // Update metrics at new block events + triePrefetchFetchMeter.Mark(fetched) + triePrefetchSkipMeter.Mark(skipped) + fetched, skipped = 0, 0 + // Make the tries accessible + p.accountTrie = accountTrie + p.storageTries = storageTries + p.accountTrieRoot = accountTrieRoot + if cmd.root != (common.Hash{}) { + log.Error("Trie prefetcher paused with non-empty root") + } + paused = true + } + p.deliveryCh <- struct{}{} + case req := <-p.requestCh: + if paused { + continue + } + if sRoot := req.storageRoot; sRoot != nil { + // Storage slots to fetch + var ( + storageTrie Trie + err error + ) + if storageTrie = storageTries[*sRoot]; storageTrie == nil { + if storageTrie, err = p.db.OpenTrie(*sRoot); err != nil { + log.Warn("trie prefetcher failed opening storage trie", "root", *sRoot, "err", err) + skipped += int64(len(req.slots)) + continue + } + storageTries[*sRoot] = storageTrie + } + for _, key := range req.slots { + storageTrie.TryGet(key[:]) + } + fetched += int64(len(req.slots)) + } else { // an account + for _, addr := range req.addresses { + accountTrie.TryGet(addr[:]) + } + fetched += int64(len(req.addresses)) + } + } + } +} + +// Close stops the prefetcher +func (p *TriePrefetcher) Close() { + if p.quitCh != nil { + close(p.quitCh) + p.quitCh = nil + } +} + +// Resume causes the prefetcher to clear out old data, and get ready to +// fetch data concerning the new root +func (p *TriePrefetcher) Resume(root common.Hash) { + p.paused = false + p.cmdCh <- &cmd{ + root: root, + } + // Wait for it + <-p.deliveryCh +} + +// Pause causes the prefetcher to pause prefetching, and make tries +// accessible to callers via GetTrie +func (p *TriePrefetcher) Pause() { + if p.paused { + return + } + p.paused = true + p.cmdCh <- &cmd{ + root: common.Hash{}, + } + // Wait for it + <-p.deliveryCh +} + +// PrefetchAddresses adds an address for prefetching +func (p *TriePrefetcher) PrefetchAddresses(addresses []common.Address) { + cmd := fetchRequest{ + addresses: addresses, + } + // We do an async send here, to not cause the caller to block + //p.requestCh <- cmd + select { + case p.requestCh <- cmd: + default: + triePrefetchDropMeter.Mark(int64(len(addresses))) + } +} + +// PrefetchStorage adds a storage root and a set of keys for prefetching +func (p *TriePrefetcher) PrefetchStorage(root common.Hash, slots []common.Hash) { + cmd := fetchRequest{ + storageRoot: &root, + slots: slots, + } + // We do an async send here, to not cause the caller to block + //p.requestCh <- cmd + select { + case p.requestCh <- cmd: + default: + triePrefetchDropMeter.Mark(int64(len(slots))) + } +} + +// GetTrie returns the trie matching the root hash, or nil if the prefetcher +// doesn't have it. +func (p *TriePrefetcher) GetTrie(root common.Hash) Trie { + if root == p.accountTrieRoot { + return p.accountTrie + } + if storageTrie, ok := p.storageTries[root]; ok { + // Two accounts may well have the same storage root, but we cannot allow + // them both to make updates to the same trie instance. Therefore, + // we need to either delete the trie now, or deliver a copy of the trie. + delete(p.storageTries, root) + return storageTrie + } + trieDeliveryMissMeter.Mark(1) + return nil +} diff --git a/crypto/crypto.go b/crypto/crypto.go index a4a49136a8..40969a2895 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -60,10 +60,23 @@ type KeccakState interface { Read([]byte) (int, error) } +// NewKeccakState creates a new KeccakState +func NewKeccakState() KeccakState { + return sha3.NewLegacyKeccak256().(KeccakState) +} + +// HashData hashes the provided data using the KeccakState and returns a 32 byte hash +func HashData(kh KeccakState, data []byte) (h common.Hash) { + kh.Reset() + kh.Write(data) + kh.Read(h[:]) + return h +} + // Keccak256 calculates and returns the Keccak256 hash of the input data. func Keccak256(data ...[]byte) []byte { b := make([]byte, 32) - d := sha3.NewLegacyKeccak256().(KeccakState) + d := NewKeccakState() for _, b := range data { d.Write(b) } @@ -74,7 +87,7 @@ func Keccak256(data ...[]byte) []byte { // Keccak256Hash calculates and returns the Keccak256 hash of the input data, // converting it to an internal Hash data structure. func Keccak256Hash(data ...[]byte) (h common.Hash) { - d := sha3.NewLegacyKeccak256().(KeccakState) + d := NewKeccakState() for _, b := range data { d.Write(b) } diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index f71ae8232a..f9b0d3e834 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -42,6 +42,13 @@ func TestKeccak256Hash(t *testing.T) { checkhash(t, "Sha3-256-array", func(in []byte) []byte { h := Keccak256Hash(in); return h[:] }, msg, exp) } +func TestKeccak256Hasher(t *testing.T) { + msg := []byte("abc") + exp, _ := hex.DecodeString("4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45") + hasher := NewKeccakState() + checkhash(t, "Sha3-256-array", func(in []byte) []byte { h := HashData(hasher, in); return h[:] }, msg, exp) +} + func TestToECDSAErrors(t *testing.T) { if _, err := HexToECDSA("0000000000000000000000000000000000000000000000000000000000000000"); err == nil { t.Fatal("HexToECDSA should've returned error") From 42f9f1f0738bde1126eaa6f6bed9c1ae03e304a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 8 Jan 2021 15:01:49 +0200 Subject: [PATCH 082/709] core/state: convert prefetcher to concurrent per-trie loader --- accounts/abi/bind/backends/simulated.go | 3 +- core/blockchain.go | 30 +- core/state/state_object.go | 22 +- core/state/state_test.go | 2 +- core/state/statedb.go | 127 ++++--- core/state/statedb_test.go | 6 +- core/state/trie_prefetcher.go | 453 ++++++++++++++---------- eth/api_tracer.go | 6 +- miner/worker.go | 16 +- 9 files changed, 384 insertions(+), 281 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 6e87e037f0..8be364d08f 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -125,10 +125,9 @@ func (b *SimulatedBackend) Rollback() { func (b *SimulatedBackend) rollback() { blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {}) - stateDB, _ := b.blockchain.State() b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil) + b.pendingState, _ = state.New(b.pendingBlock.Root(), b.blockchain.StateCache(), nil) } // stateByBlockNumber retrieves a state by a given blocknumber. diff --git a/core/blockchain.go b/core/blockchain.go index ccb99bded5..d6668cdcd2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -201,12 +201,11 @@ type BlockChain struct { running int32 // 0 if chain is running, 1 when stopped procInterrupt int32 // interrupt signaler for block processing - engine consensus.Engine - validator Validator // Block and state validator interface - triePrefetcher *state.TriePrefetcher // Trie prefetcher interface - prefetcher Prefetcher - processor Processor // Block transaction processor interface - vmConfig vm.Config + engine consensus.Engine + validator Validator // Block and state validator interface + prefetcher Prefetcher + processor Processor // Block transaction processor interface + vmConfig vm.Config shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. @@ -250,15 +249,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) - tp := state.NewTriePrefetcher(bc.stateCache) - - bc.wg.Add(1) - go func() { - tp.Loop() - bc.wg.Done() - }() - bc.triePrefetcher = tp - bc.processor = NewStateProcessor(chainConfig, bc, engine) var err error @@ -1001,9 +991,6 @@ func (bc *BlockChain) Stop() { bc.scope.Close() close(bc.quit) bc.StopInsert() - if bc.triePrefetcher != nil { - bc.triePrefetcher.Close() - } bc.wg.Wait() // Ensure that the entirety of the state snapshot is journalled to disk. @@ -1870,16 +1857,20 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1) } statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) - statedb.UsePrefetcher(bc.triePrefetcher) if err != nil { return it.index, err } + // Enable prefetching to pull in trie node paths while processing transactions + statedb.StartPrefetcher("chain") + defer statedb.StopPrefetcher() // stopped on write anyway, defer meant to catch early error returns + // If we have a followup block, run that against the current state to pre-cache // transactions and probabilistically some of the account/storage trie nodes. var followupInterrupt uint32 if !bc.cacheConfig.TrieCleanNoPrefetch { if followup, err := it.peek(); followup != nil && err == nil { throwaway, _ := state.New(parent.Root, bc.stateCache, bc.snaps) + go func(start time.Time, followup *types.Block, throwaway *state.StateDB, interrupt *uint32) { bc.prefetcher.Prefetch(followup, throwaway, bc.vmConfig, &followupInterrupt) @@ -1933,7 +1924,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } - // Update the metrics touched during block commit accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them diff --git a/core/state/state_object.go b/core/state/state_object.go index 43c5074d92..f93f47d5f5 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -162,7 +162,7 @@ func (s *stateObject) getTrie(db Database) Trie { if s.data.Root != emptyRoot && s.db.prefetcher != nil { // When the miner is creating the pending state, there is no // prefetcher - s.trie = s.db.prefetcher.GetTrie(s.data.Root) + s.trie = s.db.prefetcher.trie(s.data.Root) } if s.trie == nil { var err error @@ -309,14 +309,16 @@ func (s *stateObject) setState(key, value common.Hash) { // finalise moves all dirty storage slots into the pending area to be hashed or // committed later. It is invoked at the end of every transaction. -func (s *stateObject) finalise() { - trieChanges := make([]common.Hash, 0, len(s.dirtyStorage)) +func (s *stateObject) finalise(prefetch bool) { + slotsToPrefetch := make([][]byte, 0, len(s.dirtyStorage)) for key, value := range s.dirtyStorage { s.pendingStorage[key] = value - trieChanges = append(trieChanges, key) + if value != s.originStorage[key] { + slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure + } } - if len(trieChanges) > 0 && s.db.prefetcher != nil && s.data.Root != emptyRoot { - s.db.prefetcher.PrefetchStorage(s.data.Root, trieChanges) + if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot { + s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch) } if len(s.dirtyStorage) > 0 { s.dirtyStorage = make(Storage) @@ -327,7 +329,7 @@ func (s *stateObject) finalise() { // It will return nil if the trie has not been loaded and no changes have been made func (s *stateObject) updateTrie(db Database) Trie { // Make sure all dirty slots are finalized into the pending storage area - s.finalise() + s.finalise(false) // Don't prefetch any more, pull directly if need be if len(s.pendingStorage) == 0 { return s.trie } @@ -340,6 +342,8 @@ func (s *stateObject) updateTrie(db Database) Trie { // Insert all the pending updates into the trie tr := s.getTrie(db) hasher := s.db.hasher + + usedStorage := make([][]byte, 0, len(s.pendingStorage)) for key, value := range s.pendingStorage { // Skip noop changes, persist actual changes if value == s.originStorage[key] { @@ -366,6 +370,10 @@ func (s *stateObject) updateTrie(db Database) Trie { } storage[crypto.HashData(hasher, key[:])] = v // v will be nil if value is 0x00 } + usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure + } + if s.db.prefetcher != nil { + s.db.prefetcher.used(s.data.Root, usedStorage) } if len(s.pendingStorage) > 0 { s.pendingStorage = make(Storage) diff --git a/core/state/state_test.go b/core/state/state_test.go index 526d7f8177..22e93d7a95 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -170,7 +170,7 @@ func TestSnapshot2(t *testing.T) { state.setStateObject(so0) root, _ := state.Commit(false) - state.Reset(root) + state, _ = New(root, state.db, state.snaps) // and one with deleted == true so1 := state.getStateObject(stateobjaddr1) diff --git a/core/state/statedb.go b/core/state/statedb.go index ce50962e8b..49f457a99f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -63,7 +63,7 @@ func (n *proofList) Delete(key []byte) error { // * Accounts type StateDB struct { db Database - prefetcher *TriePrefetcher + prefetcher *triePrefetcher originalRoot common.Hash // The pre-state root, before any changes were made trie Trie hasher crypto.KeccakState @@ -149,10 +149,25 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) return sdb, nil } -func (s *StateDB) UsePrefetcher(prefetcher *TriePrefetcher) { - if prefetcher != nil { - s.prefetcher = prefetcher - s.prefetcher.Resume(s.originalRoot) +// StartPrefetcher initializes a new trie prefetcher to pull in nodes from the +// state trie concurrently while the state is mutated so that when we reach the +// commit phase, most of the needed data is already hot. +func (s *StateDB) StartPrefetcher(namespace string) { + if s.prefetcher != nil { + s.prefetcher.close() + s.prefetcher = nil + } + if s.snap != nil { + s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace) + } +} + +// StopPrefetcher terminates a running prefetcher and reports any leftover stats +// from the gathered metrics. +func (s *StateDB) StopPrefetcher() { + if s.prefetcher != nil { + s.prefetcher.close() + s.prefetcher = nil } } @@ -167,37 +182,6 @@ func (s *StateDB) Error() error { return s.dbErr } -// Reset clears out all ephemeral state objects from the state db, but keeps -// the underlying state trie to avoid reloading data for the next operations. -func (s *StateDB) Reset(root common.Hash) error { - tr, err := s.db.OpenTrie(root) - if err != nil { - return err - } - s.trie = tr - s.stateObjects = make(map[common.Address]*stateObject) - s.stateObjectsPending = make(map[common.Address]struct{}) - s.stateObjectsDirty = make(map[common.Address]struct{}) - s.thash = common.Hash{} - s.bhash = common.Hash{} - s.txIndex = 0 - s.logs = make(map[common.Hash][]*types.Log) - s.logSize = 0 - s.preimages = make(map[common.Hash][]byte) - s.clearJournalAndRefund() - - if s.snaps != nil { - s.snapAccounts, s.snapDestructs, s.snapStorage = nil, nil, nil - if s.snap = s.snaps.Snapshot(root); s.snap != nil { - s.snapDestructs = make(map[common.Hash]struct{}) - s.snapAccounts = make(map[common.Hash][]byte) - s.snapStorage = make(map[common.Hash]map[common.Hash][]byte) - } - } - s.accessList = newAccessList() - return nil -} - func (s *StateDB) AddLog(log *types.Log) { s.journal.append(addLogChange{txhash: s.thash}) @@ -737,6 +721,13 @@ func (s *StateDB) Copy() *StateDB { // However, it doesn't cost us much to copy an empty list, so we do it anyway // to not blow up if we ever decide copy it in the middle of a transaction state.accessList = s.accessList.Copy() + + // If there's a prefetcher running, make an inactive copy of it that can + // only access data but does not actively preload (since the user will not + // know that they need to explicitly terminate an active copy). + if s.prefetcher != nil { + state.prefetcher = s.prefetcher.copy() + } return state } @@ -773,7 +764,7 @@ func (s *StateDB) GetRefund() uint64 { // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { - var addressesToPrefetch []common.Address + addressesToPrefetch := make([][]byte, 0, len(s.journal.dirties)) for addr := range s.journal.dirties { obj, exist := s.stateObjects[addr] if !exist { @@ -798,21 +789,19 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { delete(s.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a ressurrect) } } else { - obj.finalise() + obj.finalise(true) // Prefetch slots in the background } s.stateObjectsPending[addr] = struct{}{} s.stateObjectsDirty[addr] = struct{}{} + // At this point, also ship the address off to the precacher. The precacher // will start loading tries, and when the change is eventually committed, // the commit-phase will be a lot faster - if s.prefetcher != nil { - addressesToPrefetch = append(addressesToPrefetch, addr) - } + addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure } - if s.prefetcher != nil { - s.prefetcher.PrefetchAddresses(addressesToPrefetch) + if s.prefetcher != nil && len(addressesToPrefetch) > 0 { + s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch) } - // Invalidate journal because reverting across transactions is not allowed. s.clearJournalAndRefund() } @@ -824,29 +813,49 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Finalise all the dirty storage states and write them into the tries s.Finalise(deleteEmptyObjects) - // Now we're about to start to write changes to the trie. The trie is so - // far _untouched_. We can check with the prefetcher, if it can give us - // a trie which has the same root, but also has some content loaded into it. - // If so, use that one instead. + // If there was a trie prefetcher operating, it gets aborted and irrevocably + // modified after we start retrieving tries. Remove it from the statedb after + // this round of use. + // + // This is weird pre-byzantium since the first tx runs with a prefetcher and + // the remainder without, but pre-byzantium even the initial prefetcher is + // useless, so no sleep lost. + prefetcher := s.prefetcher if s.prefetcher != nil { - s.prefetcher.Pause() - // We only want to do this _once_, if someone calls IntermediateRoot again, - // we shouldn't fetch the trie again - if s.originalRoot != (common.Hash{}) { - if trie := s.prefetcher.GetTrie(s.originalRoot); trie != nil { - s.trie = trie - } - s.originalRoot = common.Hash{} + defer func() { + s.prefetcher.close() + s.prefetcher = nil + }() + } + // Although naively it makes sense to retrieve the account trie and then do + // the contract storage and account updates sequentially, that short circuits + // the account prefetcher. Instead, let's process all the storage updates + // first, giving the account prefeches just a few more milliseconds of time + // to pull useful data from disk. + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; !obj.deleted { + obj.updateRoot(s.db) + } + } + // Now we're about to start to write changes to the trie. The trie is so far + // _untouched_. We can check with the prefetcher, if it can give us a trie + // which has the same root, but also has some content loaded into it. + if prefetcher != nil { + if trie := prefetcher.trie(s.originalRoot); trie != nil { + s.trie = trie } } + usedAddrs := make([][]byte, 0, len(s.stateObjectsPending)) for addr := range s.stateObjectsPending { - obj := s.stateObjects[addr] - if obj.deleted { + if obj := s.stateObjects[addr]; obj.deleted { s.deleteStateObject(obj) } else { - obj.updateRoot(s.db) s.updateStateObject(obj) } + usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure + } + if prefetcher != nil { + prefetcher.used(s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 70d01ff3dd..220e28525c 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -474,7 +474,7 @@ func TestTouchDelete(t *testing.T) { s := newStateTest() s.state.GetOrNewStateObject(common.Address{}) root, _ := s.state.Commit(false) - s.state.Reset(root) + s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() s.state.AddBalance(common.Address{}, new(big.Int)) @@ -676,7 +676,7 @@ func TestDeleteCreateRevert(t *testing.T) { state.SetBalance(addr, big.NewInt(1)) root, _ := state.Commit(false) - state.Reset(root) + state, _ = New(root, state.db, state.snaps) // Simulate self-destructing in one transaction, then create-reverting in another state.Suicide(addr) @@ -688,7 +688,7 @@ func TestDeleteCreateRevert(t *testing.T) { // Commit the entire state and make sure we don't crash and have the correct state root, _ = state.Commit(true) - state.Reset(root) + state, _ = New(root, state.db, state.snaps) if state.getStateObject(addr) != nil { t.Fatalf("self-destructed contract came alive") diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 8a1aab3253..ac5e95c5c2 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -17,233 +17,318 @@ package state import ( + "sync" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" ) var ( - // trieDeliveryMeter counts how many times the prefetcher was unable to supply - // the statedb with a prefilled trie. This meter should be zero -- if it's not, that - // needs to be investigated - trieDeliveryMissMeter = metrics.NewRegisteredMeter("trie/prefetch/deliverymiss", nil) - - triePrefetchFetchMeter = metrics.NewRegisteredMeter("trie/prefetch/fetch", nil) - triePrefetchSkipMeter = metrics.NewRegisteredMeter("trie/prefetch/skip", nil) - triePrefetchDropMeter = metrics.NewRegisteredMeter("trie/prefetch/drop", nil) + // triePrefetchMetricsPrefix is the prefix under which to publis the metrics. + triePrefetchMetricsPrefix = "trie/prefetch/" ) -// TriePrefetcher is an active prefetcher, which receives accounts or storage -// items on two channels, and does trie-loading of the items. -// The goal is to get as much useful content into the caches as possible -type TriePrefetcher struct { - requestCh chan (fetchRequest) // Chan to receive requests for data to fetch - cmdCh chan (*cmd) // Chan to control activity, pause/new root - quitCh chan (struct{}) - deliveryCh chan (struct{}) - db Database - - paused bool - - storageTries map[common.Hash]Trie - accountTrie Trie - accountTrieRoot common.Hash +// triePrefetcher is an active prefetcher, which receives accounts or storage +// items and does trie-loading of them. The goal is to get as much useful content +// into the caches as possible. +// +// Note, the prefetcher's API is not thread safe. +type triePrefetcher struct { + db Database // Database to fetch trie nodes through + root common.Hash // Root hash of theaccount trie for metrics + fetches map[common.Hash]Trie // Partially or fully fetcher tries + fetchers map[common.Hash]*subfetcher // Subfetchers for each trie + + deliveryMissMeter metrics.Meter + accountLoadMeter metrics.Meter + accountDupMeter metrics.Meter + accountSkipMeter metrics.Meter + accountWasteMeter metrics.Meter + storageLoadMeter metrics.Meter + storageDupMeter metrics.Meter + storageSkipMeter metrics.Meter + storageWasteMeter metrics.Meter } -func NewTriePrefetcher(db Database) *TriePrefetcher { - return &TriePrefetcher{ - requestCh: make(chan fetchRequest, 200), - cmdCh: make(chan *cmd), - quitCh: make(chan struct{}), - deliveryCh: make(chan struct{}), - db: db, +// newTriePrefetcher +func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { + prefix := triePrefetchMetricsPrefix + namespace + p := &triePrefetcher{ + db: db, + root: root, + fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map + + deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), + accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), + accountDupMeter: metrics.GetOrRegisterMeter(prefix+"/account/dup", nil), + accountSkipMeter: metrics.GetOrRegisterMeter(prefix+"/account/skip", nil), + accountWasteMeter: metrics.GetOrRegisterMeter(prefix+"/account/waste", nil), + storageLoadMeter: metrics.GetOrRegisterMeter(prefix+"/storage/load", nil), + storageDupMeter: metrics.GetOrRegisterMeter(prefix+"/storage/dup", nil), + storageSkipMeter: metrics.GetOrRegisterMeter(prefix+"/storage/skip", nil), + storageWasteMeter: metrics.GetOrRegisterMeter(prefix+"/storage/waste", nil), } + return p } -type cmd struct { - root common.Hash -} +// close iterates over all the subfetchers, aborts any that were left spinning +// and reports the stats to the metrics subsystem. +func (p *triePrefetcher) close() { + for _, fetcher := range p.fetchers { + fetcher.abort() // safe to do multiple times -type fetchRequest struct { - slots []common.Hash - storageRoot *common.Hash - addresses []common.Address -} + if metrics.Enabled { + if fetcher.root == p.root { + p.accountLoadMeter.Mark(int64(len(fetcher.seen))) + p.accountDupMeter.Mark(int64(fetcher.dups)) + p.accountSkipMeter.Mark(int64(len(fetcher.tasks))) -func (p *TriePrefetcher) Loop() { - var ( - accountTrieRoot common.Hash - accountTrie Trie - storageTries map[common.Hash]Trie - - err error - // Some tracking of performance - skipped int64 - fetched int64 - - paused = true - ) - // The prefetcher loop has two distinct phases: - // 1: Paused: when in this state, the accumulated tries are accessible to outside - // callers. - // 2: Active prefetching, awaiting slots and accounts to prefetch - for { - select { - case <-p.quitCh: - return - case cmd := <-p.cmdCh: - // Clear out any old requests - drain: - for { - select { - case req := <-p.requestCh: - if req.slots != nil { - skipped += int64(len(req.slots)) - } else { - skipped += int64(len(req.addresses)) - } - default: - break drain + for _, key := range fetcher.used { + delete(fetcher.seen, string(key)) } - } - if paused { - // Clear old data - p.storageTries = nil - p.accountTrie = nil - p.accountTrieRoot = common.Hash{} - // Resume again - storageTries = make(map[common.Hash]Trie) - accountTrieRoot = cmd.root - accountTrie, err = p.db.OpenTrie(accountTrieRoot) - if err != nil { - log.Error("Trie prefetcher failed opening trie", "root", accountTrieRoot, "err", err) - } - if accountTrieRoot == (common.Hash{}) { - log.Error("Trie prefetcher unpaused with bad root") - } - paused = false + p.accountWasteMeter.Mark(int64(len(fetcher.seen))) } else { - // Update metrics at new block events - triePrefetchFetchMeter.Mark(fetched) - triePrefetchSkipMeter.Mark(skipped) - fetched, skipped = 0, 0 - // Make the tries accessible - p.accountTrie = accountTrie - p.storageTries = storageTries - p.accountTrieRoot = accountTrieRoot - if cmd.root != (common.Hash{}) { - log.Error("Trie prefetcher paused with non-empty root") - } - paused = true - } - p.deliveryCh <- struct{}{} - case req := <-p.requestCh: - if paused { - continue - } - if sRoot := req.storageRoot; sRoot != nil { - // Storage slots to fetch - var ( - storageTrie Trie - err error - ) - if storageTrie = storageTries[*sRoot]; storageTrie == nil { - if storageTrie, err = p.db.OpenTrie(*sRoot); err != nil { - log.Warn("trie prefetcher failed opening storage trie", "root", *sRoot, "err", err) - skipped += int64(len(req.slots)) - continue - } - storageTries[*sRoot] = storageTrie - } - for _, key := range req.slots { - storageTrie.TryGet(key[:]) - } - fetched += int64(len(req.slots)) - } else { // an account - for _, addr := range req.addresses { - accountTrie.TryGet(addr[:]) + p.storageLoadMeter.Mark(int64(len(fetcher.seen))) + p.storageDupMeter.Mark(int64(fetcher.dups)) + p.storageSkipMeter.Mark(int64(len(fetcher.tasks))) + + for _, key := range fetcher.used { + delete(fetcher.seen, string(key)) } - fetched += int64(len(req.addresses)) + p.storageWasteMeter.Mark(int64(len(fetcher.seen))) } } } + // Clear out all fetchers (will crash on a second call, deliberate) + p.fetchers = nil } -// Close stops the prefetcher -func (p *TriePrefetcher) Close() { - if p.quitCh != nil { - close(p.quitCh) - p.quitCh = nil +// copy creates a deep-but-inactive copy of the trie prefetcher. Any trie data +// already loaded will be copied over, but no goroutines will be started. This +// is mostly used in the miner which creates a copy of it's actively mutated +// state to be sealed while it may further mutate the state. +func (p *triePrefetcher) copy() *triePrefetcher { + copy := &triePrefetcher{ + db: p.db, + root: p.root, + fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map + + deliveryMissMeter: p.deliveryMissMeter, + accountLoadMeter: p.accountLoadMeter, + accountDupMeter: p.accountDupMeter, + accountSkipMeter: p.accountSkipMeter, + accountWasteMeter: p.accountWasteMeter, + storageLoadMeter: p.storageLoadMeter, + storageDupMeter: p.storageDupMeter, + storageSkipMeter: p.storageSkipMeter, + storageWasteMeter: p.storageWasteMeter, + } + // If the prefetcher is already a copy, duplicate the data + if p.fetches != nil { + for root, fetch := range p.fetches { + copy.fetches[root] = p.db.CopyTrie(fetch) + } + return copy + } + // Otherwise we're copying an active fetcher, retrieve the current states + for root, fetcher := range p.fetchers { + copy.fetches[root] = fetcher.peek() } + return copy } -// Resume causes the prefetcher to clear out old data, and get ready to -// fetch data concerning the new root -func (p *TriePrefetcher) Resume(root common.Hash) { - p.paused = false - p.cmdCh <- &cmd{ - root: root, +// prefetch schedules a batch of trie items to prefetch. +func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) { + // If the prefetcher is an inactive one, bail out + if p.fetches != nil { + return + } + // Active fetcher, schedule the retrievals + fetcher := p.fetchers[root] + if fetcher == nil { + fetcher = newSubfetcher(p.db, root) + p.fetchers[root] = fetcher } - // Wait for it - <-p.deliveryCh + fetcher.schedule(keys) } -// Pause causes the prefetcher to pause prefetching, and make tries -// accessible to callers via GetTrie -func (p *TriePrefetcher) Pause() { - if p.paused { - return +// trie returns the trie matching the root hash, or nil if the prefetcher doesn't +// have it. +func (p *triePrefetcher) trie(root common.Hash) Trie { + // If the prefetcher is inactive, return from existing deep copies + if p.fetches != nil { + trie := p.fetches[root] + if trie == nil { + p.deliveryMissMeter.Mark(1) + return nil + } + return p.db.CopyTrie(trie) } - p.paused = true - p.cmdCh <- &cmd{ - root: common.Hash{}, + // Otherwise the prefetcher is active, bail if no trie was prefetched for this root + fetcher := p.fetchers[root] + if fetcher == nil { + p.deliveryMissMeter.Mark(1) + return nil } - // Wait for it - <-p.deliveryCh + // Interrupt the prefetcher if it's by any chance still running and return + // a copy of any pre-loaded trie. + fetcher.abort() // safe to do multiple times + + trie := fetcher.peek() + if trie == nil { + p.deliveryMissMeter.Mark(1) + return nil + } + return trie } -// PrefetchAddresses adds an address for prefetching -func (p *TriePrefetcher) PrefetchAddresses(addresses []common.Address) { - cmd := fetchRequest{ - addresses: addresses, +// used marks a batch of state items used to allow creating statistics as to +// how useful or wasteful the prefetcher is. +func (p *triePrefetcher) used(root common.Hash, used [][]byte) { + if fetcher := p.fetchers[root]; fetcher != nil { + fetcher.used = used } - // We do an async send here, to not cause the caller to block - //p.requestCh <- cmd +} + +// subfetcher is a trie fetcher goroutine responsible for pulling entries for a +// single trie. It is spawned when a new root is encountered and lives until the +// main prefetcher is paused and either all requested items are processed or if +// the trie being worked on is retrieved from the prefetcher. +type subfetcher struct { + db Database // Database to load trie nodes through + root common.Hash // Root hash of the trie to prefetch + trie Trie // Trie being populated with nodes + + tasks [][]byte // Items queued up for retrieval + lock sync.Mutex // Lock protecting the task queue + + wake chan struct{} // Wake channel if a new task is scheduled + stop chan struct{} // Channel to interrupt processing + term chan struct{} // Channel to signal iterruption + copy chan chan Trie // Channel to request a copy of the current trie + + seen map[string]struct{} // Tracks the entries already loaded + dups int // Number of duplicate preload tasks + used [][]byte // Tracks the entries used in the end +} + +// newSubfetcher creates a goroutine to prefetch state items belonging to a +// particular root hash. +func newSubfetcher(db Database, root common.Hash) *subfetcher { + sf := &subfetcher{ + db: db, + root: root, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + term: make(chan struct{}), + copy: make(chan chan Trie), + seen: make(map[string]struct{}), + } + go sf.loop() + return sf +} + +// schedule adds a batch of trie keys to the queue to prefetch. +func (sf *subfetcher) schedule(keys [][]byte) { + // Append the tasks to the current queue + sf.lock.Lock() + sf.tasks = append(sf.tasks, keys...) + sf.lock.Unlock() + + // Notify the prefetcher, it's fine if it's already terminated select { - case p.requestCh <- cmd: + case sf.wake <- struct{}{}: default: - triePrefetchDropMeter.Mark(int64(len(addresses))) } } -// PrefetchStorage adds a storage root and a set of keys for prefetching -func (p *TriePrefetcher) PrefetchStorage(root common.Hash, slots []common.Hash) { - cmd := fetchRequest{ - storageRoot: &root, - slots: slots, +// peek tries to retrieve a deep copy of the fetcher's trie in whatever form it +// is currently. +func (sf *subfetcher) peek() Trie { + ch := make(chan Trie) + select { + case sf.copy <- ch: + // Subfetcher still alive, return copy from it + return <-ch + + case <-sf.term: + // Subfetcher already terminated, return a copy directly + if sf.trie == nil { + return nil + } + return sf.db.CopyTrie(sf.trie) } - // We do an async send here, to not cause the caller to block - //p.requestCh <- cmd +} + +// abort interrupts the subfetcher immediately. It is safe to call abort multiple +// times but it is not thread safe. +func (sf *subfetcher) abort() { select { - case p.requestCh <- cmd: + case <-sf.stop: default: - triePrefetchDropMeter.Mark(int64(len(slots))) + close(sf.stop) } + <-sf.term } -// GetTrie returns the trie matching the root hash, or nil if the prefetcher -// doesn't have it. -func (p *TriePrefetcher) GetTrie(root common.Hash) Trie { - if root == p.accountTrieRoot { - return p.accountTrie - } - if storageTrie, ok := p.storageTries[root]; ok { - // Two accounts may well have the same storage root, but we cannot allow - // them both to make updates to the same trie instance. Therefore, - // we need to either delete the trie now, or deliver a copy of the trie. - delete(p.storageTries, root) - return storageTrie - } - trieDeliveryMissMeter.Mark(1) - return nil +// loop waits for new tasks to be scheduled and keeps loading them until it runs +// out of tasks or its underlying trie is retrieved for committing. +func (sf *subfetcher) loop() { + // No matter how the loop stops, signal anyone waiting that it's terminated + defer close(sf.term) + + // Start by opening the trie and stop processing if it fails + trie, err := sf.db.OpenTrie(sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie + + // Trie opened successfully, keep prefetching items + for { + select { + case <-sf.wake: + // Subfetcher was woken up, retrieve any tasks to avoid spinning the lock + sf.lock.Lock() + tasks := sf.tasks + sf.tasks = nil + sf.lock.Unlock() + + // Prefetch any tasks until the loop is interrupted + for i, task := range tasks { + select { + case <-sf.stop: + // If termination is requested, add any leftover back and return + sf.lock.Lock() + sf.tasks = append(sf.tasks, tasks[i:]...) + sf.lock.Unlock() + return + + case ch := <-sf.copy: + // Somebody wants a copy of the current trie, grant them + ch <- sf.db.CopyTrie(sf.trie) + + default: + // No termination request yet, prefetch the next entry + taskid := string(task) + if _, ok := sf.seen[taskid]; ok { + sf.dups++ + } else { + sf.trie.TryGet(task) + sf.seen[taskid] = struct{}{} + } + } + } + + case ch := <-sf.copy: + // Somebody wants a copy of the current trie, grant them + ch <- sf.db.CopyTrie(sf.trie) + + case <-sf.stop: + // Termination is requested, abort and leave remaining tasks + return + } + } } diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 8e71945ee2..5dffb2a464 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -299,7 +299,8 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl failed = err break } - if err := statedb.Reset(root); err != nil { + statedb, err = state.New(root, database, nil) + if err != nil { failed = err break } @@ -699,7 +700,8 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* if err != nil { return nil, err } - if err := statedb.Reset(root); err != nil { + statedb, err = state.New(root, database, nil) + if err != nil { return nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) } database.TrieDB().Reference(root, common.Hash{}) diff --git a/miner/worker.go b/miner/worker.go index 2c5032c656..82d08d4c7e 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -303,6 +303,9 @@ func (w *worker) isRunning() bool { // close terminates all background threads maintained by the worker. // Note the worker does not support being closed multiple times. func (w *worker) close() { + if w.current != nil && w.current.state != nil { + w.current.state.StopPrefetcher() + } atomic.StoreInt32(&w.running, 0) close(w.exitCh) } @@ -642,10 +645,14 @@ func (w *worker) resultLoop() { // makeCurrent creates a new environment for the current cycle. func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { + // Retrieve the parent state to execute on top and start a prefetcher for + // the miner to speed block sealing up a bit state, err := w.chain.StateAt(parent.Root()) if err != nil { return err } + state.StartPrefetcher("miner") + env := &environment{ signer: types.NewEIP155Signer(w.chainConfig.ChainID), state: state, @@ -654,7 +661,6 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { uncles: mapset.NewSet(), header: header, } - // when 08 is processed ancestors contain 07 (quick block) for _, ancestor := range w.chain.GetBlocksFromHash(parent.Hash(), 7) { for _, uncle := range ancestor.Uncles() { @@ -663,9 +669,14 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { env.family.Add(ancestor.Hash()) env.ancestors.Add(ancestor.Hash()) } - // Keep track of transactions which return errors so they can be removed env.tcount = 0 + + // Swap out the old work with the new one, terminating any leftover prefetcher + // processes in the mean time and starting a new one. + if w.current != nil && w.current.state != nil { + w.current.state.StopPrefetcher() + } w.current = env return nil } @@ -719,7 +730,6 @@ func (w *worker) updateSnapshot() { w.current.receipts, new(trie.Trie), ) - w.snapshotState = w.current.state.Copy() } From c4307a9339fd4e66598fd5c378e8f6f97ad878e2 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Thu, 21 Jan 2021 12:17:10 +0100 Subject: [PATCH 083/709] eth/filters: fix potential deadlock in filter timeout loop (#22178) This fixes #22131 and adds a test reproducing the issue. --- eth/backend.go | 2 +- eth/filters/api.go | 33 +++++++----- eth/filters/filter_system_test.go | 86 ++++++++++++++++++++++++++++--- les/client.go | 2 +- 4 files changed, 101 insertions(+), 22 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index c1732d3ceb..a6390facb3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -335,7 +335,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.APIBackend, false), + Service: filters.NewPublicFilterAPI(s.APIBackend, false, 5*time.Minute), Public: true, }, { Namespace: "admin", diff --git a/eth/filters/api.go b/eth/filters/api.go index b6f974c3ba..4b36a5379e 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -34,10 +34,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var ( - deadline = 5 * time.Minute // consider a filter inactive if it has not been polled for within deadline -) - // filter is a helper struct that holds meta information over the filter type // and associated subscription in the event system. type filter struct { @@ -59,25 +55,28 @@ type PublicFilterAPI struct { events *EventSystem filtersMu sync.Mutex filters map[rpc.ID]*filter + timeout time.Duration } // NewPublicFilterAPI returns a new PublicFilterAPI instance. -func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI { +func NewPublicFilterAPI(backend Backend, lightMode bool, timeout time.Duration) *PublicFilterAPI { api := &PublicFilterAPI{ backend: backend, chainDb: backend.ChainDb(), events: NewEventSystem(backend, lightMode), filters: make(map[rpc.ID]*filter), + timeout: timeout, } - go api.timeoutLoop() + go api.timeoutLoop(timeout) return api } // timeoutLoop runs every 5 minutes and deletes filters that have not been recently used. // Tt is started when the api is created. -func (api *PublicFilterAPI) timeoutLoop() { - ticker := time.NewTicker(5 * time.Minute) +func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { + var toUninstall []*Subscription + ticker := time.NewTicker(timeout) defer ticker.Stop() for { <-ticker.C @@ -85,13 +84,21 @@ func (api *PublicFilterAPI) timeoutLoop() { for id, f := range api.filters { select { case <-f.deadline.C: - f.s.Unsubscribe() + toUninstall = append(toUninstall, f.s) delete(api.filters, id) default: continue } } api.filtersMu.Unlock() + + // Unsubscribes are processed outside the lock to avoid the following scenario: + // event loop attempts broadcasting events to still active filters while + // Unsubscribe is waiting for it to process the uninstall request. + for _, s := range toUninstall { + s.Unsubscribe() + } + toUninstall = nil } } @@ -109,7 +116,7 @@ func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { ) api.filtersMu.Lock() - api.filters[pendingTxSub.ID] = &filter{typ: PendingTransactionsSubscription, deadline: time.NewTimer(deadline), hashes: make([]common.Hash, 0), s: pendingTxSub} + api.filters[pendingTxSub.ID] = &filter{typ: PendingTransactionsSubscription, deadline: time.NewTimer(api.timeout), hashes: make([]common.Hash, 0), s: pendingTxSub} api.filtersMu.Unlock() go func() { @@ -179,7 +186,7 @@ func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { ) api.filtersMu.Lock() - api.filters[headerSub.ID] = &filter{typ: BlocksSubscription, deadline: time.NewTimer(deadline), hashes: make([]common.Hash, 0), s: headerSub} + api.filters[headerSub.ID] = &filter{typ: BlocksSubscription, deadline: time.NewTimer(api.timeout), hashes: make([]common.Hash, 0), s: headerSub} api.filtersMu.Unlock() go func() { @@ -296,7 +303,7 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { } api.filtersMu.Lock() - api.filters[logsSub.ID] = &filter{typ: LogsSubscription, crit: crit, deadline: time.NewTimer(deadline), logs: make([]*types.Log, 0), s: logsSub} + api.filters[logsSub.ID] = &filter{typ: LogsSubscription, crit: crit, deadline: time.NewTimer(api.timeout), logs: make([]*types.Log, 0), s: logsSub} api.filtersMu.Unlock() go func() { @@ -421,7 +428,7 @@ func (api *PublicFilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { // receive timer value and reset timer <-f.deadline.C } - f.deadline.Reset(deadline) + f.deadline.Reset(api.timeout) switch f.typ { case PendingTransactionsSubscription, BlocksSubscription: diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index b534c1d47f..52150366c1 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -22,6 +22,7 @@ import ( "math/big" "math/rand" "reflect" + "runtime" "testing" "time" @@ -38,6 +39,10 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) +var ( + deadline = 5 * time.Minute +) + type testBackend struct { mux *event.TypeMux db ethdb.Database @@ -163,7 +168,7 @@ func TestBlockSubscription(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) genesis = new(core.Genesis).MustCommit(db) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) chainEvents = []core.ChainEvent{} @@ -215,7 +220,7 @@ func TestPendingTxFilter(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -270,7 +275,7 @@ func TestLogFilterCreation(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) testCases = []struct { crit FilterCriteria @@ -314,7 +319,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) ) // different situations where log filter creation should fail. @@ -336,7 +341,7 @@ func TestInvalidGetLogsRequest(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") ) @@ -361,7 +366,7 @@ func TestLogFilter(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -475,7 +480,7 @@ func TestPendingLogsSubscription(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false) + api = NewPublicFilterAPI(backend, false, deadline) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -601,6 +606,73 @@ func TestPendingLogsSubscription(t *testing.T) { } } +// TestPendingTxFilterDeadlock tests if the event loop hangs when pending +// txes arrive at the same time that one of multiple filters is timing out. +// Please refer to #22131 for more details. +func TestPendingTxFilterDeadlock(t *testing.T) { + t.Parallel() + timeout := 100 * time.Millisecond + + var ( + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false, timeout) + done = make(chan struct{}) + ) + + go func() { + // Bombard feed with txes until signal was received to stop + i := uint64(0) + for { + select { + case <-done: + return + default: + } + + tx := types.NewTransaction(i, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil) + backend.txFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx}}) + i++ + } + }() + + // Create a bunch of filters that will + // timeout either in 100ms or 200ms + fids := make([]rpc.ID, 20) + for i := 0; i < len(fids); i++ { + fid := api.NewPendingTransactionFilter() + fids[i] = fid + // Wait for at least one tx to arrive in filter + for { + hashes, err := api.GetFilterChanges(fid) + if err != nil { + t.Fatalf("Filter should exist: %v\n", err) + } + if len(hashes.([]common.Hash)) > 0 { + break + } + runtime.Gosched() + } + } + + // Wait until filters have timed out + time.Sleep(3 * timeout) + + // If tx loop doesn't consume `done` after a second + // it's hanging. + select { + case done <- struct{}{}: + // Check that all filters have been uninstalled + for _, fid := range fids { + if _, err := api.GetFilterChanges(fid); err == nil { + t.Errorf("Filter %s should have been uninstalled\n", fid) + } + } + case <-time.After(1 * time.Second): + t.Error("Tx sending loop hangs") + } +} + func flattenLogs(pl [][]*types.Log) []*types.Log { var logs []*types.Log for _, l := range pl { diff --git a/les/client.go b/les/client.go index 47997a098b..5ee50a7c3a 100644 --- a/les/client.go +++ b/les/client.go @@ -252,7 +252,7 @@ func (s *LightEthereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, true), + Service: filters.NewPublicFilterAPI(s.ApiBackend, true, 5*time.Minute), Public: true, }, { Namespace: "net", From 231040c633f24b2b0d56aaeb704a0738ba4adb9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Zimnoch?= Date: Thu, 21 Jan 2021 13:47:38 +0100 Subject: [PATCH 084/709] event: add ResubscribeErr (#22191) This adds a way to get the error of the failing subscription for logging/debugging purposes. Co-authored-by: Felix Lange --- event/subscription.go | 32 ++++++++++++++++++++++++++++---- event/subscription_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/event/subscription.go b/event/subscription.go index c80d171f3a..6c62874719 100644 --- a/event/subscription.go +++ b/event/subscription.go @@ -95,6 +95,26 @@ func (s *funcSub) Err() <-chan error { // Resubscribe applies backoff between calls to fn. The time between calls is adapted // based on the error rate, but will never exceed backoffMax. func Resubscribe(backoffMax time.Duration, fn ResubscribeFunc) Subscription { + return ResubscribeErr(backoffMax, func(ctx context.Context, _ error) (Subscription, error) { + return fn(ctx) + }) +} + +// A ResubscribeFunc attempts to establish a subscription. +type ResubscribeFunc func(context.Context) (Subscription, error) + +// ResubscribeErr calls fn repeatedly to keep a subscription established. When the +// subscription is established, ResubscribeErr waits for it to fail and calls fn again. This +// process repeats until Unsubscribe is called or the active subscription ends +// successfully. +// +// The difference between Resubscribe and ResubscribeErr is that with ResubscribeErr, +// the error of the failing subscription is available to the callback for logging +// purposes. +// +// ResubscribeErr applies backoff between calls to fn. The time between calls is adapted +// based on the error rate, but will never exceed backoffMax. +func ResubscribeErr(backoffMax time.Duration, fn ResubscribeErrFunc) Subscription { s := &resubscribeSub{ waitTime: backoffMax / 10, backoffMax: backoffMax, @@ -106,15 +126,18 @@ func Resubscribe(backoffMax time.Duration, fn ResubscribeFunc) Subscription { return s } -// A ResubscribeFunc attempts to establish a subscription. -type ResubscribeFunc func(context.Context) (Subscription, error) +// A ResubscribeErrFunc attempts to establish a subscription. +// For every call but the first, the second argument to this function is +// the error that occurred with the previous subscription. +type ResubscribeErrFunc func(context.Context, error) (Subscription, error) type resubscribeSub struct { - fn ResubscribeFunc + fn ResubscribeErrFunc err chan error unsub chan struct{} unsubOnce sync.Once lastTry mclock.AbsTime + lastSubErr error waitTime, backoffMax time.Duration } @@ -149,7 +172,7 @@ func (s *resubscribeSub) subscribe() Subscription { s.lastTry = mclock.Now() ctx, cancel := context.WithCancel(context.Background()) go func() { - rsub, err := s.fn(ctx) + rsub, err := s.fn(ctx, s.lastSubErr) sub = rsub subscribed <- err }() @@ -178,6 +201,7 @@ func (s *resubscribeSub) waitForError(sub Subscription) bool { defer sub.Unsubscribe() select { case err := <-sub.Err(): + s.lastSubErr = err return err == nil case <-s.unsub: return true diff --git a/event/subscription_test.go b/event/subscription_test.go index c48be3aa30..ba081705c4 100644 --- a/event/subscription_test.go +++ b/event/subscription_test.go @@ -19,6 +19,8 @@ package event import ( "context" "errors" + "fmt" + "reflect" "testing" "time" ) @@ -118,3 +120,37 @@ func TestResubscribeAbort(t *testing.T) { t.Fatal(err) } } + +func TestResubscribeWithErrorHandler(t *testing.T) { + t.Parallel() + + var i int + nfails := 6 + subErrs := make([]string, 0) + sub := ResubscribeErr(100*time.Millisecond, func(ctx context.Context, lastErr error) (Subscription, error) { + i++ + var lastErrVal string + if lastErr != nil { + lastErrVal = lastErr.Error() + } + subErrs = append(subErrs, lastErrVal) + sub := NewSubscription(func(unsubscribed <-chan struct{}) error { + if i < nfails { + return fmt.Errorf("err-%v", i) + } else { + return nil + } + }) + return sub, nil + }) + + <-sub.Err() + if i != nfails { + t.Fatalf("resubscribe function called %d times, want %d times", i, nfails) + } + + expectedSubErrs := []string{"", "err-1", "err-2", "err-3", "err-4", "err-5"} + if !reflect.DeepEqual(subErrs, expectedSubErrs) { + t.Fatalf("unexpected subscription errors %v, want %v", subErrs, expectedSubErrs) + } +} From 9e1bd0f3671d19d4964ed8c8a95edfd12413d8c3 Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 22 Jan 2021 17:11:24 +0800 Subject: [PATCH 085/709] trie: fix range prover (#22210) Fixes a special case when the trie only has a single trie node and the range proof only contains a single element. --- trie/proof.go | 44 +++++++++++++++++++++++++++++--------------- trie/proof_test.go | 19 +++++++++++++++++++ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/trie/proof.go b/trie/proof.go index e7102f12b7..61c35a8423 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -216,7 +216,7 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV // // Note we have the assumption here the given boundary keys are different // and right is larger than left. -func unsetInternal(n node, left []byte, right []byte) error { +func unsetInternal(n node, left []byte, right []byte) (bool, error) { left, right = keybytesToHex(left), keybytesToHex(right) // Step down to the fork point. There are two scenarios can happen: @@ -278,45 +278,55 @@ findFork: // - left proof points to the shortnode, but right proof is greater // - right proof points to the shortnode, but left proof is less if shortForkLeft == -1 && shortForkRight == -1 { - return errors.New("empty range") + return false, errors.New("empty range") } if shortForkLeft == 1 && shortForkRight == 1 { - return errors.New("empty range") + return false, errors.New("empty range") } if shortForkLeft != 0 && shortForkRight != 0 { + // The fork point is root node, unset the entire trie + if parent == nil { + return true, nil + } parent.(*fullNode).Children[left[pos-1]] = nil - return nil + return false, nil } // Only one proof points to non-existent key. if shortForkRight != 0 { - // Unset left proof's path if _, ok := rn.Val.(valueNode); ok { + // The fork point is root node, unset the entire trie + if parent == nil { + return true, nil + } parent.(*fullNode).Children[left[pos-1]] = nil - return nil + return false, nil } - return unset(rn, rn.Val, left[pos:], len(rn.Key), false) + return false, unset(rn, rn.Val, left[pos:], len(rn.Key), false) } if shortForkLeft != 0 { - // Unset right proof's path. if _, ok := rn.Val.(valueNode); ok { + // The fork point is root node, unset the entire trie + if parent == nil { + return true, nil + } parent.(*fullNode).Children[right[pos-1]] = nil - return nil + return false, nil } - return unset(rn, rn.Val, right[pos:], len(rn.Key), true) + return false, unset(rn, rn.Val, right[pos:], len(rn.Key), true) } - return nil + return false, nil case *fullNode: // unset all internal nodes in the forkpoint for i := left[pos] + 1; i < right[pos]; i++ { rn.Children[i] = nil } if err := unset(rn, rn.Children[left[pos]], left[pos:], 1, false); err != nil { - return err + return false, err } if err := unset(rn, rn.Children[right[pos]], right[pos:], 1, true); err != nil { - return err + return false, err } - return nil + return false, nil default: panic(fmt.Sprintf("%T: invalid node: %v", n, n)) } @@ -560,7 +570,8 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } // Remove all internal references. All the removed parts should // be re-filled(or re-constructed) by the given leaves range. - if err := unsetInternal(root, firstKey, lastKey); err != nil { + empty, err := unsetInternal(root, firstKey, lastKey) + if err != nil { return nil, nil, nil, false, err } // Rebuild the trie with the leaf stream, the shape of trie @@ -570,6 +581,9 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key triedb = NewDatabase(diskdb) ) tr := &Trie{root: root, db: triedb} + if empty { + tr.root = nil + } for index, key := range keys { tr.TryUpdate(key, values[index]) } diff --git a/trie/proof_test.go b/trie/proof_test.go index 3ecd318886..304affa9f2 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -384,6 +384,25 @@ func TestOneElementRangeProof(t *testing.T) { if err != nil { t.Fatalf("Expected no error, got %v", err) } + + // Test the mini trie with only a single element. + tinyTrie := new(Trie) + entry := &kv{randBytes(32), randBytes(20), false} + tinyTrie.Update(entry.k, entry.v) + + first = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() + last = entry.k + proof = memorydb.New() + if err := tinyTrie.Prove(first, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := tinyTrie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + _, _, _, _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } } // TestAllElementsProof tests the range proof with all elements. From f26c19cbcd60175598824c7e09b64b8896daf721 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 22 Jan 2021 20:15:27 +0100 Subject: [PATCH 086/709] common/mclock: remove dependency on github.com/aristanetworks/goarista (#22211) It takes three lines of code to get to runtime.nanotime, no need to pull a dependency for that. --- common/mclock/mclock.go | 12 ++++++++---- common/mclock/mclock.s | 1 + go.mod | 1 - go.sum | 2 -- 4 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 common/mclock/mclock.s diff --git a/common/mclock/mclock.go b/common/mclock/mclock.go index 3aca257cb3..c05738cf2b 100644 --- a/common/mclock/mclock.go +++ b/common/mclock/mclock.go @@ -20,15 +20,19 @@ package mclock import ( "time" - "github.com/aristanetworks/goarista/monotime" + _ "unsafe" // for go:linkname ) +//go:noescape +//go:linkname nanotime runtime.nanotime +func nanotime() int64 + // AbsTime represents absolute monotonic time. -type AbsTime time.Duration +type AbsTime int64 // Now returns the current absolute monotonic time. func Now() AbsTime { - return AbsTime(monotime.Now()) + return AbsTime(nanotime()) } // Add returns t + d as absolute time. @@ -74,7 +78,7 @@ type System struct{} // Now returns the current monotonic time. func (c System) Now() AbsTime { - return AbsTime(monotime.Now()) + return Now() } // Sleep blocks for the given duration. diff --git a/common/mclock/mclock.s b/common/mclock/mclock.s new file mode 100644 index 0000000000..99a7a878f0 --- /dev/null +++ b/common/mclock/mclock.s @@ -0,0 +1 @@ +// This file exists in order to be able to use go:linkname. diff --git a/go.mod b/go.mod index d630dc6996..9162754ea3 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.5.7 - github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 github.com/aws/aws-sdk-go v1.25.48 github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 github.com/cespare/cp v0.1.0 diff --git a/go.sum b/go.sum index f8939df2ba..327af03735 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= From db35d77b63ab462990ad95481e116f9340d52de2 Mon Sep 17 00:00:00 2001 From: ligi Date: Sun, 24 Jan 2021 11:37:08 +0100 Subject: [PATCH 087/709] cmd, geth: CLI help fixes (#22220) * cmd, geth: Reflect command being optional - closes 22218 * cmd, geth: Set current year to 2021 --- internal/flags/helpers.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index f61ed4b689..eb5f5547b1 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -51,10 +51,10 @@ OPTIONS: AppHelpTemplate = `NAME: {{.App.Name}} - {{.App.Usage}} - Copyright 2013-2019 The go-ethereum Authors + Copyright 2013-2021 The go-ethereum Authors USAGE: - {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} + {{.App.HelpName}} [options]{{if .App.Commands}} [command] [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} {{if .App.Version}} VERSION: {{.App.Version}} @@ -77,7 +77,7 @@ COPYRIGHT: ClefAppHelpTemplate = `NAME: {{.App.Name}} - {{.App.Usage}} - Copyright 2013-2019 The go-ethereum Authors + Copyright 2013-2021 The go-ethereum Authors USAGE: {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} From 3708454f58923582585c5a231243324729011a39 Mon Sep 17 00:00:00 2001 From: ligi Date: Sun, 24 Jan 2021 11:37:39 +0100 Subject: [PATCH 088/709] cmd, geth: CLI help fixes (#22220) * cmd, geth: Reflect command being optional - closes 22218 * cmd, geth: Set current year to 2021 From 797b0812ab742b2a24e3c9277fa6564ff9eb9094 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 25 Jan 2021 07:17:05 +0100 Subject: [PATCH 089/709] eth/protocols/snap: snap sync testing (#22179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/protocols/snap: make timeout configurable * eth/protocols/snap: snap sync testing * eth/protocols/snap: test to trigger panic * eth/protocols/snap: fix race condition on timeouts * eth/protocols/snap: return error on cancelled sync * squashme: updates + test causing panic + properly serve accounts in order * eth/protocols/snap: revert failing storage response * eth/protocols/snap: revert on bad responses (storage, code) * eth/protocols/snap: fix account handling stall * eth/protocols/snap: fix remaining revertal-issues * eth/protocols/snap: timeouthandler for bytecode requests * eth/protocols/snap: debugging + fix log message * eth/protocols/snap: fix misspelliings in docs * eth/protocols/snap: fix race in bytecode handling * eth/protocols/snap: undo deduplication of storage roots * synctests: refactor + minify panic testcase * eth/protocols/snap: minor polishes * eth: minor polishes to make logs more useful * eth/protocols/snap: remove excessive logs from the test runs * eth/protocols/snap: stress tests with concurrency * eth/protocols/snap: further fixes to test cancel channel handling * eth/protocols/snap: extend test timeouts on CI Co-authored-by: Péter Szilágyi --- eth/downloader/downloader.go | 4 +- eth/handler.go | 16 +- eth/protocols/snap/peer.go | 5 + eth/protocols/snap/protocol.go | 1 + eth/protocols/snap/sync.go | 349 ++++++----- eth/protocols/snap/sync_test.go | 1020 +++++++++++++++++++++++++++++++ 6 files changed, 1248 insertions(+), 147 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 31c1cb47c3..421803876e 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -298,7 +298,7 @@ func (d *Downloader) RegisterPeer(id string, version uint, peer Peer) error { // Tests use short IDs, don't choke on them logger = log.New("peer", id) } else { - logger = log.New("peer", id[:16]) + logger = log.New("peer", id[:8]) } logger.Trace("Registering sync peer") if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil { @@ -325,7 +325,7 @@ func (d *Downloader) UnregisterPeer(id string) error { // Tests use short IDs, don't choke on them logger = log.New("peer", id) } else { - logger = log.New("peer", id[:16]) + logger = log.New("peer", id[:8]) } logger.Trace("Unregistering sync peer") if err := d.peers.Unregister(id); err != nil { diff --git a/eth/handler.go b/eth/handler.go index a9506c4995..f6366d9af1 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -326,24 +326,32 @@ func (h *handler) runSnapPeer(peer *snap.Peer, handler snap.Handler) error { } func (h *handler) removePeer(id string) { + // Create a custom logger to avoid printing the entire id + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:8]) + } // Remove the eth peer if it exists eth := h.peers.ethPeer(id) if eth != nil { - log.Debug("Removing Ethereum peer", "peer", id) + logger.Debug("Removing Ethereum peer") h.downloader.UnregisterPeer(id) h.txFetcher.Drop(id) if err := h.peers.unregisterEthPeer(id); err != nil { - log.Error("Peer removal failed", "peer", id, "err", err) + logger.Error("Ethereum peer removal failed", "err", err) } } // Remove the snap peer if it exists snap := h.peers.snapPeer(id) if snap != nil { - log.Debug("Removing Snapshot peer", "peer", id) + logger.Debug("Removing Snapshot peer") h.downloader.SnapSyncer.Unregister(id) if err := h.peers.unregisterSnapPeer(id); err != nil { - log.Error("Peer removal failed", "peer", id, "err", err) + logger.Error("Snapshot peer removel failed", "err", err) } } // Hard disconnect at the networking layer diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 73eaaadd09..4f3d550f1f 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -56,6 +56,11 @@ func (p *Peer) Version() uint { return p.version } +// Log overrides the P2P logget with the higher level one containing only the id. +func (p *Peer) Log() log.Logger { + return p.logger +} + // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes uint64) error { diff --git a/eth/protocols/snap/protocol.go b/eth/protocols/snap/protocol.go index a1e4349691..f1a25a2066 100644 --- a/eth/protocols/snap/protocol.go +++ b/eth/protocols/snap/protocol.go @@ -61,6 +61,7 @@ var ( errDecode = errors.New("invalid message") errInvalidMsgCode = errors.New("invalid message code") errBadRequest = errors.New("bad request") + errCancelled = errors.New("sync cancelled") ) // Packet represents a p2p message in the `snap` protocol. diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index d6f0eb5472..e7720026bf 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -73,10 +73,6 @@ const ( // waste bandwidth. maxTrieRequestCount = 512 - // requestTimeout is the maximum time a peer is allowed to spend on serving - // a single network request. - requestTimeout = 10 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? - // accountConcurrency is the number of chunks to split the account trie into // to allow concurrent retrievals. accountConcurrency = 16 @@ -86,6 +82,12 @@ const ( storageConcurrency = 16 ) +var ( + // requestTimeout is the maximum time a peer is allowed to spend on serving + // a single network request. + requestTimeout = 10 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? +) + // accountRequest tracks a pending account range request to ensure responses are // to actual requests and to validate any security constraints. // @@ -331,6 +333,33 @@ type syncProgress struct { BytecodeHealNops uint64 // Number of bytecodes not requested } +// SyncPeer abstracts out the methods required for a peer to be synced against +// with the goal of allowing the construction of mock peers without the full +// blown networking. +type SyncPeer interface { + // ID retrieves the peer's unique identifier. + ID() string + + // RequestAccountRange fetches a batch of accounts rooted in a specific account + // trie, starting with the origin. + RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error + + // RequestStorageRange fetches a batch of storage slots belonging to one or + // more accounts. If slots from only one accout is requested, an origin marker + // may also be used to retrieve from there. + RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error + + // RequestByteCodes fetches a batch of bytecodes by hash. + RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error + + // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in + // a specificstate trie. + RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error + + // Log retrieves the peer's own contextual logger. + Log() log.Logger +} + // Syncer is an Ethereum account and storage trie syncer based on snapshots and // the snap protocol. It's purpose is to download all the accounts and storage // slots from remote peers and reassemble chunks of the state trie, on top of @@ -346,14 +375,15 @@ type Syncer struct { db ethdb.KeyValueStore // Database to store the trie nodes into (and dedup) bloom *trie.SyncBloom // Bloom filter to deduplicate nodes for state fixup - root common.Hash // Current state trie root being synced - tasks []*accountTask // Current account task set being synced - healer *healTask // Current state healing task being executed - update chan struct{} // Notification channel for possible sync progression + root common.Hash // Current state trie root being synced + tasks []*accountTask // Current account task set being synced + snapped bool // Flag to signal that snap phase is done + healer *healTask // Current state healing task being executed + update chan struct{} // Notification channel for possible sync progression - peers map[string]*Peer // Currently active peers to download from - peerJoin *event.Feed // Event feed to react to peers joining - peerDrop *event.Feed // Event feed to react to peers dropping + peers map[string]SyncPeer // Currently active peers to download from + peerJoin *event.Feed // Event feed to react to peers joining + peerDrop *event.Feed // Event feed to react to peers dropping // Request tracking during syncing phase statelessPeers map[string]struct{} // Peers that failed to deliver state data @@ -410,12 +440,14 @@ type Syncer struct { lock sync.RWMutex // Protects fields that can change outside of sync (peers, reqs, root) } +// NewSyncer creates a new snapshot syncer to download the Ethereum state over the +// snap protocol. func NewSyncer(db ethdb.KeyValueStore, bloom *trie.SyncBloom) *Syncer { return &Syncer{ db: db, bloom: bloom, - peers: make(map[string]*Peer), + peers: make(map[string]SyncPeer), peerJoin: new(event.Feed), peerDrop: new(event.Feed), update: make(chan struct{}, 1), @@ -447,27 +479,29 @@ func NewSyncer(db ethdb.KeyValueStore, bloom *trie.SyncBloom) *Syncer { } // Register injects a new data source into the syncer's peerset. -func (s *Syncer) Register(peer *Peer) error { +func (s *Syncer) Register(peer SyncPeer) error { // Make sure the peer is not registered yet + id := peer.ID() + s.lock.Lock() - if _, ok := s.peers[peer.id]; ok { - log.Error("Snap peer already registered", "id", peer.id) + if _, ok := s.peers[id]; ok { + log.Error("Snap peer already registered", "id", id) s.lock.Unlock() return errors.New("already registered") } - s.peers[peer.id] = peer + s.peers[id] = peer // Mark the peer as idle, even if no sync is running - s.accountIdlers[peer.id] = struct{}{} - s.storageIdlers[peer.id] = struct{}{} - s.bytecodeIdlers[peer.id] = struct{}{} - s.trienodeHealIdlers[peer.id] = struct{}{} - s.bytecodeHealIdlers[peer.id] = struct{}{} + s.accountIdlers[id] = struct{}{} + s.storageIdlers[id] = struct{}{} + s.bytecodeIdlers[id] = struct{}{} + s.trienodeHealIdlers[id] = struct{}{} + s.bytecodeHealIdlers[id] = struct{}{} s.lock.Unlock() // Notify any active syncs that a new peer can be assigned data - s.peerJoin.Send(peer.id) + s.peerJoin.Send(id) return nil } @@ -566,6 +600,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { s.assignAccountTasks(cancel) s.assignBytecodeTasks(cancel) s.assignStorageTasks(cancel) + if len(s.tasks) == 0 { // Sync phase done, run heal phase s.assignTrienodeHealTasks(cancel) @@ -580,7 +615,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { case id := <-peerDrop: s.revertRequests(id) case <-cancel: - return nil + return errCancelled case req := <-s.accountReqFails: s.revertAccountRequest(req) @@ -622,6 +657,7 @@ func (s *Syncer) loadSyncStatus() { log.Debug("Scheduled account sync task", "from", task.Next, "last", task.Last) } s.tasks = progress.Tasks + s.snapped = len(s.tasks) == 0 s.accountSynced = progress.AccountSynced s.accountBytes = progress.AccountBytes @@ -701,6 +737,11 @@ func (s *Syncer) cleanAccountTasks() { i-- } } + if len(s.tasks) == 0 { + s.lock.Lock() + s.snapped = true + s.lock.Unlock() + } } // cleanStorageTasks iterates over all the account tasks and storage sub-tasks @@ -798,7 +839,7 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { delete(s.accountIdlers, idle) s.pend.Add(1) - go func(peer *Peer, root common.Hash) { + go func(peer SyncPeer, root common.Hash) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -885,7 +926,7 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { delete(s.bytecodeIdlers, idle) s.pend.Add(1) - go func(peer *Peer) { + go func(peer SyncPeer) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -962,7 +1003,6 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { // Found an incomplete storage chunk, schedule it accounts = append(accounts, account) roots = append(roots, st.root) - subtask = st break // Large contract chunks are downloaded individually } @@ -1010,7 +1050,7 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { delete(s.storageIdlers, idle) s.pend.Add(1) - go func(peer *Peer, root common.Hash) { + go func(peer SyncPeer, root common.Hash) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -1125,7 +1165,7 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { delete(s.trienodeHealIdlers, idle) s.pend.Add(1) - go func(peer *Peer, root common.Hash) { + go func(peer SyncPeer, root common.Hash) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -1223,7 +1263,7 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { delete(s.bytecodeHealIdlers, idle) s.pend.Add(1) - go func(peer *Peer) { + go func(peer SyncPeer) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -1522,7 +1562,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { break } } - // Itereate over all the accounts and assemble which ones need further sub- + // Iterate over all the accounts and assemble which ones need further sub- // filling before the entire account range can be persisted. res.task.needCode = make([]bool, len(res.accounts)) res.task.needState = make([]bool, len(res.accounts)) @@ -1566,7 +1606,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { } } // Delete any subtasks that have been aborted but not resumed. This may undo - // some progress if a newpeer gives us less accounts than an old one, but for + // some progress if a new peer gives us less accounts than an old one, but for // now we have to live with that. for hash := range res.task.SubTasks { if _, ok := resumed[hash]; !ok { @@ -1650,95 +1690,92 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { ) // Iterate over all the accounts and reconstruct their storage tries from the // delivered slots - delivered := make(map[common.Hash]bool) - for i := 0; i < len(res.hashes); i++ { - delivered[res.roots[i]] = true - } for i, account := range res.accounts { // If the account was not delivered, reschedule it if i >= len(res.hashes) { - if !delivered[res.roots[i]] { - res.mainTask.stateTasks[account] = res.roots[i] - } + res.mainTask.stateTasks[account] = res.roots[i] continue } // State was delivered, if complete mark as not needed any more, otherwise // mark the account as needing healing - for j, acc := range res.mainTask.res.accounts { - if res.roots[i] == acc.Root { - // If the packet contains multiple contract storage slots, all - // but the last are surely complete. The last contract may be - // chunked, so check it's continuation flag. - if res.subTask == nil && res.mainTask.needState[j] && (i < len(res.hashes)-1 || !res.cont) { - res.mainTask.needState[j] = false - res.mainTask.pend-- - } - // If the last contract was chunked, mark it as needing healing - // to avoid writing it out to disk prematurely. - if res.subTask == nil && !res.mainTask.needHeal[j] && i == len(res.hashes)-1 && res.cont { - res.mainTask.needHeal[j] = true - } - // If the last contract was chunked, we need to switch to large - // contract handling mode - if res.subTask == nil && i == len(res.hashes)-1 && res.cont { - // If we haven't yet started a large-contract retrieval, create - // the subtasks for it within the main account task - if tasks, ok := res.mainTask.SubTasks[account]; !ok { - var ( - next common.Hash - ) - step := new(big.Int).Sub( - new(big.Int).Div( - new(big.Int).Exp(common.Big2, common.Big256, nil), - big.NewInt(storageConcurrency), - ), common.Big1, - ) - for k := 0; k < storageConcurrency; k++ { - last := common.BigToHash(new(big.Int).Add(next.Big(), step)) - if k == storageConcurrency-1 { - // Make sure we don't overflow if the step is not a proper divisor - last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") - } - tasks = append(tasks, &storageTask{ - Next: next, - Last: last, - root: acc.Root, - }) - log.Debug("Created storage sync task", "account", account, "root", acc.Root, "from", next, "last", last) - next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) + for j, hash := range res.mainTask.res.hashes { + if account != hash { + continue + } + acc := res.mainTask.res.accounts[j] + + // If the packet contains multiple contract storage slots, all + // but the last are surely complete. The last contract may be + // chunked, so check it's continuation flag. + if res.subTask == nil && res.mainTask.needState[j] && (i < len(res.hashes)-1 || !res.cont) { + res.mainTask.needState[j] = false + res.mainTask.pend-- + } + // If the last contract was chunked, mark it as needing healing + // to avoid writing it out to disk prematurely. + if res.subTask == nil && !res.mainTask.needHeal[j] && i == len(res.hashes)-1 && res.cont { + res.mainTask.needHeal[j] = true + } + // If the last contract was chunked, we need to switch to large + // contract handling mode + if res.subTask == nil && i == len(res.hashes)-1 && res.cont { + // If we haven't yet started a large-contract retrieval, create + // the subtasks for it within the main account task + if tasks, ok := res.mainTask.SubTasks[account]; !ok { + var ( + next common.Hash + ) + step := new(big.Int).Sub( + new(big.Int).Div( + new(big.Int).Exp(common.Big2, common.Big256, nil), + big.NewInt(storageConcurrency), + ), common.Big1, + ) + for k := 0; k < storageConcurrency; k++ { + last := common.BigToHash(new(big.Int).Add(next.Big(), step)) + if k == storageConcurrency-1 { + // Make sure we don't overflow if the step is not a proper divisor + last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") } - res.mainTask.SubTasks[account] = tasks - - // Since we've just created the sub-tasks, this response - // is surely for the first one (zero origin) - res.subTask = tasks[0] + tasks = append(tasks, &storageTask{ + Next: next, + Last: last, + root: acc.Root, + }) + log.Debug("Created storage sync task", "account", account, "root", acc.Root, "from", next, "last", last) + next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) } + res.mainTask.SubTasks[account] = tasks + + // Since we've just created the sub-tasks, this response + // is surely for the first one (zero origin) + res.subTask = tasks[0] } - // If we're in large contract delivery mode, forward the subtask - if res.subTask != nil { - // Ensure the response doesn't overflow into the subsequent task - last := res.subTask.Last.Big() - for k, hash := range res.hashes[i] { - if hash.Big().Cmp(last) > 0 { - // Chunk overflown, cut off excess, but also update the boundary - for l := k; l < len(res.hashes[i]); l++ { - if err := res.tries[i].Prove(res.hashes[i][l][:], 0, res.overflow); err != nil { - panic(err) // Account range was already proven, what happened - } + } + // If we're in large contract delivery mode, forward the subtask + if res.subTask != nil { + // Ensure the response doesn't overflow into the subsequent task + last := res.subTask.Last.Big() + for k, hash := range res.hashes[i] { + if hash.Big().Cmp(last) > 0 { + // Chunk overflown, cut off excess, but also update the boundary + for l := k; l < len(res.hashes[i]); l++ { + if err := res.tries[i].Prove(res.hashes[i][l][:], 0, res.overflow); err != nil { + panic(err) // Account range was already proven, what happened } - res.hashes[i] = res.hashes[i][:k] - res.slots[i] = res.slots[i][:k] - res.cont = false // Mark range completed - break } - } - // Forward the relevant storage chunk (even if created just now) - if res.cont { - res.subTask.Next = common.BigToHash(new(big.Int).Add(res.hashes[i][len(res.hashes[i])-1].Big(), big.NewInt(1))) - } else { - res.subTask.done = true + res.hashes[i] = res.hashes[i][:k] + res.slots[i] = res.slots[i][:k] + res.cont = false // Mark range completed + break } } + // Forward the relevant storage chunk (even if created just now) + if res.cont { + res.subTask.Next = common.BigToHash(new(big.Int).Add(res.hashes[i][len(res.hashes[i])-1].Big(), big.NewInt(1))) + } else { + res.subTask.done = true + } } } // Iterate over all the reconstructed trie nodes and push them to disk @@ -1941,7 +1978,7 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { // OnAccounts is a callback method to invoke when a range of accounts are // received from a remote peer. -func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, accounts [][]byte, proof [][]byte) error { +func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, accounts [][]byte, proof [][]byte) error { size := common.StorageSize(len(hashes) * common.HashLength) for _, account := range accounts { size += common.StorageSize(len(account)) @@ -1949,15 +1986,15 @@ func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, account for _, node := range proof { size += common.StorageSize(len(node)) } - logger := peer.logger.New("reqid", id) + logger := peer.Log().New("reqid", id) logger.Trace("Delivering range of accounts", "hashes", len(hashes), "accounts", len(accounts), "proofs", len(proof), "bytes", size) // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. s.lock.Lock() - if _, ok := s.peers[peer.id]; ok { - s.accountIdlers[peer.id] = struct{}{} + if _, ok := s.peers[peer.ID()]; ok { + s.accountIdlers[peer.ID()] = struct{}{} } select { case s.update <- struct{}{}: @@ -1975,7 +2012,11 @@ func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, account // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content - req.timeout.Stop() + if !req.timeout.Stop() { + // The timeout is already triggered, and this request will be reverted+rescheduled + s.lock.Unlock() + return nil + } // Response is valid, but check if peer is signalling that it does not have // the requested data. For account range queries that means the state being @@ -1983,7 +2024,7 @@ func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, account // synced to our head. if len(hashes) == 0 && len(accounts) == 0 && len(proof) == 0 { logger.Debug("Peer rejected account range request", "root", s.root) - s.statelessPeers[peer.id] = struct{}{} + s.statelessPeers[peer.ID()] = struct{}{} s.lock.Unlock() // Signal this request as failed, and ready for rescheduling @@ -2011,6 +2052,8 @@ func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, account db, tr, notary, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) if err != nil { logger.Warn("Account range failed proof", "err", err) + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertAccountRequest(req) return err } // Partial trie reconstructed, send it to the scheduler for storage filling @@ -2050,9 +2093,9 @@ func (s *Syncer) OnAccounts(peer *Peer, id uint64, hashes []common.Hash, account // OnByteCodes is a callback method to invoke when a batch of contract // bytes codes are received from a remote peer. -func (s *Syncer) OnByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { +func (s *Syncer) OnByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error { s.lock.RLock() - syncing := len(s.tasks) > 0 + syncing := !s.snapped s.lock.RUnlock() if syncing { @@ -2063,20 +2106,20 @@ func (s *Syncer) OnByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { // onByteCodes is a callback method to invoke when a batch of contract // bytes codes are received from a remote peer in the syncing phase. -func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { +func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error { var size common.StorageSize for _, code := range bytecodes { size += common.StorageSize(len(code)) } - logger := peer.logger.New("reqid", id) + logger := peer.Log().New("reqid", id) logger.Trace("Delivering set of bytecodes", "bytecodes", len(bytecodes), "bytes", size) // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. s.lock.Lock() - if _, ok := s.peers[peer.id]; ok { - s.bytecodeIdlers[peer.id] = struct{}{} + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeIdlers[peer.ID()] = struct{}{} } select { case s.update <- struct{}{}: @@ -2094,14 +2137,18 @@ func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content - req.timeout.Stop() + if !req.timeout.Stop() { + // The timeout is already triggered, and this request will be reverted+rescheduled + s.lock.Unlock() + return nil + } // Response is valid, but check if peer is signalling that it does not have // the requested data. For bytecode range queries that means the peer is not // yet synced. if len(bytecodes) == 0 { logger.Debug("Peer rejected bytecode request") - s.statelessPeers[peer.id] = struct{}{} + s.statelessPeers[peer.ID()] = struct{}{} s.lock.Unlock() // Signal this request as failed, and ready for rescheduling @@ -2132,6 +2179,8 @@ func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected bytecodes", "count", len(bytecodes)-i) + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertBytecodeRequest(req) return errors.New("unexpected bytecode") } // Response validated, send it to the scheduler for filling @@ -2150,7 +2199,7 @@ func (s *Syncer) onByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { // OnStorage is a callback method to invoke when ranges of storage slots // are received from a remote peer. -func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots [][][]byte, proof [][]byte) error { +func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slots [][][]byte, proof [][]byte) error { // Gather some trace stats to aid in debugging issues var ( hashCount int @@ -2170,15 +2219,15 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots for _, node := range proof { size += common.StorageSize(len(node)) } - logger := peer.logger.New("reqid", id) + logger := peer.Log().New("reqid", id) logger.Trace("Delivering ranges of storage slots", "accounts", len(hashes), "hashes", hashCount, "slots", slotCount, "proofs", len(proof), "size", size) // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. s.lock.Lock() - if _, ok := s.peers[peer.id]; ok { - s.storageIdlers[peer.id] = struct{}{} + if _, ok := s.peers[peer.ID()]; ok { + s.storageIdlers[peer.ID()] = struct{}{} } select { case s.update <- struct{}{}: @@ -2196,17 +2245,23 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content - req.timeout.Stop() + if !req.timeout.Stop() { + // The timeout is already triggered, and this request will be reverted+rescheduled + s.lock.Unlock() + return nil + } // Reject the response if the hash sets and slot sets don't match, or if the // peer sent more data than requested. if len(hashes) != len(slots) { s.lock.Unlock() + s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Hash and slot set size mismatch", "hashset", len(hashes), "slotset", len(slots)) return errors.New("hash and slot set size mismatch") } if len(hashes) > len(req.accounts) { s.lock.Unlock() + s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Hash set larger than requested", "hashset", len(hashes), "requested", len(req.accounts)) return errors.New("hash set larger than requested") } @@ -2216,11 +2271,9 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots // synced to our head. if len(hashes) == 0 { logger.Debug("Peer rejected storage request") - s.statelessPeers[peer.id] = struct{}{} + s.statelessPeers[peer.ID()] = struct{}{} s.lock.Unlock() - - // Signal this request as failed, and ready for rescheduling - s.scheduleRevertStorageRequest(req) + s.scheduleRevertStorageRequest(req) // reschedule request return nil } s.lock.Unlock() @@ -2250,6 +2303,7 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots // space and hash to the origin root. dbs[i], tries[i], _, _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) if err != nil { + s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage slots failed proof", "err", err) return err } @@ -2264,6 +2318,7 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots } dbs[i], tries[i], notary, cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) if err != nil { + s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage range failed proof", "err", err) return err } @@ -2302,20 +2357,20 @@ func (s *Syncer) OnStorage(peer *Peer, id uint64, hashes [][]common.Hash, slots // OnTrieNodes is a callback method to invoke when a batch of trie nodes // are received from a remote peer. -func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { +func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error { var size common.StorageSize for _, node := range trienodes { size += common.StorageSize(len(node)) } - logger := peer.logger.New("reqid", id) + logger := peer.Log().New("reqid", id) logger.Trace("Delivering set of healing trienodes", "trienodes", len(trienodes), "bytes", size) // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. s.lock.Lock() - if _, ok := s.peers[peer.id]; ok { - s.trienodeHealIdlers[peer.id] = struct{}{} + if _, ok := s.peers[peer.ID()]; ok { + s.trienodeHealIdlers[peer.ID()] = struct{}{} } select { case s.update <- struct{}{}: @@ -2333,14 +2388,18 @@ func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content - req.timeout.Stop() + if !req.timeout.Stop() { + // The timeout is already triggered, and this request will be reverted+rescheduled + s.lock.Unlock() + return nil + } // Response is valid, but check if peer is signalling that it does not have // the requested data. For bytecode range queries that means the peer is not // yet synced. if len(trienodes) == 0 { logger.Debug("Peer rejected trienode heal request") - s.statelessPeers[peer.id] = struct{}{} + s.statelessPeers[peer.ID()] = struct{}{} s.lock.Unlock() // Signal this request as failed, and ready for rescheduling @@ -2371,6 +2430,8 @@ func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertTrienodeHealRequest(req) return errors.New("unexpected healing trienode") } // Response validated, send it to the scheduler for filling @@ -2390,20 +2451,20 @@ func (s *Syncer) OnTrieNodes(peer *Peer, id uint64, trienodes [][]byte) error { // onHealByteCodes is a callback method to invoke when a batch of contract // bytes codes are received from a remote peer in the healing phase. -func (s *Syncer) onHealByteCodes(peer *Peer, id uint64, bytecodes [][]byte) error { +func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error { var size common.StorageSize for _, code := range bytecodes { size += common.StorageSize(len(code)) } - logger := peer.logger.New("reqid", id) + logger := peer.Log().New("reqid", id) logger.Trace("Delivering set of healing bytecodes", "bytecodes", len(bytecodes), "bytes", size) // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. s.lock.Lock() - if _, ok := s.peers[peer.id]; ok { - s.bytecodeHealIdlers[peer.id] = struct{}{} + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeHealIdlers[peer.ID()] = struct{}{} } select { case s.update <- struct{}{}: @@ -2421,14 +2482,18 @@ func (s *Syncer) onHealByteCodes(peer *Peer, id uint64, bytecodes [][]byte) erro // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content - req.timeout.Stop() + if !req.timeout.Stop() { + // The timeout is already triggered, and this request will be reverted+rescheduled + s.lock.Unlock() + return nil + } // Response is valid, but check if peer is signalling that it does not have // the requested data. For bytecode range queries that means the peer is not // yet synced. if len(bytecodes) == 0 { logger.Debug("Peer rejected bytecode heal request") - s.statelessPeers[peer.id] = struct{}{} + s.statelessPeers[peer.ID()] = struct{}{} s.lock.Unlock() // Signal this request as failed, and ready for rescheduling @@ -2459,6 +2524,8 @@ func (s *Syncer) onHealByteCodes(peer *Peer, id uint64, bytecodes [][]byte) erro } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing bytecodes", "count", len(bytecodes)-i) + // Signal this request as failed, and ready for rescheduling + s.scheduleRevertBytecodeHealRequest(req) return errors.New("unexpected healing bytecode") } // Response validated, send it to the scheduler for filling diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 4f28b99bfe..0b048786e8 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -17,15 +17,29 @@ package snap import ( + "bytes" "crypto/rand" + "encoding/binary" "fmt" + "math/big" + "sort" "testing" + "time" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" ) func TestHashing(t *testing.T) { + t.Parallel() + var bytecodes = make([][]byte, 10) for i := 0; i < len(bytecodes); i++ { buf := make([]byte, 100) @@ -96,3 +110,1009 @@ func BenchmarkHashing(b *testing.B) { } }) } + +type storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error +type accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error +type trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error +type codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error + +type testPeer struct { + id string + test *testing.T + remote *Syncer + logger log.Logger + accountTrie *trie.Trie + accountValues entrySlice + storageTries map[common.Hash]*trie.Trie + storageValues map[common.Hash]entrySlice + + accountRequestHandler accountHandlerFunc + storageRequestHandler storageHandlerFunc + trieRequestHandler trieHandlerFunc + codeRequestHandler codeHandlerFunc + cancelCh chan struct{} +} + +func newTestPeer(id string, t *testing.T, cancelCh chan struct{}) *testPeer { + peer := &testPeer{ + id: id, + test: t, + logger: log.New("id", id), + accountRequestHandler: defaultAccountRequestHandler, + trieRequestHandler: defaultTrieRequestHandler, + storageRequestHandler: defaultStorageRequestHandler, + codeRequestHandler: defaultCodeRequestHandler, + cancelCh: cancelCh, + } + //stderrHandler := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) + //peer.logger.SetHandler(stderrHandler) + return peer + +} + +func (t *testPeer) ID() string { return t.id } +func (t *testPeer) Log() log.Logger { return t.logger } + +func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { + t.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) + go t.accountRequestHandler(t, id, root, origin, bytes) + return nil +} + +func (t *testPeer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { + t.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) + go t.trieRequestHandler(t, id, root, paths, bytes) + return nil +} + +func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { + if len(accounts) == 1 && origin != nil { + t.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) + } else { + t.logger.Trace("Fetching ranges of small storage slots", "reqid", id, "root", root, "accounts", len(accounts), "first", accounts[0], "bytes", common.StorageSize(bytes)) + } + go t.storageRequestHandler(t, id, root, accounts, origin, limit, bytes) + return nil +} + +func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { + t.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) + go t.codeRequestHandler(t, id, hashes, bytes) + return nil +} + +// defaultTrieRequestHandler is a well-behaving handler for trie healing requests +func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { + // Pass the response + var nodes [][]byte + for _, pathset := range paths { + switch len(pathset) { + case 1: + blob, _, err := t.accountTrie.TryGetNode(pathset[0]) + if err != nil { + t.logger.Info("Error handling req", "error", err) + break + } + nodes = append(nodes, blob) + default: + account := t.storageTries[(common.BytesToHash(pathset[0]))] + for _, path := range pathset[1:] { + blob, _, err := account.TryGetNode(path) + if err != nil { + t.logger.Info("Error handling req", "error", err) + break + } + nodes = append(nodes, blob) + } + } + } + t.remote.OnTrieNodes(t, requestId, nodes) + return nil +} + +// defaultAccountRequestHandler is a well-behaving handler for AccountRangeRequests +func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, cap uint64) error { + keys, vals, proofs := createAccountRequestResponse(t, root, origin, cap) + if err := t.remote.OnAccounts(t, id, keys, vals, proofs); err != nil { + t.logger.Error("remote error on delivery", "error", err) + t.test.Errorf("Remote side rejected our delivery: %v", err) + t.remote.Unregister(t.id) + close(t.cancelCh) + return err + } + return nil +} + +func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, cap uint64) (keys []common.Hash, vals [][]byte, proofs [][]byte) { + var size uint64 + for _, entry := range t.accountValues { + if size > cap { + break + } + if bytes.Compare(origin[:], entry.k) <= 0 { + keys = append(keys, common.BytesToHash(entry.k)) + vals = append(vals, entry.v) + size += uint64(32 + len(entry.v)) + } + } + // Unless we send the entire trie, we need to supply proofs + // Actually, we need to supply proofs either way! This seems tob be an implementation + // quirk in go-ethereum + proof := light.NewNodeSet() + if err := t.accountTrie.Prove(origin[:], 0, proof); err != nil { + t.logger.Error("Could not prove inexistence of origin", "origin", origin, + "error", err) + } + if len(keys) > 0 { + lastK := (keys[len(keys)-1])[:] + if err := t.accountTrie.Prove(lastK, 0, proof); err != nil { + t.logger.Error("Could not prove last item", + "error", err) + } + } + for _, blob := range proof.NodeList() { + proofs = append(proofs, blob) + } + return keys, vals, proofs +} + +// defaultStorageRequestHandler is a well-behaving storage request handler +func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) error { + hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, bOrigin, bLimit, max) + if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { + t.logger.Error("remote error on delivery", "error", err) + t.test.Errorf("Remote side rejected our delivery: %v", err) + close(t.cancelCh) + } + return nil +} + +func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { + var bytecodes [][]byte + for _, h := range hashes { + bytecodes = append(bytecodes, getCode(h)) + } + if err := t.remote.OnByteCodes(t, id, bytecodes); err != nil { + t.logger.Error("remote error on delivery", "error", err) + t.test.Errorf("Remote side rejected our delivery: %v", err) + close(t.cancelCh) + } + return nil +} + +func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var ( + size uint64 + limit = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + ) + if len(bLimit) > 0 { + limit = common.BytesToHash(bLimit) + } + var origin common.Hash + if len(bOrigin) > 0 { + origin = common.BytesToHash(bOrigin) + } + + var limitExceeded bool + var incomplete bool + for _, account := range accounts { + + var keys []common.Hash + var vals [][]byte + for _, entry := range t.storageValues[account] { + if limitExceeded { + incomplete = true + break + } + if bytes.Compare(entry.k, origin[:]) < 0 { + incomplete = true + continue + } + keys = append(keys, common.BytesToHash(entry.k)) + vals = append(vals, entry.v) + size += uint64(32 + len(entry.v)) + if bytes.Compare(entry.k, limit[:]) >= 0 { + limitExceeded = true + } + if size > max { + limitExceeded = true + } + } + hashes = append(hashes, keys) + slots = append(slots, vals) + + if incomplete { + // If we're aborting, we need to prove the first and last item + // This terminates the response (and thus the loop) + proof := light.NewNodeSet() + stTrie := t.storageTries[account] + + // Here's a potential gotcha: when constructing the proof, we cannot + // use the 'origin' slice directly, but must use the full 32-byte + // hash form. + if err := stTrie.Prove(origin[:], 0, proof); err != nil { + t.logger.Error("Could not prove inexistence of origin", "origin", origin, + "error", err) + } + if len(keys) > 0 { + lastK := (keys[len(keys)-1])[:] + if err := stTrie.Prove(lastK, 0, proof); err != nil { + t.logger.Error("Could not prove last item", "error", err) + } + } + for _, blob := range proof.NodeList() { + proofs = append(proofs, blob) + } + break + } + } + return hashes, slots, proofs +} + +// emptyRequestAccountRangeFn is a rejects AccountRangeRequests +func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { + var proofs [][]byte + var keys []common.Hash + var vals [][]byte + t.remote.OnAccounts(t, requestId, keys, vals, proofs) + return nil +} + +func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { + return nil +} + +func emptyTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { + var nodes [][]byte + t.remote.OnTrieNodes(t, requestId, nodes) + return nil +} + +func nonResponsiveTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { + return nil +} + +func emptyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + var hashes [][]common.Hash + var slots [][][]byte + var proofs [][]byte + t.remote.OnStorage(t, requestId, hashes, slots, proofs) + return nil +} + +func nonResponsiveStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + return nil +} + +//func emptyCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +// var bytecodes [][]byte +// t.remote.OnByteCodes(t, id, bytecodes) +// return nil +//} + +func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { + var bytecodes [][]byte + for _, h := range hashes { + // Send back the hashes + bytecodes = append(bytecodes, h[:]) + } + if err := t.remote.OnByteCodes(t, id, bytecodes); err != nil { + t.logger.Error("remote error on delivery", "error", err) + // Mimic the real-life handler, which drops a peer on errors + t.remote.Unregister(t.id) + } + return nil +} + +func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { + var bytecodes [][]byte + for _, h := range hashes[:1] { + bytecodes = append(bytecodes, getCode(h)) + } + if err := t.remote.OnByteCodes(t, id, bytecodes); err != nil { + t.logger.Error("remote error on delivery", "error", err) + // Mimic the real-life handler, which drops a peer on errors + t.remote.Unregister(t.id) + } + return nil +} + +// starvingStorageRequestHandler is somewhat well-behaving storage handler, but it caps the returned results to be very small +func starvingStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + return defaultStorageRequestHandler(t, requestId, root, accounts, origin, limit, 500) +} + +func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { + return defaultAccountRequestHandler(t, requestId, root, origin, 500) +} + +//func misdeliveringAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { +// return defaultAccountRequestHandler(t, requestId-1, root, origin, 500) +//} + +func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { + hashes, accounts, proofs := createAccountRequestResponse(t, root, origin, cap) + if len(proofs) > 0 { + proofs = proofs[1:] + } + if err := t.remote.OnAccounts(t, requestId, hashes, accounts, proofs); err != nil { + t.logger.Info("remote error on delivery (as expected)", "error", err) + // Mimic the real-life handler, which drops a peer on errors + t.remote.Unregister(t.id) + } + return nil +} + +// corruptStorageRequestHandler doesn't provide good proofs +func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, origin, limit, max) + if len(proofs) > 0 { + proofs = proofs[1:] + } + if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { + t.logger.Info("remote error on delivery (as expected)", "error", err) + // Mimic the real-life handler, which drops a peer on errors + t.remote.Unregister(t.id) + } + return nil +} + +func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + hashes, slots, _ := createStorageRequestResponse(t, root, accounts, origin, limit, max) + if err := t.remote.OnStorage(t, requestId, hashes, slots, nil); err != nil { + t.logger.Info("remote error on delivery (as expected)", "error", err) + // Mimic the real-life handler, which drops a peer on errors + t.remote.Unregister(t.id) + } + return nil +} + +// TestSyncBloatedProof tests a scenario where we provide only _one_ value, but +// also ship the entire trie inside the proof. If the attack is successful, +// the remote side does not do any follow-up requests +func TestSyncBloatedProof(t *testing.T) { + t.Parallel() + + sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + cancel := make(chan struct{}) + source := newTestPeer("source", t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + + source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { + var proofs [][]byte + var keys []common.Hash + var vals [][]byte + + // The values + for _, entry := range t.accountValues { + if bytes.Compare(origin[:], entry.k) <= 0 { + keys = append(keys, common.BytesToHash(entry.k)) + vals = append(vals, entry.v) + } + } + // The proofs + proof := light.NewNodeSet() + if err := t.accountTrie.Prove(origin[:], 0, proof); err != nil { + t.logger.Error("Could not prove origin", "origin", origin, "error", err) + } + // The bloat: add proof of every single element + for _, entry := range t.accountValues { + if err := t.accountTrie.Prove(entry.k, 0, proof); err != nil { + t.logger.Error("Could not prove item", "error", err) + } + } + // And remove one item from the elements + if len(keys) > 2 { + keys = append(keys[:1], keys[2:]...) + vals = append(vals[:1], vals[2:]...) + } + for _, blob := range proof.NodeList() { + proofs = append(proofs, blob) + } + if err := t.remote.OnAccounts(t, requestId, keys, vals, proofs); err != nil { + t.logger.Info("remote error on delivery", "error", err) + // This is actually correct, signal to exit the test successfully + close(t.cancelCh) + } + return nil + } + syncer := setupSyncer(source) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err == nil { + t.Fatal("No error returned from incomplete/cancelled sync") + } +} + +func setupSyncer(peers ...*testPeer) *Syncer { + stateDb := rawdb.NewMemoryDatabase() + syncer := NewSyncer(stateDb, trie.NewSyncBloom(1, stateDb)) + for _, peer := range peers { + syncer.Register(peer) + peer.remote = syncer + } + return syncer +} + +// TestSync tests a basic sync with one peer +func TestSync(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + return source + } + + syncer := setupSyncer(mkSource("sourceA")) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } +} + +// TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a +// panic within the prover +func TestSyncTinyTriePanic(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems := makeAccountTrieNoStorage(1) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + return source + } + + syncer := setupSyncer( + mkSource("nice-a"), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +// TestMultiSync tests a basic sync with multiple peers +func TestMultiSync(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + return source + } + + syncer := setupSyncer(mkSource("sourceA"), mkSource("sourceB")) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } +} + +// TestSyncWithStorage tests basic sync using accounts + storage + code +func TestSyncWithStorage(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + return source + } + syncer := setupSyncer(mkSource("sourceA")) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } +} + +// TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all +func TestMultiSyncManyUseless(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + + mkSource := func(name string, a, b, c bool) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + + if !a { + source.accountRequestHandler = emptyRequestAccountRangeFn + } + if !b { + source.storageRequestHandler = emptyStorageRequestHandler + } + if !c { + source.trieRequestHandler = emptyTrieRequestHandler + } + return source + } + + syncer := setupSyncer( + mkSource("full", true, true, true), + mkSource("noAccounts", false, true, true), + mkSource("noStorage", true, false, true), + mkSource("noTrie", true, true, false), + ) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } +} + +// TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all +func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { + // We're setting the timeout to very low, to increase the chance of the timeout + // being triggered. This was previously a cause of panic, when a response + // arrived simultaneously as a timeout was triggered. + defer func(old time.Duration) { requestTimeout = old }(requestTimeout) + requestTimeout = time.Millisecond + + cancel := make(chan struct{}) + + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + + mkSource := func(name string, a, b, c bool) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + + if !a { + source.accountRequestHandler = emptyRequestAccountRangeFn + } + if !b { + source.storageRequestHandler = emptyStorageRequestHandler + } + if !c { + source.trieRequestHandler = emptyTrieRequestHandler + } + return source + } + + syncer := setupSyncer( + mkSource("full", true, true, true), + mkSource("noAccounts", false, true, true), + mkSource("noStorage", true, false, true), + mkSource("noTrie", true, true, false), + ) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } +} + +// TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all +func TestMultiSyncManyUnresponsive(t *testing.T) { + // We're setting the timeout to very low, to make the test run a bit faster + defer func(old time.Duration) { requestTimeout = old }(requestTimeout) + requestTimeout = time.Millisecond + + cancel := make(chan struct{}) + + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + + mkSource := func(name string, a, b, c bool) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + + if !a { + source.accountRequestHandler = nonResponsiveRequestAccountRangeFn + } + if !b { + source.storageRequestHandler = nonResponsiveStorageRequestHandler + } + if !c { + source.trieRequestHandler = nonResponsiveTrieRequestHandler + } + return source + } + + syncer := setupSyncer( + mkSource("full", true, true, true), + mkSource("noAccounts", false, true, true), + mkSource("noStorage", true, false, true), + mkSource("noTrie", true, true, false), + ) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } +} + +func checkStall(t *testing.T, cancel chan struct{}) chan struct{} { + testDone := make(chan struct{}) + go func() { + select { + case <-time.After(time.Minute): // TODO(karalabe): Make tests smaller, this is too much + t.Log("Sync stalled") + close(cancel) + case <-testDone: + return + } + }() + return testDone +} + +// TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is +// consistently returning very small results +func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + + mkSource := func(name string, slow bool) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + + if slow { + source.accountRequestHandler = starvingAccountRequestHandler + } + return source + } + + syncer := setupSyncer( + mkSource("nice-a", false), + mkSource("nice-b", false), + mkSource("nice-c", false), + mkSource("capped", true), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +// TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver +// code requests properly. +func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + + mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.codeRequestHandler = codeFn + return source + } + // One is capped, one is corrupt. If we don't use a capped one, there's a 50% + // chance that the full set of codes requested are sent only to the + // non-corrupt peer, which delivers everything in one go, and makes the + // test moot + syncer := setupSyncer( + mkSource("capped", cappedCodeRequestHandler), + mkSource("corrupt", corruptCodeRequestHandler), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + + mkSource := func(name string, accFn accountHandlerFunc) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.accountRequestHandler = accFn + return source + } + // One is capped, one is corrupt. If we don't use a capped one, there's a 50% + // chance that the full set of codes requested are sent only to the + // non-corrupt peer, which delivers everything in one go, and makes the + // test moot + syncer := setupSyncer( + mkSource("capped", defaultAccountRequestHandler), + mkSource("corrupt", corruptAccountRequestHandler), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +// TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes +// one by one +func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + + mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.codeRequestHandler = codeFn + return source + } + // Count how many times it's invoked. Remember, there are only 8 unique hashes, + // so it shouldn't be more than that + var counter int + syncer := setupSyncer( + mkSource("capped", func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { + counter++ + return cappedCodeRequestHandler(t, id, hashes, max) + }), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) + // There are only 8 unique hashes, and 3K accounts. However, the code + // deduplication is per request batch. If it were a perfect global dedup, + // we would expect only 8 requests. If there were no dedup, there would be + // 3k requests. + // We expect somewhere below 100 requests for these 8 unique hashes. + if threshold := 100; counter > threshold { + t.Fatalf("Error, expected < %d invocations, got %d", threshold, counter) + } +} + +// TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is +// consistently returning very small results +func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false) + + mkSource := func(name string, slow bool) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + + if slow { + source.storageRequestHandler = starvingStorageRequestHandler + } + return source + } + + syncer := setupSyncer( + mkSource("nice-a", false), + mkSource("slow", true), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +// TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is +// sometimes sending bad proofs +func TestSyncWithStorageAndCorruptPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + + mkSource := func(name string, handler storageHandlerFunc) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + source.storageRequestHandler = handler + return source + } + + syncer := setupSyncer( + mkSource("nice-a", defaultStorageRequestHandler), + mkSource("nice-b", defaultStorageRequestHandler), + mkSource("nice-c", defaultStorageRequestHandler), + mkSource("corrupt", corruptStorageRequestHandler), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { + t.Parallel() + + cancel := make(chan struct{}) + + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + + mkSource := func(name string, handler storageHandlerFunc) *testPeer { + source := newTestPeer(name, t, cancel) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + source.storageRequestHandler = handler + return source + } + + syncer := setupSyncer( + mkSource("nice-a", defaultStorageRequestHandler), + mkSource("nice-b", defaultStorageRequestHandler), + mkSource("nice-c", defaultStorageRequestHandler), + mkSource("corrupt", noProofStorageRequestHandler), + ) + done := checkStall(t, cancel) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) +} + +type kv struct { + k, v []byte + t bool +} + +// Some helpers for sorting +type entrySlice []*kv + +func (p entrySlice) Len() int { return len(p) } +func (p entrySlice) Less(i, j int) bool { return bytes.Compare(p[i].k, p[j].k) < 0 } +func (p entrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func key32(i uint64) []byte { + key := make([]byte, 32) + binary.LittleEndian.PutUint64(key, i) + return key +} + +var ( + codehashes = []common.Hash{ + crypto.Keccak256Hash([]byte{0}), + crypto.Keccak256Hash([]byte{1}), + crypto.Keccak256Hash([]byte{2}), + crypto.Keccak256Hash([]byte{3}), + crypto.Keccak256Hash([]byte{4}), + crypto.Keccak256Hash([]byte{5}), + crypto.Keccak256Hash([]byte{6}), + crypto.Keccak256Hash([]byte{7}), + } +) + +// getACodeHash returns a pseudo-random code hash +func getACodeHash(i uint64) []byte { + h := codehashes[int(i)%len(codehashes)] + return common.CopyBytes(h[:]) +} + +// convenience function to lookup the code from the code hash +func getCode(hash common.Hash) []byte { + if hash == emptyCode { + return nil + } + for i, h := range codehashes { + if h == hash { + return []byte{byte(i)} + } + } + return nil +} + +// makeAccountTrieNoStorage spits out a trie, along with the leafs +func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { + db := trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie, _ := trie.New(common.Hash{}, db) + var entries entrySlice + for i := uint64(1); i <= uint64(n); i++ { + value, _ := rlp.EncodeToBytes(state.Account{ + Nonce: i, + Balance: big.NewInt(int64(i)), + Root: emptyRoot, + CodeHash: getACodeHash(i), + }) + key := key32(i) + elem := &kv{key, value, false} + accTrie.Update(elem.k, elem.v) + entries = append(entries, elem) + } + sort.Sort(entries) + // Push to disk layer + accTrie.Commit(nil) + return accTrie, entries +} + +// makeAccountTrieWithStorage spits out a trie, along with the leafs +func makeAccountTrieWithStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, + map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { + + var ( + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie, _ = trie.New(common.Hash{}, db) + entries entrySlice + storageTries = make(map[common.Hash]*trie.Trie) + storageEntries = make(map[common.Hash]entrySlice) + ) + + // Make a storage trie which we reuse for the whole lot + stTrie, stEntries := makeStorageTrie(slots, db) + stRoot := stTrie.Hash() + // Create n accounts in the trie + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + codehash := emptyCode[:] + if code { + codehash = getACodeHash(i) + } + value, _ := rlp.EncodeToBytes(state.Account{ + Nonce: i, + Balance: big.NewInt(int64(i)), + Root: stRoot, + CodeHash: codehash, + }) + elem := &kv{key, value, false} + accTrie.Update(elem.k, elem.v) + entries = append(entries, elem) + // we reuse the same one for all accounts + storageTries[common.BytesToHash(key)] = stTrie + storageEntries[common.BytesToHash(key)] = stEntries + } + sort.Sort(entries) + stTrie.Commit(nil) + accTrie.Commit(nil) + return accTrie, entries, storageTries, storageEntries +} + +// makeStorageTrie fills a storage trie with n items, returning the +// not-yet-committed trie and the sorted entries +func makeStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { + trie, _ := trie.New(common.Hash{}, db) + var entries entrySlice + for i := uint64(1); i <= uint64(n); i++ { + // store 'i' at slot 'i' + slotValue := key32(i) + rlpSlotValue, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(slotValue[:])) + + slotKey := key32(i) + key := crypto.Keccak256Hash(slotKey[:]) + + elem := &kv{key[:], rlpSlotValue, false} + trie.Update(elem.k, elem.v) + entries = append(entries, elem) + } + sort.Sort(entries) + return trie, entries +} From 1770fe718af661334391766455c43157e378b9fa Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 25 Jan 2021 10:42:07 +0100 Subject: [PATCH 090/709] go.mod: update dependencies (#22216) This updates go module dependencies as discussed in #22050. --- go.mod | 22 ++-- go.sum | 346 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 338 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 9162754ea3..d3c75420b8 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.5.7 github.com/aws/aws-sdk-go v1.25.48 - github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 + github.com/btcsuite/btcd v0.20.1-beta github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 github.com/davecgh/go-spew v1.1.1 @@ -17,28 +17,27 @@ require ( github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 - github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c + github.com/edsrzf/mmap-go v1.0.0 github.com/fatih/color v1.3.0 - github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc + github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 - github.com/golang/protobuf v1.4.2 + github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa - github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 - github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 + github.com/gorilla/websocket v1.4.2 + github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 github.com/huin/goupnp v1.0.0 - github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 + github.com/influxdata/influxdb v1.8.3 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e - github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 + github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 - github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 @@ -49,11 +48,10 @@ require ( github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 github.com/rjeczalik/notify v0.9.1 - github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 - github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect + github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v2.20.5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 diff --git a/go.sum b/go.sum index 327af03735..98420f89f9 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,23 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -20,29 +40,61 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -50,6 +102,7 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= @@ -57,30 +110,53 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmak github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -88,10 +164,17 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -99,10 +182,20 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= +github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -114,18 +207,47 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo8u40n2JMnyAsd6x7+SbvoOMHvQOU/n10P4= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -135,17 +257,26 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= @@ -157,75 +288,178 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -233,17 +467,82 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -255,6 +554,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= @@ -265,10 +565,20 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From c0862f4f4c21c0ff54636d5122f09f310133d504 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 25 Jan 2021 11:31:18 +0100 Subject: [PATCH 091/709] graphql: change receipt status to decimal instead of hex (#22187) This PR fixes the receipt status field to be decimal instead of a hex string, as called for by the spec. Co-authored-by: Martin Holst Swende --- graphql/graphql.go | 16 ++++++++-------- graphql/graphql_test.go | 6 ++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index ea587106b4..5dc0f723d3 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -292,12 +292,12 @@ func (t *Transaction) getReceipt(ctx context.Context) (*types.Receipt, error) { return receipts[t.index], nil } -func (t *Transaction) Status(ctx context.Context) (*hexutil.Uint64, error) { +func (t *Transaction) Status(ctx context.Context) (*Long, error) { receipt, err := t.getReceipt(ctx) if err != nil || receipt == nil { return nil, err } - ret := hexutil.Uint64(receipt.Status) + ret := Long(receipt.Status) return &ret, nil } @@ -810,9 +810,9 @@ type CallData struct { // CallResult encapsulates the result of an invocation of the `call` accessor. type CallResult struct { - data hexutil.Bytes // The return data from the call - gasUsed Long // The amount of gas used - status hexutil.Uint64 // The return status of the call - 0 for failure or 1 for success. + data hexutil.Bytes // The return data from the call + gasUsed Long // The amount of gas used + status Long // The return status of the call - 0 for failure or 1 for success. } func (c *CallResult) Data() hexutil.Bytes { @@ -823,7 +823,7 @@ func (c *CallResult) GasUsed() Long { return c.gasUsed } -func (c *CallResult) Status() hexutil.Uint64 { +func (c *CallResult) Status() Long { return c.status } @@ -840,7 +840,7 @@ func (b *Block) Call(ctx context.Context, args struct { if err != nil { return nil, err } - status := hexutil.Uint64(1) + status := Long(1) if result.Failed() { status = 0 } @@ -910,7 +910,7 @@ func (p *Pending) Call(ctx context.Context, args struct { if err != nil { return nil, err } - status := hexutil.Uint64(1) + status := Long(1) if result.Failed() { status = 0 } diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index e9c129c44c..71320012d5 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -130,6 +130,12 @@ func TestGraphQLBlockSerialization(t *testing.T) { want: `{"data":{"block":{"estimateGas":53000}}}`, code: 200, }, + // should return `status` as decimal + { + body: `{"query": "{block {number call (data : {from : \"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b\", to: \"0x6295ee1b4f6dd65047762f924ecd367c17eabf8f\", data :\"0x12a7b914\"}){data status}}}"}`, + want: `{"data":{"block":{"number":10,"call":{"data":"0x","status":1}}}}`, + code: 200, + }, } { resp, err := http.Post(fmt.Sprintf("%s/graphql", stack.HTTPEndpoint()), "application/json", strings.NewReader(tt.body)) if err != nil { From 59a79137b9c8740a3623cdf35512e1eb49c8a20b Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 25 Jan 2021 19:46:09 +0800 Subject: [PATCH 092/709] go.mod: upgrade github.com/huin/goupnp (#22227) This updates the goupnp dependency, fixing huin/goupnp#33 --- go.mod | 2 +- go.sum | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index d3c75420b8..ffea94f6cf 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 - github.com/huin/goupnp v1.0.0 + github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 github.com/influxdata/influxdb v1.8.3 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e diff --git a/go.sum b/go.sum index 98420f89f9..fefe5afb14 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 h1:HarGZ5h9HD9LgEg1yRVMXyfiw4wlXiLiYM2oMjeA/SE= +github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -441,6 +441,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -460,7 +462,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -468,7 +469,6 @@ golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kY golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -502,7 +502,6 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -555,7 +554,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= @@ -566,7 +564,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= From 04a72260c5bcca0ec7c4a63532fb29f68db03384 Mon Sep 17 00:00:00 2001 From: Melvin Junhee Woo Date: Mon, 25 Jan 2021 22:25:55 +0900 Subject: [PATCH 093/709] snapshot: merge loops for better performance (#22160) --- core/state/snapshot/difflayer.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index cc82df9a54..9c86a679d1 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -191,19 +191,15 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s if blob == nil { panic(fmt.Sprintf("account %#x nil", accountHash)) } + // Determine memory size and track the dirty writes + dl.memory += uint64(common.HashLength + len(blob)) + snapshotDirtyAccountWriteMeter.Mark(int64(len(blob))) } for accountHash, slots := range storage { if slots == nil { panic(fmt.Sprintf("storage %#x nil", accountHash)) } - } - // Determine memory size and track the dirty writes - for _, data := range accounts { - dl.memory += uint64(common.HashLength + len(data)) - snapshotDirtyAccountWriteMeter.Mark(int64(len(data))) - } - // Determine memory size and track the dirty writes - for _, slots := range storage { + // Determine memory size and track the dirty writes for _, data := range slots { dl.memory += uint64(common.HashLength + len(data)) snapshotDirtyStorageWriteMeter.Mark(int64(len(data))) From 49cdcf5c70735dc85bd9c22b45811a3ec7cef54d Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 25 Jan 2021 21:29:45 +0800 Subject: [PATCH 094/709] core: reset to genesis when middle block is missing (#22135) When a sethead/rewind finds that the targeted block is missing, it resets to genesis instead of crashing. Closes #22129 --- core/blockchain.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d6668cdcd2..c05ebfd549 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -524,8 +524,13 @@ func (bc *BlockChain) SetHeadBeyondRoot(head uint64, root common.Hash) (uint64, if _, err := state.New(newHeadBlock.Root(), bc.stateCache, bc.snaps); err != nil { log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) if pivot == nil || newHeadBlock.NumberU64() > *pivot { - newHeadBlock = bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) - continue + parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) + if parent != nil { + newHeadBlock = parent + continue + } + log.Error("Missing block in the middle, aiming genesis", "number", newHeadBlock.NumberU64()-1, "hash", newHeadBlock.ParentHash()) + newHeadBlock = bc.genesisBlock } else { log.Trace("Rewind passed pivot, aiming genesis", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "pivot", *pivot) newHeadBlock = bc.genesisBlock From adf130def83fbf7b7902ff4bacab7bb369517dcb Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 25 Jan 2021 21:36:39 +0800 Subject: [PATCH 095/709] eth/tracers: move tracing APIs into eth/tracers (#22161) This moves the tracing RPC API implementation to package eth/tracers. By doing so, package eth no longer depends on tracing and the duktape JS engine. The change also enables tracing using the light client. All tracing methods work with the light client, but it's a lot slower compared to using a full node. --- cmd/utils/flags.go | 3 + eth/api.go | 3 +- eth/api_backend.go | 12 + eth/state_accessor.go | 230 +++++++++++ eth/{api_tracer.go => tracers/api.go} | 558 +++++++++++--------------- eth/tracers/api_test.go | 487 ++++++++++++++++++++++ les/api_backend.go | 12 + les/state_accessor.go | 88 ++++ 8 files changed, 1071 insertions(+), 322 deletions(-) create mode 100644 eth/state_accessor.go rename eth/{api_tracer.go => tracers/api.go} (55%) create mode 100644 eth/tracers/api_test.go create mode 100644 les/state_accessor.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index df036cbbf0..954b5de2c2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -45,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/graphql" @@ -1724,6 +1725,7 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend { if err != nil { Fatalf("Failed to register the Ethereum service: %v", err) } + stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) return backend.ApiBackend } backend, err := eth.New(stack, cfg) @@ -1736,6 +1738,7 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend { Fatalf("Failed to create the LES server: %v", err) } } + stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend } diff --git a/eth/api.go b/eth/api.go index be1dcbb52a..53ef91392b 100644 --- a/eth/api.go +++ b/eth/api.go @@ -426,10 +426,11 @@ func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, c if block == nil { return StorageRangeResult{}, fmt.Errorf("block %#x not found", blockHash) } - _, _, statedb, err := api.computeTxEnv(block, txIndex, 0) + _, _, statedb, release, err := api.eth.stateAtTransaction(block, txIndex, 0) if err != nil { return StorageRangeResult{}, err } + defer release() st := statedb.StorageTrie(contractAddress) if st == nil { return StorageRangeResult{}, fmt.Errorf("account %x doesn't exist", contractAddress) diff --git a/eth/api_backend.go b/eth/api_backend.go index 2f7020475f..17de83a28f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -326,3 +326,15 @@ func (b *EthAPIBackend) Miner() *miner.Miner { func (b *EthAPIBackend) StartMining(threads int) error { return b.eth.StartMining(threads) } + +func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { + return b.eth.stateAtBlock(block, reexec) +} + +func (b *EthAPIBackend) StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { + return b.eth.statesInRange(fromBlock, toBlock, reexec) +} + +func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { + return b.eth.stateAtTransaction(block, txIndex, reexec) +} diff --git a/eth/state_accessor.go b/eth/state_accessor.go new file mode 100644 index 0000000000..869b3d7636 --- /dev/null +++ b/eth/state_accessor.go @@ -0,0 +1,230 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "errors" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie" +) + +// stateAtBlock retrieves the state database associated with a certain block. +// If no state is locally available for the given block, a number of blocks are +// attempted to be reexecuted to generate the desired state. +func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64) (statedb *state.StateDB, release func(), err error) { + // If we have the state fully available, use that + statedb, err = eth.blockchain.StateAt(block.Root()) + if err == nil { + return statedb, func() {}, nil + } + // Otherwise try to reexec blocks until we find a state or reach our limit + origin := block.NumberU64() + database := state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16, Preimages: true}) + + for i := uint64(0); i < reexec; i++ { + if block.NumberU64() == 0 { + return nil, nil, errors.New("genesis state is missing") + } + parent := eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, nil, fmt.Errorf("missing block %v %d", block.ParentHash(), block.NumberU64()-1) + } + block = parent + + statedb, err = state.New(block.Root(), database, nil) + if err == nil { + break + } + } + if err != nil { + switch err.(type) { + case *trie.MissingNodeError: + return nil, nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) + default: + return nil, nil, err + } + } + // State was available at historical point, regenerate + var ( + start = time.Now() + logged time.Time + parent common.Hash + ) + defer func() { + if err != nil && parent != (common.Hash{}) { + database.TrieDB().Dereference(parent) + } + }() + for block.NumberU64() < origin { + // Print progress logs if long enough time elapsed + if time.Since(logged) > 8*time.Second { + log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "remaining", origin-block.NumberU64()-1, "elapsed", time.Since(start)) + logged = time.Now() + } + // Retrieve the next block to regenerate and process it + if block = eth.blockchain.GetBlockByNumber(block.NumberU64() + 1); block == nil { + return nil, nil, fmt.Errorf("block #%d not found", block.NumberU64()+1) + } + _, _, _, err := eth.blockchain.Processor().Process(block, statedb, vm.Config{}) + if err != nil { + return nil, nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) + } + // Finalize the state so any modifications are written to the trie + root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(block.Number())) + if err != nil { + return nil, nil, err + } + statedb, err = state.New(root, database, nil) + if err != nil { + return nil, nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) + } + database.TrieDB().Reference(root, common.Hash{}) + if parent != (common.Hash{}) { + database.TrieDB().Dereference(parent) + } + parent = root + } + nodes, imgs := database.TrieDB().Size() + log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) + return statedb, func() { database.TrieDB().Dereference(parent) }, nil +} + +// statesInRange retrieves a batch of state databases associated with the specific +// block ranges. If no state is locally available for the given range, a number of +// blocks are attempted to be reexecuted to generate the ancestor state. +func (eth *Ethereum) statesInRange(fromBlock, toBlock *types.Block, reexec uint64) (states []*state.StateDB, release func(), err error) { + statedb, err := eth.blockchain.StateAt(fromBlock.Root()) + if err != nil { + statedb, _, err = eth.stateAtBlock(fromBlock, reexec) + } + if err != nil { + return nil, nil, err + } + states = append(states, statedb.Copy()) + + var ( + logged time.Time + parent common.Hash + start = time.Now() + refs = []common.Hash{fromBlock.Root()} + database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16, Preimages: true}) + ) + // Release all resources(including the states referenced by `stateAtBlock`) + // if error is returned. + defer func() { + if err != nil { + for _, ref := range refs { + database.TrieDB().Dereference(ref) + } + } + }() + for i := fromBlock.NumberU64() + 1; i <= toBlock.NumberU64(); i++ { + // Print progress logs if long enough time elapsed + if time.Since(logged) > 8*time.Second { + logged = time.Now() + log.Info("Regenerating historical state", "block", i, "target", fromBlock.NumberU64(), "remaining", toBlock.NumberU64()-i, "elapsed", time.Since(start)) + } + // Retrieve the next block to regenerate and process it + block := eth.blockchain.GetBlockByNumber(i) + if block == nil { + return nil, nil, fmt.Errorf("block #%d not found", i) + } + _, _, _, err := eth.blockchain.Processor().Process(block, statedb, vm.Config{}) + if err != nil { + return nil, nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) + } + // Finalize the state so any modifications are written to the trie + root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(block.Number())) + if err != nil { + return nil, nil, err + } + statedb, err := eth.blockchain.StateAt(root) + if err != nil { + return nil, nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) + } + states = append(states, statedb.Copy()) + + // Reference the trie twice, once for us, once for the tracer + database.TrieDB().Reference(root, common.Hash{}) + database.TrieDB().Reference(root, common.Hash{}) + refs = append(refs, root) + + // Dereference all past tries we ourselves are done working with + if parent != (common.Hash{}) { + database.TrieDB().Dereference(parent) + } + parent = root + } + // release is handler to release all states referenced, including + // the one referenced in `stateAtBlock`. + release = func() { + for _, ref := range refs { + database.TrieDB().Dereference(ref) + } + } + return states, release, nil +} + +// stateAtTransaction returns the execution environment of a certain transaction. +func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { + // Short circuit if it's genesis block. + if block.NumberU64() == 0 { + return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") + } + // Create the parent state database + parent := eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) + } + statedb, release, err := eth.stateAtBlock(parent, reexec) + if err != nil { + return nil, vm.BlockContext{}, nil, nil, err + } + if txIndex == 0 && len(block.Transactions()) == 0 { + return nil, vm.BlockContext{}, statedb, release, nil + } + // Recompute transactions up to the target index. + signer := types.MakeSigner(eth.blockchain.Config(), block.Number()) + for idx, tx := range block.Transactions() { + // Assemble the transaction call message and return if the requested offset + msg, _ := tx.AsMessage(signer) + txContext := core.NewEVMTxContext(msg) + context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) + if idx == txIndex { + return msg, context, statedb, release, nil + } + // Not yet the searched for transaction, execute on top of the current state + vmenv := vm.NewEVM(context, txContext, statedb, eth.blockchain.Config(), vm.Config{}) + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + release() + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + } + // Ensure any modifications are committed to the state + // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect + statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + } + release() + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) +} diff --git a/eth/api_tracer.go b/eth/tracers/api.go similarity index 55% rename from eth/api_tracer.go rename to eth/tracers/api.go index 5dffb2a464..dca990ab95 100644 --- a/eth/api_tracer.go +++ b/eth/tracers/api.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2021 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package eth +package tracers import ( "bufio" @@ -30,18 +30,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" ) const ( @@ -55,6 +55,105 @@ const ( defaultTraceReexec = uint64(128) ) +// Backend interface provides the common API services (that are provided by +// both full and light clients) with access to necessary functions. +type Backend interface { + HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) + HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) + BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) + BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) + GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) + RPCGasCap() uint64 + ChainConfig() *params.ChainConfig + Engine() consensus.Engine + ChainDb() ethdb.Database + StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) + StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) + StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) +} + +// API is the collection of tracing APIs exposed over the private debugging endpoint. +type API struct { + backend Backend +} + +// NewAPI creates a new API definition for the tracing methods of the Ethereum service. +func NewAPI(backend Backend) *API { + return &API{backend: backend} +} + +type chainContext struct { + api *API + ctx context.Context +} + +func (context *chainContext) Engine() consensus.Engine { + return context.api.backend.Engine() +} + +func (context *chainContext) GetHeader(hash common.Hash, number uint64) *types.Header { + header, err := context.api.backend.HeaderByNumber(context.ctx, rpc.BlockNumber(number)) + if err != nil { + return nil + } + if header.Hash() == hash { + return header + } + header, err = context.api.backend.HeaderByHash(context.ctx, hash) + if err != nil { + return nil + } + return header +} + +// chainContext construts the context reader which is used by the evm for reading +// the necessary chain context. +func (api *API) chainContext(ctx context.Context) core.ChainContext { + return &chainContext{api: api, ctx: ctx} +} + +// blockByNumber is the wrapper of the chain access function offered by the backend. +// It will return an error if the block is not found. +func (api *API) blockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + block, err := api.backend.BlockByNumber(ctx, number) + if err != nil { + return nil, err + } + if block == nil { + return nil, fmt.Errorf("block #%d not found", number) + } + return block, nil +} + +// blockByHash is the wrapper of the chain access function offered by the backend. +// It will return an error if the block is not found. +func (api *API) blockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + block, err := api.backend.BlockByHash(ctx, hash) + if err != nil { + return nil, err + } + if block == nil { + return nil, fmt.Errorf("block %s not found", hash.Hex()) + } + return block, nil +} + +// blockByNumberAndHash is the wrapper of the chain access function offered by +// the backend. It will return an error if the block is not found. +// +// Note this function is friendly for the light client which can only retrieve the +// historical(before the CHT) header/block by number. +func (api *API) blockByNumberAndHash(ctx context.Context, number rpc.BlockNumber, hash common.Hash) (*types.Block, error) { + block, err := api.blockByNumber(ctx, number) + if err != nil { + return nil, err + } + if block.Hash() == hash { + return block, nil + } + return api.blockByHash(ctx, hash) +} + // TraceConfig holds extra parameters to trace functions. type TraceConfig struct { *vm.LogConfig @@ -81,7 +180,6 @@ type txTraceResult struct { type blockTraceTask struct { statedb *state.StateDB // Intermediate state prepped for tracing block *types.Block // Block to trace the transactions from - rootref common.Hash // Trie root reference held for this task results []*txTraceResult // Trace results procudes by the task } @@ -102,32 +200,14 @@ type txTraceTask struct { // TraceChain returns the structured logs created during the execution of EVM // between two blocks (excluding start) and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceChain(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) { - // Fetch the block interval that we want to trace - var from, to *types.Block - - switch start { - case rpc.PendingBlockNumber: - from = api.eth.miner.PendingBlock() - case rpc.LatestBlockNumber: - from = api.eth.blockchain.CurrentBlock() - default: - from = api.eth.blockchain.GetBlockByNumber(uint64(start)) - } - switch end { - case rpc.PendingBlockNumber: - to = api.eth.miner.PendingBlock() - case rpc.LatestBlockNumber: - to = api.eth.blockchain.CurrentBlock() - default: - to = api.eth.blockchain.GetBlockByNumber(uint64(end)) - } - // Trace the chain if we've found all our blocks - if from == nil { - return nil, fmt.Errorf("starting block #%d not found", start) +func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) { // Fetch the block interval that we want to trace + from, err := api.blockByNumber(ctx, start) + if err != nil { + return nil, err } - if to == nil { - return nil, fmt.Errorf("end block #%d not found", end) + to, err := api.blockByNumber(ctx, end) + if err != nil { + return nil, err } if from.Number().Cmp(to.Number()) >= 0 { return nil, fmt.Errorf("end block (#%d) needs to come after start block (#%d)", end, start) @@ -138,7 +218,7 @@ func (api *PrivateDebugAPI) TraceChain(ctx context.Context, start, end rpc.Block // traceChain configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item // per transaction, dependent on the requested tracer. -func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Block, config *TraceConfig) (*rpc.Subscription, error) { +func (api *API) traceChain(ctx context.Context, start, end *types.Block, config *TraceConfig) (*rpc.Subscription, error) { // Tracing a chain is a **long** operation, only do with subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -146,46 +226,25 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl } sub := notifier.CreateSubscription() - // Ensure we have a valid starting state before doing any work - origin := start.NumberU64() - database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16, Preimages: true}) - - if number := start.NumberU64(); number > 0 { - start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) - if start == nil { - return nil, fmt.Errorf("parent block #%d not found", number-1) - } + // Shift the border to a block ahead in order to get the states + // before these blocks. + endBlock, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(end.NumberU64()-1), end.ParentHash()) + if err != nil { + return nil, err + } + // Prepare all the states for tracing. Note this procedure can take very + // long time. Timeout mechanism is necessary. + reexec := defaultTraceReexec + if config != nil && config.Reexec != nil { + reexec = *config.Reexec } - statedb, err := state.New(start.Root(), database, nil) + states, release, err := api.backend.StatesInRange(ctx, start, endBlock, reexec) if err != nil { - // If the starting state is missing, allow some number of blocks to be reexecuted - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - // Find the most recent block that has the state available - for i := uint64(0); i < reexec; i++ { - start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) - if start == nil { - break - } - if statedb, err = state.New(start.Root(), database, nil); err == nil { - break - } - } - // If we still don't have the state available, bail out - if err != nil { - switch err.(type) { - case *trie.MissingNodeError: - return nil, errors.New("required historical state unavailable") - default: - return nil, err - } - } + return nil, err } - // Execute all the transaction contained within the chain concurrently for each block - blocks := int(end.NumberU64() - origin) + defer release() // Release all the resources in the last step. + blocks := int(end.NumberU64() - start.NumberU64()) threads := runtime.NumCPU() if threads > blocks { threads = blocks @@ -202,8 +261,8 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Fetch and execute the next block trace tasks for task := range tasks { - signer := types.MakeSigner(api.eth.blockchain.Config(), task.block.Number()) - blockCtx := core.NewEVMBlockContext(task.block.Header(), api.eth.blockchain, nil) + signer := types.MakeSigner(api.backend.ChainConfig(), task.block.Number()) + blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { msg, _ := tx.AsMessage(signer) @@ -214,7 +273,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl break } // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - task.statedb.Finalise(api.eth.blockchain.Config().IsEIP158(task.block.Number())) + task.statedb.Finalise(api.backend.ChainConfig().IsEIP158(task.block.Number())) task.results[i] = &txTraceResult{Result: res} } // Stream the result back to the user or abort on teardown @@ -235,7 +294,6 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl number uint64 traced uint64 failed error - proot common.Hash ) // Ensure everything is properly cleaned up on any exit path defer func() { @@ -262,60 +320,23 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl } // Print progress logs if long enough time elapsed if time.Since(logged) > 8*time.Second { - if number > origin { - nodes, imgs := database.TrieDB().Size() - log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin), "memory", nodes+imgs) - } else { - log.Info("Preparing state for chain trace", "block", number, "start", origin, "elapsed", time.Since(begin)) - } logged = time.Now() + log.Info("Tracing chain segment", "start", start.NumberU64(), "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin)) } // Retrieve the next block to trace - block := api.eth.blockchain.GetBlockByNumber(number) - if block == nil { - failed = fmt.Errorf("block #%d not found", number) - break - } - // Send the block over to the concurrent tracers (if not in the fast-forward phase) - if number > origin { - txs := block.Transactions() - - select { - case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: block, rootref: proot, results: make([]*txTraceResult, len(txs))}: - case <-notifier.Closed(): - return - } - traced += uint64(len(txs)) - } - // Generate the next state snapshot fast without tracing - _, _, _, err := api.eth.blockchain.Processor().Process(block, statedb, vm.Config{}) + block, err := api.blockByNumber(ctx, rpc.BlockNumber(number)) if err != nil { failed = err break } - // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(api.eth.blockchain.Config().IsEIP158(block.Number())) - if err != nil { - failed = err - break - } - statedb, err = state.New(root, database, nil) - if err != nil { - failed = err - break - } - // Reference the trie twice, once for us, once for the tracer - database.TrieDB().Reference(root, common.Hash{}) - if number >= origin { - database.TrieDB().Reference(root, common.Hash{}) - } - // Dereference all past tries we ourselves are done working with - if proot != (common.Hash{}) { - database.TrieDB().Dereference(proot) + // Send the block over to the concurrent tracers (if not in the fast-forward phase) + txs := block.Transactions() + select { + case tasks <- &blockTraceTask{statedb: states[int(number-start.NumberU64()-1)], block: block, results: make([]*txTraceResult, len(txs))}: + case <-notifier.Closed(): + return } - proot = root - - // TODO(karalabe): Do we need the preimages? Won't they accumulate too much? + traced += uint64(len(txs)) } }() @@ -323,7 +344,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl go func() { var ( done = make(map[uint64]*blockTraceResult) - next = origin + 1 + next = start.NumberU64() + 1 ) for res := range results { // Queue up next received result @@ -334,9 +355,6 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl } done[uint64(result.Block)] = result - // Dereference any paret tries held in memory by this task - database.TrieDB().Dereference(res.rootref) - // Stream completed traces to the user, aborting on the first error for result, ok := done[next]; ok; result, ok = done[next] { if len(result.Traces) > 0 || next == end.NumberU64() { @@ -352,38 +370,27 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // TraceBlockByNumber returns the structured logs created during the execution of // EVM and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceBlockByNumber(ctx context.Context, number rpc.BlockNumber, config *TraceConfig) ([]*txTraceResult, error) { - // Fetch the block that we want to trace - var block *types.Block - - switch number { - case rpc.PendingBlockNumber: - block = api.eth.miner.PendingBlock() - case rpc.LatestBlockNumber: - block = api.eth.blockchain.CurrentBlock() - default: - block = api.eth.blockchain.GetBlockByNumber(uint64(number)) - } - // Trace the block if it was found - if block == nil { - return nil, fmt.Errorf("block #%d not found", number) +func (api *API) TraceBlockByNumber(ctx context.Context, number rpc.BlockNumber, config *TraceConfig) ([]*txTraceResult, error) { + block, err := api.blockByNumber(ctx, number) + if err != nil { + return nil, err } return api.traceBlock(ctx, block, config) } // TraceBlockByHash returns the structured logs created during the execution of // EVM and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { - block := api.eth.blockchain.GetBlockByHash(hash) - if block == nil { - return nil, fmt.Errorf("block %#x not found", hash) +func (api *API) TraceBlockByHash(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { + block, err := api.blockByHash(ctx, hash) + if err != nil { + return nil, err } return api.traceBlock(ctx, block, config) } // TraceBlock returns the structured logs created during the execution of EVM // and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceBlock(ctx context.Context, blob []byte, config *TraceConfig) ([]*txTraceResult, error) { +func (api *API) TraceBlock(ctx context.Context, blob []byte, config *TraceConfig) ([]*txTraceResult, error) { block := new(types.Block) if err := rlp.Decode(bytes.NewReader(blob), block); err != nil { return nil, fmt.Errorf("could not decode block: %v", err) @@ -393,7 +400,7 @@ func (api *PrivateDebugAPI) TraceBlock(ctx context.Context, blob []byte, config // TraceBlockFromFile returns the structured logs created during the execution of // EVM and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, config *TraceConfig) ([]*txTraceResult, error) { +func (api *API) TraceBlockFromFile(ctx context.Context, file string, config *TraceConfig) ([]*txTraceResult, error) { blob, err := ioutil.ReadFile(file) if err != nil { return nil, fmt.Errorf("could not read file: %v", err) @@ -404,8 +411,8 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, // TraceBadBlock returns the structured logs created during the execution of // EVM against a block pulled from the pool of bad ones and returns them as a JSON // object. -func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { - for _, block := range rawdb.ReadAllBadBlocks(api.eth.chainDb) { +func (api *API) TraceBadBlock(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { + for _, block := range rawdb.ReadAllBadBlocks(api.backend.ChainDb()) { if block.Hash() == hash { return api.traceBlock(ctx, block, config) } @@ -416,10 +423,10 @@ func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, hash common.Hash, // StandardTraceBlockToFile dumps the structured logs created during the // execution of EVM to the local file system and returns a list of files // to the caller. -func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { - block := api.eth.blockchain.GetBlockByHash(hash) - if block == nil { - return nil, fmt.Errorf("block %#x not found", hash) +func (api *API) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { + block, err := api.blockByHash(ctx, hash) + if err != nil { + return nil, err } return api.standardTraceBlockToFile(ctx, block, config) } @@ -427,8 +434,8 @@ func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, hash c // StandardTraceBadBlockToFile dumps the structured logs created during the // execution of EVM against a block pulled from the pool of bad ones to the // local file system and returns a list of files to the caller. -func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { - for _, block := range rawdb.ReadAllBadBlocks(api.eth.chainDb) { +func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { + for _, block := range rawdb.ReadAllBadBlocks(api.backend.ChainDb()) { if block.Hash() == hash { return api.standardTraceBlockToFile(ctx, block, config) } @@ -439,27 +446,27 @@ func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, has // traceBlock configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item // per transaction, dependent on the requestd tracer. -func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { - // Create the parent state database - if err := api.eth.engine.VerifyHeader(api.eth.blockchain, block.Header(), true); err != nil { - return nil, err +func (api *API) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { + if block.NumberU64() == 0 { + return nil, errors.New("genesis is not traceable") } - parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return nil, fmt.Errorf("parent %#x not found", block.ParentHash()) + parent, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(block.NumberU64()-1), block.ParentHash()) + if err != nil { + return nil, err } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, err := api.computeStateDB(parent, reexec) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec) if err != nil { return nil, err } + defer release() + // Execute all the transaction contained within the block concurrently var ( - signer = types.MakeSigner(api.eth.blockchain.Config(), block.Number()) - + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) txs = block.Transactions() results = make([]*txTraceResult, len(txs)) @@ -470,7 +477,7 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, if threads > len(txs) { threads = len(txs) } - blockCtx := core.NewEVMBlockContext(block.Header(), api.eth.blockchain, nil) + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) for th := 0; th < threads; th++ { pend.Add(1) go func() { @@ -497,7 +504,7 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, msg, _ := tx.AsMessage(signer) txContext := core.NewEVMTxContext(msg) - vmenv := vm.NewEVM(blockCtx, txContext, statedb, api.eth.blockchain.Config(), vm.Config{}) + vmenv := vm.NewEVM(blockCtx, txContext, statedb, api.backend.ChainConfig(), vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { failed = err break @@ -519,29 +526,30 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, // standardTraceBlockToFile configures a new tracer which uses standard JSON output, // and traces either a full block or an individual transaction. The return value will // be one filename per transaction traced. -func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { +func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { // If we're tracing a single transaction, make sure it's present if config != nil && config.TxHash != (common.Hash{}) { if !containsTx(block, config.TxHash) { return nil, fmt.Errorf("transaction %#x not found in block", config.TxHash) } } - // Create the parent state database - if err := api.eth.engine.VerifyHeader(api.eth.blockchain, block.Header(), true); err != nil { - return nil, err + if block.NumberU64() == 0 { + return nil, errors.New("genesis is not traceable") } - parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return nil, fmt.Errorf("parent %#x not found", block.ParentHash()) + parent, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(block.NumberU64()-1), block.ParentHash()) + if err != nil { + return nil, err } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, err := api.computeStateDB(parent, reexec) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec) if err != nil { return nil, err } + defer release() + // Retrieve the tracing configurations, or use default values var ( logConfig vm.LogConfig @@ -555,10 +563,10 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block // Execute transaction, either tracing all or just the requested one var ( - signer = types.MakeSigner(api.eth.blockchain.Config(), block.Number()) dumps []string - chainConfig = api.eth.blockchain.Config() - vmctx = core.NewEVMBlockContext(block.Header(), api.eth.blockchain, nil) + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) + chainConfig = api.backend.ChainConfig() + vmctx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) canon = true ) // Check if there are any overrides: the caller may wish to enable a future @@ -645,139 +653,73 @@ func containsTx(block *types.Block, hash common.Hash) bool { return false } -// computeStateDB retrieves the state database associated with a certain block. -// If no state is locally available for the given block, a number of blocks are -// attempted to be reexecuted to generate the desired state. -func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*state.StateDB, error) { - // If we have the state fully available, use that - statedb, err := api.eth.blockchain.StateAt(block.Root()) - if err == nil { - return statedb, nil - } - // Otherwise try to reexec blocks until we find a state or reach our limit - origin := block.NumberU64() - database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16, Preimages: true}) - - for i := uint64(0); i < reexec; i++ { - block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) - if block == nil { - break - } - if statedb, err = state.New(block.Root(), database, nil); err == nil { - break - } - } - if err != nil { - switch err.(type) { - case *trie.MissingNodeError: - return nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) - default: - return nil, err - } - } - // State was available at historical point, regenerate - var ( - start = time.Now() - logged time.Time - proot common.Hash - ) - for block.NumberU64() < origin { - // Print progress logs if long enough time elapsed - if time.Since(logged) > 8*time.Second { - log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "remaining", origin-block.NumberU64()-1, "elapsed", time.Since(start)) - logged = time.Now() - } - // Retrieve the next block to regenerate and process it - if block = api.eth.blockchain.GetBlockByNumber(block.NumberU64() + 1); block == nil { - return nil, fmt.Errorf("block #%d not found", block.NumberU64()+1) - } - _, _, _, err := api.eth.blockchain.Processor().Process(block, statedb, vm.Config{}) - if err != nil { - return nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) - } - // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(api.eth.blockchain.Config().IsEIP158(block.Number())) - if err != nil { - return nil, err - } - statedb, err = state.New(root, database, nil) - if err != nil { - return nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) - } - database.TrieDB().Reference(root, common.Hash{}) - if proot != (common.Hash{}) { - database.TrieDB().Dereference(proot) - } - proot = root - } - nodes, imgs := database.TrieDB().Size() - log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) - return statedb, nil -} - // TraceTransaction returns the structured logs created during the execution of EVM // and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { - // Retrieve the transaction and assemble its EVM context - tx, blockHash, _, index := rawdb.ReadTransaction(api.eth.ChainDb(), hash) - if tx == nil { - return nil, fmt.Errorf("transaction %#x not found", hash) +func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { + _, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) + if err != nil { + return nil, err + } + // It shouldn't happen in practice. + if blockNumber == 0 { + return nil, errors.New("genesis is not traceable") } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec } - // Retrieve the block - block := api.eth.blockchain.GetBlockByHash(blockHash) - if block == nil { - return nil, fmt.Errorf("block %#x not found", blockHash) + block, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(blockNumber), blockHash) + if err != nil { + return nil, err } - msg, vmctx, statedb, err := api.computeTxEnv(block, int(index), reexec) + msg, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index), reexec) if err != nil { return nil, err } - // Trace the transaction and return + defer release() + return api.traceTx(ctx, msg, vmctx, statedb, config) } -// TraceCall lets you trace a given eth_call. It collects the structured logs created during the execution of EVM -// if the given transaction was added on top of the provided block and returns them as a JSON object. +// TraceCall lets you trace a given eth_call. It collects the structured logs +// created during the execution of EVM if the given transaction was added on +// top of the provided block and returns them as a JSON object. // You can provide -2 as a block number to trace on top of the pending block. -func (api *PrivateDebugAPI) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceConfig) (interface{}, error) { - // First try to retrieve the state - statedb, header, err := api.eth.APIBackend.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) +func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceConfig) (interface{}, error) { + // Try to retrieve the specified block + var ( + err error + block *types.Block + ) + if hash, ok := blockNrOrHash.Hash(); ok { + block, err = api.blockByHash(ctx, hash) + } else if number, ok := blockNrOrHash.Number(); ok { + block, err = api.blockByNumber(ctx, number) + } if err != nil { - // Try to retrieve the specified block - var block *types.Block - if hash, ok := blockNrOrHash.Hash(); ok { - block = api.eth.blockchain.GetBlockByHash(hash) - } else if number, ok := blockNrOrHash.Number(); ok { - block = api.eth.blockchain.GetBlockByNumber(uint64(number)) - } - if block == nil { - return nil, fmt.Errorf("block %v not found: %v", blockNrOrHash, err) - } - // try to recompute the state - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - _, _, statedb, err = api.computeTxEnv(block, 0, reexec) - if err != nil { - return nil, err - } + return nil, err + } + // try to recompute the state + reexec := defaultTraceReexec + if config != nil && config.Reexec != nil { + reexec = *config.Reexec } + statedb, release, err := api.backend.StateAtBlock(ctx, block, reexec) + if err != nil { + return nil, err + } + defer release() // Execute the trace - msg := args.ToMessage(api.eth.APIBackend.RPCGasCap()) - vmctx := core.NewEVMBlockContext(header, api.eth.blockchain, nil) + msg := args.ToMessage(api.backend.RPCGasCap()) + vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) return api.traceTx(ctx, msg, vmctx, statedb, config) } // traceTx configures a new tracer according to the provided configuration, and // executes the given message in the provided environment. The return value will // be tracer dependent. -func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { +func (api *API) traceTx(ctx context.Context, message core.Message, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { // Assemble the structured logger or the JavaScript tracer var ( tracer vm.Tracer @@ -794,14 +736,14 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v } } // Constuct the JavaScript tracer to execute with - if tracer, err = tracers.New(*config.Tracer, txContext); err != nil { + if tracer, err = New(*config.Tracer, txContext); err != nil { return nil, err } // Handle timeouts and RPC cancellations deadlineCtx, cancel := context.WithTimeout(ctx, timeout) go func() { <-deadlineCtx.Done() - tracer.(*tracers.Tracer).Stop(errors.New("execution timeout")) + tracer.(*Tracer).Stop(errors.New("execution timeout")) }() defer cancel() @@ -812,7 +754,7 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v tracer = vm.NewStructLogger(config.LogConfig) } // Run the transaction with tracing enabled. - vmenv := vm.NewEVM(vmctx, txContext, statedb, api.eth.blockchain.Config(), vm.Config{Debug: true, Tracer: tracer}) + vmenv := vm.NewEVM(vmctx, txContext, statedb, api.backend.ChainConfig(), vm.Config{Debug: true, Tracer: tracer}) result, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) if err != nil { @@ -833,7 +775,7 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v StructLogs: ethapi.FormatLogs(tracer.StructLogs()), }, nil - case *tracers.Tracer: + case *Tracer: return tracer.GetResult() default: @@ -841,41 +783,15 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v } } -// computeTxEnv returns the execution environment of a certain transaction. -func (api *PrivateDebugAPI) computeTxEnv(block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { - // Create the parent state database - parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return nil, vm.BlockContext{}, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) - } - statedb, err := api.computeStateDB(parent, reexec) - if err != nil { - return nil, vm.BlockContext{}, nil, err - } - - if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, nil - } - - // Recompute transactions up to the target index. - signer := types.MakeSigner(api.eth.blockchain.Config(), block.Number()) - - for idx, tx := range block.Transactions() { - // Assemble the transaction call message and return if the requested offset - msg, _ := tx.AsMessage(signer) - txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), api.eth.blockchain, nil) - if idx == txIndex { - return msg, context, statedb, nil - } - // Not yet the searched for transaction, execute on top of the current state - vmenv := vm.NewEVM(context, txContext, statedb, api.eth.blockchain.Config(), vm.Config{}) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) - } - // Ensure any modifications are committed to the state - // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) +// APIs return the collection of RPC services the tracer package offers. +func APIs(backend Backend) []rpc.API { + // Append all the local APIs and return + return []rpc.API{ + { + Namespace: "debug", + Version: "1.0", + Service: NewAPI(backend), + Public: false, + }, } - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go new file mode 100644 index 0000000000..688b983bab --- /dev/null +++ b/eth/tracers/api_test.go @@ -0,0 +1,487 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracers + +import ( + "bytes" + "context" + "crypto/ecdsa" + "errors" + "fmt" + "math/big" + "reflect" + "sort" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +var ( + errStateNotFound = errors.New("state not found") + errBlockNotFound = errors.New("block not found") + errTransactionNotFound = errors.New("transaction not found") +) + +type testBackend struct { + chainConfig *params.ChainConfig + engine consensus.Engine + chaindb ethdb.Database + chain *core.BlockChain +} + +func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i int, b *core.BlockGen)) *testBackend { + backend := &testBackend{ + chainConfig: params.TestChainConfig, + engine: ethash.NewFaker(), + chaindb: rawdb.NewMemoryDatabase(), + } + // Generate blocks for testing + gspec.Config = backend.chainConfig + var ( + gendb = rawdb.NewMemoryDatabase() + genesis = gspec.MustCommit(gendb) + ) + blocks, _ := core.GenerateChain(backend.chainConfig, genesis, backend.engine, gendb, n, generator) + + // Import the canonical chain + gspec.MustCommit(backend.chaindb) + cacheConfig := &core.CacheConfig{ + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 0, + TrieDirtyDisabled: true, // Archive mode + } + chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, backend.chainConfig, backend.engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + backend.chain = chain + return backend +} + +func (b *testBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return b.chain.GetHeaderByHash(hash), nil +} + +func (b *testBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + if number == rpc.PendingBlockNumber || number == rpc.LatestBlockNumber { + return b.chain.CurrentHeader(), nil + } + return b.chain.GetHeaderByNumber(uint64(number)), nil +} + +func (b *testBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + return b.chain.GetBlockByHash(hash), nil +} + +func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + if number == rpc.PendingBlockNumber || number == rpc.LatestBlockNumber { + return b.chain.CurrentBlock(), nil + } + return b.chain.GetBlockByNumber(uint64(number)), nil +} + +func (b *testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + tx, hash, blockNumber, index := rawdb.ReadTransaction(b.chaindb, txHash) + if tx == nil { + return nil, common.Hash{}, 0, 0, errTransactionNotFound + } + return tx, hash, blockNumber, index, nil +} + +func (b *testBackend) RPCGasCap() uint64 { + return 25000000 +} + +func (b *testBackend) ChainConfig() *params.ChainConfig { + return b.chainConfig +} + +func (b *testBackend) Engine() consensus.Engine { + return b.engine +} + +func (b *testBackend) ChainDb() ethdb.Database { + return b.chaindb +} + +func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { + statedb, err := b.chain.StateAt(block.Root()) + if err != nil { + return nil, nil, errStateNotFound + } + return statedb, func() {}, nil +} + +func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { + parent := b.chain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, vm.BlockContext{}, nil, nil, errBlockNotFound + } + statedb, err := b.chain.StateAt(parent.Root()) + if err != nil { + return nil, vm.BlockContext{}, nil, nil, errStateNotFound + } + if txIndex == 0 && len(block.Transactions()) == 0 { + return nil, vm.BlockContext{}, statedb, func() {}, nil + } + // Recompute transactions up to the target index. + signer := types.MakeSigner(b.chainConfig, block.Number()) + for idx, tx := range block.Transactions() { + msg, _ := tx.AsMessage(signer) + txContext := core.NewEVMTxContext(msg) + context := core.NewEVMBlockContext(block.Header(), b.chain, nil) + if idx == txIndex { + return msg, context, statedb, func() {}, nil + } + vmenv := vm.NewEVM(context, txContext, statedb, b.chainConfig, vm.Config{}) + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + } + statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + } + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) +} + +func (b *testBackend) StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { + var result []*state.StateDB + for number := fromBlock.NumberU64(); number <= toBlock.NumberU64(); number += 1 { + block := b.chain.GetBlockByNumber(number) + if block == nil { + return nil, nil, errBlockNotFound + } + statedb, err := b.chain.StateAt(block.Root()) + if err != nil { + return nil, nil, errStateNotFound + } + result = append(result, statedb) + } + return result, func() {}, nil +} + +func TestTraceCall(t *testing.T) { + t.Parallel() + + // Initialize test accounts + accounts := newAccounts(3) + genesis := &core.Genesis{Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }} + genBlocks := 10 + signer := types.HomesteadSigner{} + api := NewAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, big.NewInt(0), nil), signer, accounts[0].key) + b.AddTx(tx) + })) + + var testSuite = []struct { + blockNumber rpc.BlockNumber + call ethapi.CallArgs + config *TraceConfig + expectErr error + expect interface{} + }{ + // Standard JSON trace upon the genesis, plain transfer. + { + blockNumber: rpc.BlockNumber(0), + call: ethapi.CallArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: nil, + expectErr: nil, + expect: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + // Standard JSON trace upon the head, plain transfer. + { + blockNumber: rpc.BlockNumber(genBlocks), + call: ethapi.CallArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: nil, + expectErr: nil, + expect: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + // Standard JSON trace upon the non-existent block, error expects + { + blockNumber: rpc.BlockNumber(genBlocks + 1), + call: ethapi.CallArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: nil, + expectErr: fmt.Errorf("block #%d not found", genBlocks+1), + expect: nil, + }, + // Standard JSON trace upon the latest block + { + blockNumber: rpc.LatestBlockNumber, + call: ethapi.CallArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: nil, + expectErr: nil, + expect: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + // Standard JSON trace upon the pending block + { + blockNumber: rpc.PendingBlockNumber, + call: ethapi.CallArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: nil, + expectErr: nil, + expect: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + } + for _, testspec := range testSuite { + result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) + if testspec.expectErr != nil { + if err == nil { + t.Errorf("Expect error %v, get nothing", testspec.expectErr) + continue + } + if !reflect.DeepEqual(err, testspec.expectErr) { + t.Errorf("Error mismatch, want %v, get %v", testspec.expectErr, err) + } + } else { + if err != nil { + t.Errorf("Expect no error, get %v", err) + continue + } + if !reflect.DeepEqual(result, testspec.expect) { + t.Errorf("Result mismatch, want %v, get %v", testspec.expect, result) + } + } + } +} + +func TestTraceTransaction(t *testing.T) { + t.Parallel() + + // Initialize test accounts + accounts := newAccounts(2) + genesis := &core.Genesis{Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + }} + target := common.Hash{} + signer := types.HomesteadSigner{} + api := NewAPI(newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, big.NewInt(0), nil), signer, accounts[0].key) + b.AddTx(tx) + target = tx.Hash() + })) + result, err := api.TraceTransaction(context.Background(), target, nil) + if err != nil { + t.Errorf("Failed to trace transaction %v", err) + } + if !reflect.DeepEqual(result, ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }) { + t.Error("Transaction tracing result is different") + } +} + +func TestTraceBlock(t *testing.T) { + t.Parallel() + + // Initialize test accounts + accounts := newAccounts(3) + genesis := &core.Genesis{Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }} + genBlocks := 10 + signer := types.HomesteadSigner{} + api := NewAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, big.NewInt(0), nil), signer, accounts[0].key) + b.AddTx(tx) + })) + + var testSuite = []struct { + blockNumber rpc.BlockNumber + config *TraceConfig + expect interface{} + expectErr error + }{ + // Trace genesis block, expect error + { + blockNumber: rpc.BlockNumber(0), + config: nil, + expect: nil, + expectErr: errors.New("genesis is not traceable"), + }, + // Trace head block + { + blockNumber: rpc.BlockNumber(genBlocks), + config: nil, + expectErr: nil, + expect: []*txTraceResult{ + { + Result: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + }, + }, + // Trace non-existent block + { + blockNumber: rpc.BlockNumber(genBlocks + 1), + config: nil, + expectErr: fmt.Errorf("block #%d not found", genBlocks+1), + expect: nil, + }, + // Trace latest block + { + blockNumber: rpc.LatestBlockNumber, + config: nil, + expectErr: nil, + expect: []*txTraceResult{ + { + Result: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + }, + }, + // Trace pending block + { + blockNumber: rpc.PendingBlockNumber, + config: nil, + expectErr: nil, + expect: []*txTraceResult{ + { + Result: ðapi.ExecutionResult{ + Gas: params.TxGas, + Failed: false, + ReturnValue: "", + StructLogs: []ethapi.StructLogRes{}, + }, + }, + }, + }, + } + for _, testspec := range testSuite { + result, err := api.TraceBlockByNumber(context.Background(), testspec.blockNumber, testspec.config) + if testspec.expectErr != nil { + if err == nil { + t.Errorf("Expect error %v, get nothing", testspec.expectErr) + continue + } + if !reflect.DeepEqual(err, testspec.expectErr) { + t.Errorf("Error mismatch, want %v, get %v", testspec.expectErr, err) + } + } else { + if err != nil { + t.Errorf("Expect no error, get %v", err) + continue + } + if !reflect.DeepEqual(result, testspec.expect) { + t.Errorf("Result mismatch, want %v, get %v", testspec.expect, result) + } + } + } +} + +type Account struct { + key *ecdsa.PrivateKey + addr common.Address +} + +type Accounts []Account + +func (a Accounts) Len() int { return len(a) } +func (a Accounts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a Accounts) Less(i, j int) bool { return bytes.Compare(a[i].addr.Bytes(), a[j].addr.Bytes()) < 0 } + +func newAccounts(n int) (accounts Accounts) { + for i := 0; i < n; i++ { + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + accounts = append(accounts, Account{key: key, addr: addr}) + } + sort.Sort(accounts) + return accounts +} diff --git a/les/api_backend.go b/les/api_backend.go index 9fbc21f459..0839614901 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -292,3 +292,15 @@ func (b *LesApiBackend) Engine() consensus.Engine { func (b *LesApiBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } + +func (b *LesApiBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { + return b.eth.stateAtBlock(ctx, block, reexec) +} + +func (b *LesApiBackend) StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { + return b.eth.statesInRange(ctx, fromBlock, toBlock, reexec) +} + +func (b *LesApiBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { + return b.eth.stateAtTransaction(ctx, block, txIndex, reexec) +} diff --git a/les/state_accessor.go b/les/state_accessor.go new file mode 100644 index 0000000000..3c9143c875 --- /dev/null +++ b/les/state_accessor.go @@ -0,0 +1,88 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "context" + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/light" +) + +// stateAtBlock retrieves the state database associated with a certain block. +func (leth *LightEthereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { + return light.NewState(ctx, block.Header(), leth.odr), func() {}, nil +} + +// statesInRange retrieves a batch of state databases associated with the specific +// block ranges. +func (leth *LightEthereum) statesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { + var states []*state.StateDB + for number := fromBlock.NumberU64(); number <= toBlock.NumberU64(); number++ { + header, err := leth.blockchain.GetHeaderByNumberOdr(ctx, number) + if err != nil { + return nil, nil, err + } + states = append(states, light.NewState(ctx, header, leth.odr)) + } + return states, nil, nil +} + +// stateAtTransaction returns the execution environment of a certain transaction. +func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { + // Short circuit if it's genesis block. + if block.NumberU64() == 0 { + return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") + } + // Create the parent state database + parent, err := leth.blockchain.GetBlock(ctx, block.ParentHash(), block.NumberU64()-1) + if err != nil { + return nil, vm.BlockContext{}, nil, nil, err + } + statedb, _, err := leth.stateAtBlock(ctx, parent, reexec) + if err != nil { + return nil, vm.BlockContext{}, nil, nil, err + } + if txIndex == 0 && len(block.Transactions()) == 0 { + return nil, vm.BlockContext{}, statedb, func() {}, nil + } + // Recompute transactions up to the target index. + signer := types.MakeSigner(leth.blockchain.Config(), block.Number()) + for idx, tx := range block.Transactions() { + // Assemble the transaction call message and return if the requested offset + msg, _ := tx.AsMessage(signer) + txContext := core.NewEVMTxContext(msg) + context := core.NewEVMBlockContext(block.Header(), leth.blockchain, nil) + if idx == txIndex { + return msg, context, statedb, func() {}, nil + } + // Not yet the searched for transaction, execute on top of the current state + vmenv := vm.NewEVM(context, txContext, statedb, leth.blockchain.Config(), vm.Config{}) + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + } + // Ensure any modifications are committed to the state + // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect + statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + } + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) +} From d2779ed7acde5d0fa3ab53fcdc11ab1697703300 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 25 Jan 2021 19:06:52 +0100 Subject: [PATCH 096/709] eth, p2p: reserve half peer slots for snap peers during snap sync (#22171) * eth, p2p: reserve half peer slots for snap peers during snap sync * eth: less logging * eth: rework the eth/snap peer reservation logic * eth: rework the eth/snap peer reservation logic (again) --- eth/handler.go | 15 +++++++++++++-- eth/peerset.go | 11 ++++++++++- eth/sync.go | 4 ++++ p2p/peer.go | 10 ++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index f6366d9af1..5ae0925bb5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -250,9 +250,20 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Debug("Ethereum handshake failed", "err", err) return err } + reject := false // reserved peer slots + if atomic.LoadUint32(&h.snapSync) == 1 && !peer.SupportsCap("snap", 1) { + // If we are running snap-sync, we want to reserve roughly half the peer + // slots for peers supporting the snap protocol. + // The logic here is; we only allow up to 5 more non-snap peers than snap-peers. + if all, snp := h.peers.Len(), h.peers.SnapLen(); all-snp > snp+5 { + reject = true + } + } // Ignore maxPeers if this is a trusted peer - if h.peers.Len() >= h.maxPeers && !peer.Peer.Info().Network.Trusted { - return p2p.DiscTooManyPeers + if !peer.Peer.Info().Network.Trusted { + if reject || h.peers.Len() >= h.maxPeers { + return p2p.DiscTooManyPeers + } } peer.Log().Debug("Ethereum peer connected", "name", peer.Name()) diff --git a/eth/peerset.go b/eth/peerset.go index bf5785ff3f..663c5ce36b 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -259,7 +259,7 @@ func (ps *peerSet) ethPeersWithoutTransaction(hash common.Hash) []*ethPeer { } // Len returns if the current number of `eth` peers in the set. Since the `snap` -// peers are tied to the existnce of an `eth` connection, that will always be a +// peers are tied to the existence of an `eth` connection, that will always be a // subset of `eth`. func (ps *peerSet) Len() int { ps.lock.RLock() @@ -268,6 +268,15 @@ func (ps *peerSet) Len() int { return len(ps.ethPeers) } +// SnapLen returns if the current number of `snap` peers in the set. Since the `snap` +// peers are tied to the existence of an `eth` connection, that will always be a +// subset of `eth`. +func (ps *peerSet) SnapLen() int { + ps.lock.RLock() + defer ps.lock.RUnlock() + return len(ps.snapPeers) +} + // ethPeerWithHighestTD retrieves the known peer with the currently highest total // difficulty. func (ps *peerSet) ethPeerWithHighestTD() *eth.Peer { diff --git a/eth/sync.go b/eth/sync.go index 03a5165245..eedb8b7476 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -329,6 +329,10 @@ func (h *handler) doSync(op *chainSyncOp) error { log.Info("Fast sync complete, auto disabling") atomic.StoreUint32(&h.fastSync, 0) } + if atomic.LoadUint32(&h.snapSync) == 1 { + log.Info("Snap sync complete, auto disabling") + atomic.StoreUint32(&h.snapSync, 0) + } // If we've successfully finished a sync cycle and passed any required checkpoint, // enable accepting transactions from the network. head := h.chain.CurrentBlock() diff --git a/p2p/peer.go b/p2p/peer.go index a9c3cf01da..43ccef5c43 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -158,6 +158,16 @@ func (p *Peer) Caps() []Cap { return p.rw.caps } +// SupportsCap returns true if the peer supports the given protocol/version +func (p *Peer) SupportsCap(protocol string, version uint) bool { + for _, cap := range p.rw.caps { + if cap.Name == protocol { + return version <= cap.Version + } + } + return false +} + // RemoteAddr returns the remote address of the network connection. func (p *Peer) RemoteAddr() net.Addr { return p.rw.fd.RemoteAddr() From 7202b410b064c17c0648c4c6c212dc4c2a787907 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 25 Jan 2021 21:40:14 +0100 Subject: [PATCH 097/709] tests/fuzzers/abi: fixed one-off panic with int.Min64 value (#22233) * tests/fuzzers/abi: fixed one-off panic with int.Min64 value * tests/fuzzers/abi: fixed one-off panic with int.Min64 value --- tests/fuzzers/abi/abifuzzer.go | 5 ++++- tests/fuzzers/abi/abifuzzer_test.go | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/fuzzers/abi/abifuzzer.go b/tests/fuzzers/abi/abifuzzer.go index 8c083b371e..60233d158a 100644 --- a/tests/fuzzers/abi/abifuzzer.go +++ b/tests/fuzzers/abi/abifuzzer.go @@ -161,7 +161,10 @@ func getUInt(fuzzer *fuzz.Fuzzer) int { var i int fuzzer.Fuzz(&i) if i < 0 { - i *= -1 + i = -i + if i < 0 { + return 0 + } } return i } diff --git a/tests/fuzzers/abi/abifuzzer_test.go b/tests/fuzzers/abi/abifuzzer_test.go index c59c45ab1a..423a3cd232 100644 --- a/tests/fuzzers/abi/abifuzzer_test.go +++ b/tests/fuzzers/abi/abifuzzer_test.go @@ -23,9 +23,7 @@ import ( // TestReplicate can be used to replicate crashers from the fuzzing tests. // Just replace testString with the data in .quoted func TestReplicate(t *testing.T) { - testString := "N\xef\xbf0\xef\xbf99000000000000" + - "000000000000" - + testString := "\x20\x20\x20\x20\x20\x20\x20\x20\x80\x00\x00\x00\x20\x20\x20\x20\x00" data := []byte(testString) runFuzzer(data) } From 573f373d2bb264af6a2e39c3b219c999fc242122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 26 Jan 2021 13:13:55 +0200 Subject: [PATCH 098/709] internal/ethapi: print tx details when submitting (#22170) This adds more info about submitted transactions in log messages. Co-authored-by: Felix Lange --- internal/ethapi/api.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b424435b50..9c3f6b9161 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1558,16 +1558,18 @@ func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (c if err := b.SendTx(ctx, tx); err != nil { return common.Hash{}, err } + // Print a log with full tx details for manual investigations and interventions + signer := types.MakeSigner(b.ChainConfig(), b.CurrentBlock().Number()) + from, err := types.Sender(signer, tx) + if err != nil { + return common.Hash{}, err + } + if tx.To() == nil { - signer := types.MakeSigner(b.ChainConfig(), b.CurrentBlock().Number()) - from, err := types.Sender(signer, tx) - if err != nil { - return common.Hash{}, err - } addr := crypto.CreateAddress(from, tx.Nonce()) - log.Info("Submitted contract creation", "fullhash", tx.Hash().Hex(), "contract", addr.Hex()) + log.Info("Submitted contract creation", "hash", tx.Hash().Hex(), "from", from, "nonce", tx.Nonce(), "contract", addr.Hex(), "value", tx.Value()) } else { - log.Info("Submitted transaction", "fullhash", tx.Hash().Hex(), "recipient", tx.To()) + log.Info("Submitted transaction", "hash", tx.Hash().Hex(), "from", from, "nonce", tx.Nonce(), "recipient", tx.To(), "value", tx.Value()) } return tx.Hash(), nil } From 14d495491ddf31464135563720705fc2c1e5eb22 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 26 Jan 2021 12:15:31 +0100 Subject: [PATCH 099/709] core/state: fix panic in state dumping (#22225) --- core/state/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/dump.go b/core/state/dump.go index 9bb946d14b..b25da714fd 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -138,7 +138,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, account.SecureKey = it.Key } addr := common.BytesToAddress(addrBytes) - obj := newObject(nil, addr, data) + obj := newObject(s, addr, data) if !excludeCode { account.Code = common.Bytes2Hex(obj.Code(s.db)) } From 681618275cc5a4819af446029a064d266190ae8c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 26 Jan 2021 12:17:11 +0100 Subject: [PATCH 100/709] core: speed up header import (#21967) This PR implements the following modifications - Don't shortcut check if block is present, thus avoid disk lookup - Don't check hash ancestry in early-check (it's still done in parallel checker) - Don't check time.Now for every single header Charts and background info can be found here: https://github.com/holiman/headerimport/blob/main/README.md With these changes, writing 1M headers goes down to from 80s to 62s. --- consensus/ethash/consensus.go | 36 +++++++++++++++++------------------ core/headerchain.go | 11 ++++++----- core/rawdb/freezer_table.go | 5 +++++ 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index ae0905ee3a..c58fe7a530 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -39,11 +39,11 @@ import ( // Ethash proof-of-work protocol constants. var ( - FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium - ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople - maxUncles = 2 // Maximum number of uncles allowed in a single block - allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks + FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople + maxUncles = 2 // Maximum number of uncles allowed in a single block + allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks // calcDifficultyEip2384 is the difficulty adjustment algorithm as specified by EIP 2384. // It offsets the bomb 4M blocks from Constantinople, so in total 9M blocks. @@ -102,7 +102,7 @@ func (ethash *Ethash) VerifyHeader(chain consensus.ChainHeaderReader, header *ty return consensus.ErrUnknownAncestor } // Sanity checks passed, do a proper verification - return ethash.verifyHeader(chain, header, parent, false, seal) + return ethash.verifyHeader(chain, header, parent, false, seal, time.Now().Unix()) } // VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers @@ -126,15 +126,16 @@ func (ethash *Ethash) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ // Create a task channel and spawn the verifiers var ( - inputs = make(chan int) - done = make(chan int, workers) - errors = make([]error, len(headers)) - abort = make(chan struct{}) + inputs = make(chan int) + done = make(chan int, workers) + errors = make([]error, len(headers)) + abort = make(chan struct{}) + unixNow = time.Now().Unix() ) for i := 0; i < workers; i++ { go func() { for index := range inputs { - errors[index] = ethash.verifyHeaderWorker(chain, headers, seals, index) + errors[index] = ethash.verifyHeaderWorker(chain, headers, seals, index, unixNow) done <- index } }() @@ -170,7 +171,7 @@ func (ethash *Ethash) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ return abort, errorsOut } -func (ethash *Ethash) verifyHeaderWorker(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool, index int) error { +func (ethash *Ethash) verifyHeaderWorker(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool, index int, unixNow int64) error { var parent *types.Header if index == 0 { parent = chain.GetHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1) @@ -180,10 +181,7 @@ func (ethash *Ethash) verifyHeaderWorker(chain consensus.ChainHeaderReader, head if parent == nil { return consensus.ErrUnknownAncestor } - if chain.GetHeader(headers[index].Hash(), headers[index].Number.Uint64()) != nil { - return nil // known block - } - return ethash.verifyHeader(chain, headers[index], parent, false, seals[index]) + return ethash.verifyHeader(chain, headers[index], parent, false, seals[index], unixNow) } // VerifyUncles verifies that the given block's uncles conform to the consensus @@ -234,7 +232,7 @@ func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Blo if ancestors[uncle.ParentHash] == nil || uncle.ParentHash == block.ParentHash() { return errDanglingUncle } - if err := ethash.verifyHeader(chain, uncle, ancestors[uncle.ParentHash], true, true); err != nil { + if err := ethash.verifyHeader(chain, uncle, ancestors[uncle.ParentHash], true, true, time.Now().Unix()); err != nil { return err } } @@ -244,14 +242,14 @@ func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Blo // verifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum ethash engine. // See YP section 4.3.4. "Block Header Validity" -func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header, uncle bool, seal bool) error { +func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header, uncle bool, seal bool, unixNow int64) error { // Ensure that the header's extra-data section is of a reasonable size if uint64(len(header.Extra)) > params.MaximumExtraDataSize { return fmt.Errorf("extra-data too long: %d > %d", len(header.Extra), params.MaximumExtraDataSize) } // Verify the header's timestamp if !uncle { - if header.Time > uint64(time.Now().Add(allowedFutureBlockTime).Unix()) { + if header.Time > uint64(unixNow+allowedFutureBlockTimeSeconds) { return consensus.ErrFutureBlock } } diff --git a/core/headerchain.go b/core/headerchain.go index dc354549c0..dcd3644cd1 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -299,17 +299,18 @@ func (hc *HeaderChain) writeHeaders(headers []*types.Header) (result *headerWrit func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) (int, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { - parentHash := chain[i-1].Hash() - if chain[i].Number.Uint64() != chain[i-1].Number.Uint64()+1 || chain[i].ParentHash != parentHash { + if chain[i].Number.Uint64() != chain[i-1].Number.Uint64()+1 { + hash := chain[i].Hash() + parentHash := chain[i-1].Hash() // Chain broke ancestry, log a message (programming error) and skip insertion - log.Error("Non contiguous header insert", "number", chain[i].Number, "hash", chain[i].Hash(), + log.Error("Non contiguous header insert", "number", chain[i].Number, "hash", hash, "parent", chain[i].ParentHash, "prevnumber", chain[i-1].Number, "prevhash", parentHash) return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].Number, - parentHash.Bytes()[:4], i, chain[i].Number, chain[i].Hash().Bytes()[:4], chain[i].ParentHash[:4]) + parentHash.Bytes()[:4], i, chain[i].Number, hash.Bytes()[:4], chain[i].ParentHash[:4]) } // If the header is a banned one, straight out abort - if BadHashes[parentHash] { + if BadHashes[chain[i].ParentHash] { return i - 1, ErrBlacklistedHash } // If it's the last header in the cunk, we need to check it too diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index b9d8a274a8..cd273222b1 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -103,6 +103,11 @@ type freezerTable struct { lock sync.RWMutex // Mutex protecting the data file descriptors } +// NewFreezerTable opens the given path as a freezer table. +func NewFreezerTable(path, name string, disableSnappy bool) (*freezerTable, error) { + return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, disableSnappy) +} + // newTable opens a freezer table with default settings - 2G files func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, disableSnappy bool) (*freezerTable, error) { return newCustomTable(path, name, readMeter, writeMeter, sizeGauge, 2*1000*1000*1000, disableSnappy) From ad038b62899ae59160f9871573ec6786e95cb918 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 26 Jan 2021 16:01:13 +0100 Subject: [PATCH 101/709] accounts/scwallet: use go-ethereum crypto instead of go-ecdh (#22212) * accounts/scwallet: use go-ethereum crypto instead of go-ecdh github.com/wsddn/go-ecdh is a wrapper package for ECDH functionality with any elliptic curve. Since 'generic' ECDH is not required in accounts/scwallet (the curve is always secp256k1), we can just use the standard library functionality and our own crypto libraries to perform ECDH and save a dependency. * Update accounts/scwallet/securechannel.go Co-authored-by: Guillaume Ballet * Use the correct key Co-authored-by: Guillaume Ballet --- accounts/scwallet/securechannel.go | 21 +++++++-------------- go.mod | 1 - go.sum | 2 -- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index 9b70c69dcc..10887a8b43 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/aes" "crypto/cipher" + "crypto/elliptic" "crypto/rand" "crypto/sha256" "crypto/sha512" @@ -27,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" pcsc "github.com/gballet/go-libpcsclite" - "github.com/wsddn/go-ecdh" "golang.org/x/crypto/pbkdf2" "golang.org/x/text/unicode/norm" ) @@ -63,26 +63,19 @@ type SecureChannelSession struct { // NewSecureChannelSession creates a new secure channel for the given card and public key. func NewSecureChannelSession(card *pcsc.Card, keyData []byte) (*SecureChannelSession, error) { // Generate an ECDSA keypair for ourselves - gen := ecdh.NewEllipticECDH(crypto.S256()) - private, public, err := gen.GenerateKey(rand.Reader) + key, err := crypto.GenerateKey() if err != nil { return nil, err } - - cardPublic, ok := gen.Unmarshal(keyData) - if !ok { - return nil, fmt.Errorf("could not unmarshal public key from card") - } - - secret, err := gen.GenerateSharedSecret(private, cardPublic) + cardPublic, err := crypto.UnmarshalPubkey(keyData) if err != nil { - return nil, err + return nil, fmt.Errorf("could not unmarshal public key from card: %v", err) } - + secret, _ := key.Curve.ScalarMult(cardPublic.X, cardPublic.Y, key.D.Bytes()) return &SecureChannelSession{ card: card, - secret: secret, - publicKey: gen.Marshal(public), + secret: secret.Bytes(), + publicKey: elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y), }, nil } diff --git a/go.mod b/go.mod index ffea94f6cf..48c6889fbe 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,6 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef - github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 diff --git a/go.sum b/go.sum index fefe5afb14..4b799d77fa 100644 --- a/go.sum +++ b/go.sum @@ -365,8 +365,6 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= From 9c5729311e89bff5acc1f33c1f4a646dfcce7dab Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 26 Jan 2021 15:43:12 +0000 Subject: [PATCH 102/709] accounts/scwallet: update documentation (#22242) --- accounts/scwallet/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/accounts/scwallet/README.md b/accounts/scwallet/README.md index cfca916b3a..4313d9c6b2 100644 --- a/accounts/scwallet/README.md +++ b/accounts/scwallet/README.md @@ -31,12 +31,16 @@ Write down the URL (`keycard://044def09` in this example). Then ask `geth` to open the wallet: ``` - > personal.openWallet("keycard://044def09") - Please enter the pairing password: + > personal.openWallet("keycard://044def09", "pairing password") ``` - Enter the pairing password that you have received during card initialization. Same with the PIN that you will subsequently be - asked for. + The pairing password has been generated during the card initialization process. + + The process needs to be repeated once more with the PIN: + + ``` + > personal.openWallet("keycard://044def09", "PIN number") + ``` If everything goes well, you should see your new account when typing `personal` on the console: From a72fa88a0d661f86a06d3d89c755a4e7dcff1e4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 26 Jan 2021 21:41:35 +0100 Subject: [PATCH 103/709] les: switch to new discv5 (#21940) This PR enables running the new discv5 protocol in both LES client and server mode. In client mode it mixes discv5 and dnsdisc iterators (if both are enabled) and filters incoming ENRs for "les" tag and fork ID. The old p2p/discv5 package and all references to it are removed. Co-authored-by: Felix Lange --- cmd/bootnode/main.go | 15 +- cmd/faucet/faucet.go | 7 +- cmd/utils/flags.go | 15 +- les/client.go | 13 +- les/commons.go | 12 - les/enr_entry.go | 47 +- les/server.go | 21 - les/serverpool.go | 12 +- les/serverpool_test.go | 3 +- mobile/discover.go | 12 +- mobile/params.go | 10 +- p2p/discv5/README | 4 - p2p/discv5/database.go | 396 ---------- p2p/discv5/database_test.go | 380 --------- p2p/discv5/metrics.go | 24 - p2p/discv5/net.go | 1269 ------------------------------- p2p/discv5/net_test.go | 330 -------- p2p/discv5/node.go | 413 ---------- p2p/discv5/node_test.go | 305 -------- p2p/discv5/nodeevent_string.go | 17 - p2p/discv5/sim_run_test.go | 126 --- p2p/discv5/sim_test.go | 432 ----------- p2p/discv5/sim_testmain_test.go | 43 -- p2p/discv5/table.go | 318 -------- p2p/discv5/table_test.go | 238 ------ p2p/discv5/ticket.go | 884 --------------------- p2p/discv5/topic.go | 407 ---------- p2p/discv5/topic_test.go | 71 -- p2p/discv5/udp.go | 429 ----------- p2p/server.go | 26 +- params/bootnodes.go | 18 + 31 files changed, 113 insertions(+), 6184 deletions(-) delete mode 100644 p2p/discv5/README delete mode 100644 p2p/discv5/database.go delete mode 100644 p2p/discv5/database_test.go delete mode 100644 p2p/discv5/metrics.go delete mode 100644 p2p/discv5/net.go delete mode 100644 p2p/discv5/net_test.go delete mode 100644 p2p/discv5/node.go delete mode 100644 p2p/discv5/node_test.go delete mode 100644 p2p/discv5/nodeevent_string.go delete mode 100644 p2p/discv5/sim_run_test.go delete mode 100644 p2p/discv5/sim_test.go delete mode 100644 p2p/discv5/sim_testmain_test.go delete mode 100644 p2p/discv5/table.go delete mode 100644 p2p/discv5/table_test.go delete mode 100644 p2p/discv5/ticket.go delete mode 100644 p2p/discv5/topic.go delete mode 100644 p2p/discv5/topic_test.go delete mode 100644 p2p/discv5/udp.go diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 6c9ff615a1..036b968ef8 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" @@ -121,17 +120,17 @@ func main() { printNotice(&nodeKey.PublicKey, *realaddr) + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, nodeKey) + cfg := discover.Config{ + PrivateKey: nodeKey, + NetRestrict: restrictList, + } if *runv5 { - if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil { + if _, err := discover.ListenV5(conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } else { - db, _ := enode.OpenDB("") - ln := enode.NewLocalNode(db, nodeKey) - cfg := discover.Config{ - PrivateKey: nodeKey, - NetRestrict: restrictList, - } if _, err := discover.ListenUDP(conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index b9c4e1819a..763b8d25f8 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -55,7 +55,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/params" @@ -154,9 +153,9 @@ func main() { log.Crit("Failed to parse genesis block json", "err", err) } // Convert the bootnodes to internal enode representations - var enodes []*discv5.Node + var enodes []*enode.Node for _, boot := range strings.Split(*bootFlag, ",") { - if url, err := discv5.ParseNode(boot); err == nil { + if url, err := enode.Parse(enode.ValidSchemes, boot); err == nil { enodes = append(enodes, url) } else { log.Error("Failed to parse bootnode URL", "url", boot, "err", err) @@ -228,7 +227,7 @@ type wsConn struct { wlock sync.Mutex } -func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) { +func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) { // Assemble the raw devp2p protocol stack stack, err := node.New(&node.Config{ Name: "geth", diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 954b5de2c2..1d6d2d86b6 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -59,7 +59,6 @@ import ( "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" @@ -842,7 +841,7 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { // setBootstrapNodesV5 creates a list of bootstrap nodes from the command line // flags, reverting to pre-configured ones if none have been specified. func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { - urls := params.MainnetBootnodes + urls := params.V5Bootnodes switch { case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(LegacyBootnodesV5Flag.Name): if ctx.GlobalIsSet(LegacyBootnodesV5Flag.Name) { @@ -850,22 +849,14 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { } else { urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) } - case ctx.GlobalBool(RopstenFlag.Name): - urls = params.RopstenBootnodes - case ctx.GlobalBool(RinkebyFlag.Name): - urls = params.RinkebyBootnodes - case ctx.GlobalBool(GoerliFlag.Name): - urls = params.GoerliBootnodes - case ctx.GlobalBool(YoloV2Flag.Name): - urls = params.YoloV2Bootnodes case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } - cfg.BootstrapNodesV5 = make([]*discv5.Node, 0, len(urls)) + cfg.BootstrapNodesV5 = make([]*enode.Node, 0, len(urls)) for _, url := range urls { if url != "" { - node, err := discv5.ParseNode(url) + node, err := enode.Parse(enode.ValidSchemes, url) if err != nil { log.Error("Bootstrap URL invalid", "enode", url, "err", err) continue diff --git a/les/client.go b/les/client.go index 5ee50a7c3a..d8cb6e385a 100644 --- a/les/client.go +++ b/les/client.go @@ -72,6 +72,7 @@ type LightEthereum struct { netRPCService *ethapi.PublicNetAPI p2pServer *p2p.Server + p2pConfig *p2p.Config } // New creates an instance of the light client. @@ -109,14 +110,11 @@ func New(stack *node.Node, config *eth.Config) (*LightEthereum, error) { bloomIndexer: eth.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), valueTracker: lpc.NewValueTracker(lespayDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), p2pServer: stack.Server(), + p2pConfig: &stack.Config().P2P, } peers.subscribe((*vtSubscription)(leth.valueTracker)) - dnsdisc, err := leth.setupDiscovery() - if err != nil { - return nil, err - } - leth.serverPool = newServerPool(lespayDb, []byte("serverpool:"), leth.valueTracker, dnsdisc, time.Second, nil, &mclock.System{}, config.UltraLightServers) + leth.serverPool = newServerPool(lespayDb, []byte("serverpool:"), leth.valueTracker, time.Second, nil, &mclock.System{}, config.UltraLightServers) peers.subscribe(leth.serverPool) leth.dialCandidates = leth.serverPool.dialIterator @@ -299,6 +297,11 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { func (s *LightEthereum) Start() error { log.Warn("Light client mode is an experimental feature") + discovery, err := s.setupDiscovery(s.p2pConfig) + if err != nil { + return err + } + s.serverPool.addSource(discovery) s.serverPool.start() // Start bloom request workers. s.wg.Add(bloomServiceThreads) diff --git a/les/commons.go b/les/commons.go index 8de1057d26..73334497ad 100644 --- a/les/commons.go +++ b/les/commons.go @@ -33,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -42,17 +41,6 @@ func errResp(code errCode, format string, v ...interface{}) error { return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) } -func lesTopic(genesisHash common.Hash, protocolVersion uint) discv5.Topic { - var name string - switch protocolVersion { - case lpv2: - name = "LES2" - default: - panic(nil) - } - return discv5.Topic(name + "@" + common.Bytes2Hex(genesisHash.Bytes()[0:8])) -} - type chainReader interface { CurrentHeader() *types.Header } diff --git a/les/enr_entry.go b/les/enr_entry.go index a357f689df..1e56c1f175 100644 --- a/les/enr_entry.go +++ b/les/enr_entry.go @@ -17,6 +17,8 @@ package les import ( + "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" @@ -25,19 +27,46 @@ import ( // lesEntry is the "les" ENR entry. This is set for LES servers only. type lesEntry struct { // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` + _ []rlp.RawValue `rlp:"tail"` } -// ENRKey implements enr.Entry. -func (e lesEntry) ENRKey() string { - return "les" +func (lesEntry) ENRKey() string { return "les" } + +// ethEntry is the "eth" ENR entry. This is redeclared here to avoid depending on package eth. +type ethEntry struct { + ForkID forkid.ID + _ []rlp.RawValue `rlp:"tail"` } +func (ethEntry) ENRKey() string { return "eth" } + // setupDiscovery creates the node discovery source for the eth protocol. -func (eth *LightEthereum) setupDiscovery() (enode.Iterator, error) { - if len(eth.config.EthDiscoveryURLs) == 0 { - return nil, nil +func (eth *LightEthereum) setupDiscovery(cfg *p2p.Config) (enode.Iterator, error) { + it := enode.NewFairMix(0) + + // Enable DNS discovery. + if len(eth.config.EthDiscoveryURLs) != 0 { + client := dnsdisc.NewClient(dnsdisc.Config{}) + dns, err := client.NewIterator(eth.config.EthDiscoveryURLs...) + if err != nil { + return nil, err + } + it.AddSource(dns) + } + + // Enable DHT. + if cfg.DiscoveryV5 && eth.p2pServer.DiscV5 != nil { + it.AddSource(eth.p2pServer.DiscV5.RandomNodes()) } - client := dnsdisc.NewClient(dnsdisc.Config{}) - return client.NewIterator(eth.config.EthDiscoveryURLs...) + + forkFilter := forkid.NewFilter(eth.blockchain) + iterator := enode.Filter(it, func(n *enode.Node) bool { return nodeIsServer(forkFilter, n) }) + return iterator, nil +} + +// nodeIsServer checks whether n is an LES server node. +func nodeIsServer(forkFilter forkid.Filter, n *enode.Node) bool { + var les lesEntry + var eth ethEntry + return n.Load(&les) == nil && n.Load(ð) == nil && forkFilter(eth.ForkID) == nil } diff --git a/les/server.go b/les/server.go index cbedce136c..6b12b6f8f3 100644 --- a/les/server.go +++ b/les/server.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nodestate" @@ -58,7 +57,6 @@ type LesServer struct { archiveMode bool // Flag whether the ethereum node runs in archive mode. handler *serverHandler broadcaster *broadcaster - lesTopics []discv5.Topic privateKey *ecdsa.PrivateKey // Flow control and capacity management @@ -77,11 +75,6 @@ type LesServer struct { func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesServer, error) { ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup) - // Collect les protocol version information supported by local node. - lesTopics := make([]discv5.Topic, len(AdvertiseProtocolVersions)) - for i, pv := range AdvertiseProtocolVersions { - lesTopics[i] = lesTopic(e.BlockChain().Genesis().Hash(), pv) - } // Calculate the number of threads used to service the light client // requests based on the user-specified value. threads := config.LightServ * 4 / 100 @@ -103,7 +96,6 @@ func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesSer ns: ns, archiveMode: e.ArchiveMode(), broadcaster: newBroadcaster(ns), - lesTopics: lesTopics, fcManager: flowcontrol.NewClientManager(nil, &mclock.System{}), servingQueue: newServingQueue(int64(time.Millisecond*10), float64(config.LightServ)/100), threadsBusy: config.LightServ/100 + 1, @@ -203,19 +195,6 @@ func (s *LesServer) Start() error { s.wg.Add(1) go s.capacityManagement() - if s.p2pSrv.DiscV5 != nil { - for _, topic := range s.lesTopics { - topic := topic - go func() { - logger := log.New("topic", topic) - logger.Info("Starting topic registration") - defer logger.Info("Terminated topic registration") - - s.p2pSrv.DiscV5.RegisterTopic(topic, s.closeCh) - }() - } - } - return nil } diff --git a/les/serverpool.go b/les/serverpool.go index 6cf4affff0..ac87acf6da 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -131,7 +131,7 @@ var ( ) // newServerPool creates a new server pool -func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, discovery enode.Iterator, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string) *serverPool { +func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string) *serverPool { s := &serverPool{ db: db, clock: clock, @@ -147,9 +147,6 @@ func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, d alwaysConnect := lpc.NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) s.mixSources = append(s.mixSources, knownSelector) s.mixSources = append(s.mixSources, alwaysConnect) - if discovery != nil { - s.mixSources = append(s.mixSources, discovery) - } iter := enode.Iterator(s.mixer) if query != nil { @@ -175,6 +172,13 @@ func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, d return s } +// addSource adds a node discovery source to the server pool (should be called before start) +func (s *serverPool) addSource(source enode.Iterator) { + if source != nil { + s.mixSources = append(s.mixSources, source) + } +} + // addPreNegFilter installs a node filter mechanism that performs a pre-negotiation query. // Nodes that are filtered out and does not appear on the output iterator are put back // into redialWait state. diff --git a/les/serverpool_test.go b/les/serverpool_test.go index 70a41b74c6..3b7ae65d5d 100644 --- a/les/serverpool_test.go +++ b/les/serverpool_test.go @@ -145,7 +145,8 @@ func (s *serverPoolTest) start() { } s.vt = lpc.NewValueTracker(s.db, s.clock, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)) - s.sp = newServerPool(s.db, []byte("serverpool:"), s.vt, s.input, 0, testQuery, s.clock, s.trusted) + s.sp = newServerPool(s.db, []byte("serverpool:"), s.vt, 0, testQuery, s.clock, s.trusted) + s.sp.addSource(s.input) s.sp.validSchemes = enode.ValidSchemesForTesting s.sp.unixTime = func() int64 { return int64(s.clock.Now()) / int64(time.Second) } s.disconnect = make(map[int][]int) diff --git a/mobile/discover.go b/mobile/discover.go index 9b3c93ccd9..2c699f08be 100644 --- a/mobile/discover.go +++ b/mobile/discover.go @@ -22,12 +22,12 @@ package geth import ( "errors" - "github.com/ethereum/go-ethereum/p2p/discv5" + "github.com/ethereum/go-ethereum/p2p/enode" ) // Enode represents a host on the network. type Enode struct { - node *discv5.Node + node *enode.Node } // NewEnode parses a node designator. @@ -53,8 +53,8 @@ type Enode struct { // and UDP discovery port 30301. // // enode://@10.3.58.6:30303?discport=30301 -func NewEnode(rawurl string) (enode *Enode, _ error) { - node, err := discv5.ParseNode(rawurl) +func NewEnode(rawurl string) (*Enode, error) { + node, err := enode.Parse(enode.ValidSchemes, rawurl) if err != nil { return nil, err } @@ -62,12 +62,12 @@ func NewEnode(rawurl string) (enode *Enode, _ error) { } // Enodes represents a slice of accounts. -type Enodes struct{ nodes []*discv5.Node } +type Enodes struct{ nodes []*enode.Node } // NewEnodes creates a slice of uninitialized enodes. func NewEnodes(size int) *Enodes { return &Enodes{ - nodes: make([]*discv5.Node, size), + nodes: make([]*enode.Node, size), } } diff --git a/mobile/params.go b/mobile/params.go index 43ac004740..0fc197c9e5 100644 --- a/mobile/params.go +++ b/mobile/params.go @@ -22,7 +22,7 @@ import ( "encoding/json" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/p2p/discv5" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,9 +62,13 @@ func GoerliGenesis() string { // FoundationBootnodes returns the enode URLs of the P2P bootstrap nodes operated // by the foundation running the V5 discovery protocol. func FoundationBootnodes() *Enodes { - nodes := &Enodes{nodes: make([]*discv5.Node, len(params.MainnetBootnodes))} + nodes := &Enodes{nodes: make([]*enode.Node, len(params.MainnetBootnodes))} for i, url := range params.MainnetBootnodes { - nodes.nodes[i] = discv5.MustParseNode(url) + var err error + nodes.nodes[i], err = enode.Parse(enode.ValidSchemes, url) + if err != nil { + panic("invalid node URL: " + err.Error()) + } } return nodes } diff --git a/p2p/discv5/README b/p2p/discv5/README deleted file mode 100644 index 617a473d7f..0000000000 --- a/p2p/discv5/README +++ /dev/null @@ -1,4 +0,0 @@ -This package is an early prototype of Discovery v5. Do not use this code. - -See https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md for the -current Discovery v5 specification. \ No newline at end of file diff --git a/p2p/discv5/database.go b/p2p/discv5/database.go deleted file mode 100644 index ca118e7f80..0000000000 --- a/p2p/discv5/database.go +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the node database, storing previously seen nodes and any collected -// metadata about them for QoS purposes. - -package discv5 - -import ( - "bytes" - "crypto/rand" - "encoding/binary" - "fmt" - "os" - "sync" - "time" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/util" -) - -var ( - nodeDBNilNodeID = NodeID{} // Special node ID to use as a nil element. - nodeDBNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped. - nodeDBCleanupCycle = time.Hour // Time period for running the expiration task. -) - -// nodeDB stores all nodes we know about. -type nodeDB struct { - lvl *leveldb.DB // Interface to the database itself - self NodeID // Own node id to prevent adding it into the database - runner sync.Once // Ensures we can start at most one expirer - quit chan struct{} // Channel to signal the expiring thread to stop -} - -// Schema layout for the node database -var ( - nodeDBVersionKey = []byte("version") // Version of the database to flush if changes - nodeDBItemPrefix = []byte("n:") // Identifier to prefix node entries with - - nodeDBDiscoverRoot = ":discover" - nodeDBDiscoverPing = nodeDBDiscoverRoot + ":lastping" - nodeDBDiscoverPong = nodeDBDiscoverRoot + ":lastpong" - nodeDBDiscoverFindFails = nodeDBDiscoverRoot + ":findfail" - nodeDBTopicRegTickets = ":tickets" -) - -// newNodeDB creates a new node database for storing and retrieving infos about -// known peers in the network. If no path is given, an in-memory, temporary -// database is constructed. -func newNodeDB(path string, version int, self NodeID) (*nodeDB, error) { - if path == "" { - return newMemoryNodeDB(self) - } - return newPersistentNodeDB(path, version, self) -} - -// newMemoryNodeDB creates a new in-memory node database without a persistent -// backend. -func newMemoryNodeDB(self NodeID) (*nodeDB, error) { - db, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - return nil, err - } - return &nodeDB{ - lvl: db, - self: self, - quit: make(chan struct{}), - }, nil -} - -// newPersistentNodeDB creates/opens a leveldb backed persistent node database, -// also flushing its contents in case of a version mismatch. -func newPersistentNodeDB(path string, version int, self NodeID) (*nodeDB, error) { - opts := &opt.Options{OpenFilesCacheCapacity: 5} - db, err := leveldb.OpenFile(path, opts) - if _, iscorrupted := err.(*errors.ErrCorrupted); iscorrupted { - db, err = leveldb.RecoverFile(path, nil) - } - if err != nil { - return nil, err - } - // The nodes contained in the cache correspond to a certain protocol version. - // Flush all nodes if the version doesn't match. - currentVer := make([]byte, binary.MaxVarintLen64) - currentVer = currentVer[:binary.PutVarint(currentVer, int64(version))] - - blob, err := db.Get(nodeDBVersionKey, nil) - switch err { - case leveldb.ErrNotFound: - // Version not found (i.e. empty cache), insert it - if err := db.Put(nodeDBVersionKey, currentVer, nil); err != nil { - db.Close() - return nil, err - } - - case nil: - // Version present, flush if different - if !bytes.Equal(blob, currentVer) { - db.Close() - if err = os.RemoveAll(path); err != nil { - return nil, err - } - return newPersistentNodeDB(path, version, self) - } - } - return &nodeDB{ - lvl: db, - self: self, - quit: make(chan struct{}), - }, nil -} - -// makeKey generates the leveldb key-blob from a node id and its particular -// field of interest. -func makeKey(id NodeID, field string) []byte { - if bytes.Equal(id[:], nodeDBNilNodeID[:]) { - return []byte(field) - } - return append(nodeDBItemPrefix, append(id[:], field...)...) -} - -// splitKey tries to split a database key into a node id and a field part. -func splitKey(key []byte) (id NodeID, field string) { - // If the key is not of a node, return it plainly - if !bytes.HasPrefix(key, nodeDBItemPrefix) { - return NodeID{}, string(key) - } - // Otherwise split the id and field - item := key[len(nodeDBItemPrefix):] - copy(id[:], item[:len(id)]) - field = string(item[len(id):]) - - return id, field -} - -// fetchInt64 retrieves an integer instance associated with a particular -// database key. -func (db *nodeDB) fetchInt64(key []byte) int64 { - blob, err := db.lvl.Get(key, nil) - if err != nil { - return 0 - } - val, read := binary.Varint(blob) - if read <= 0 { - return 0 - } - return val -} - -// storeInt64 update a specific database entry to the current time instance as a -// unix timestamp. -func (db *nodeDB) storeInt64(key []byte, n int64) error { - blob := make([]byte, binary.MaxVarintLen64) - blob = blob[:binary.PutVarint(blob, n)] - return db.lvl.Put(key, blob, nil) -} - -func (db *nodeDB) storeRLP(key []byte, val interface{}) error { - blob, err := rlp.EncodeToBytes(val) - if err != nil { - return err - } - return db.lvl.Put(key, blob, nil) -} - -func (db *nodeDB) fetchRLP(key []byte, val interface{}) error { - blob, err := db.lvl.Get(key, nil) - if err != nil { - return err - } - err = rlp.DecodeBytes(blob, val) - if err != nil { - log.Warn(fmt.Sprintf("key %x (%T) %v", key, val, err)) - } - return err -} - -// node retrieves a node with a given id from the database. -func (db *nodeDB) node(id NodeID) *Node { - var node Node - if err := db.fetchRLP(makeKey(id, nodeDBDiscoverRoot), &node); err != nil { - return nil - } - node.sha = crypto.Keccak256Hash(node.ID[:]) - return &node -} - -// updateNode inserts - potentially overwriting - a node into the peer database. -func (db *nodeDB) updateNode(node *Node) error { - return db.storeRLP(makeKey(node.ID, nodeDBDiscoverRoot), node) -} - -// deleteNode deletes all information/keys associated with a node. -func (db *nodeDB) deleteNode(id NodeID) error { - deleter := db.lvl.NewIterator(util.BytesPrefix(makeKey(id, "")), nil) - for deleter.Next() { - if err := db.lvl.Delete(deleter.Key(), nil); err != nil { - return err - } - } - return nil -} - -// ensureExpirer is a small helper method ensuring that the data expiration -// mechanism is running. If the expiration goroutine is already running, this -// method simply returns. -// -// The goal is to start the data evacuation only after the network successfully -// bootstrapped itself (to prevent dumping potentially useful seed nodes). Since -// it would require significant overhead to exactly trace the first successful -// convergence, it's simpler to "ensure" the correct state when an appropriate -// condition occurs (i.e. a successful bonding), and discard further events. -func (db *nodeDB) ensureExpirer() { - db.runner.Do(func() { go db.expirer() }) -} - -// expirer should be started in a go routine, and is responsible for looping ad -// infinitum and dropping stale data from the database. -func (db *nodeDB) expirer() { - tick := time.NewTicker(nodeDBCleanupCycle) - defer tick.Stop() - for { - select { - case <-tick.C: - if err := db.expireNodes(); err != nil { - log.Error(fmt.Sprintf("Failed to expire nodedb items: %v", err)) - } - case <-db.quit: - return - } - } -} - -// expireNodes iterates over the database and deletes all nodes that have not -// been seen (i.e. received a pong from) for some allotted time. -func (db *nodeDB) expireNodes() error { - threshold := time.Now().Add(-nodeDBNodeExpiration) - - // Find discovered nodes that are older than the allowance - it := db.lvl.NewIterator(nil, nil) - defer it.Release() - - for it.Next() { - // Skip the item if not a discovery node - id, field := splitKey(it.Key()) - if field != nodeDBDiscoverRoot { - continue - } - // Skip the node if not expired yet (and not self) - if !bytes.Equal(id[:], db.self[:]) { - if seen := db.lastPong(id); seen.After(threshold) { - continue - } - } - // Otherwise delete all associated information - db.deleteNode(id) - } - return nil -} - -// lastPing retrieves the time of the last ping packet send to a remote node, -// requesting binding. -func (db *nodeDB) lastPing(id NodeID) time.Time { - return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPing)), 0) -} - -// updateLastPing updates the last time we tried contacting a remote node. -func (db *nodeDB) updateLastPing(id NodeID, instance time.Time) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix()) -} - -// lastPong retrieves the time of the last successful contact from remote node. -func (db *nodeDB) lastPong(id NodeID) time.Time { - return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0) -} - -// updateLastPong updates the last time a remote node successfully contacted. -func (db *nodeDB) updateLastPong(id NodeID, instance time.Time) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix()) -} - -// findFails retrieves the number of findnode failures since bonding. -func (db *nodeDB) findFails(id NodeID) int { - return int(db.fetchInt64(makeKey(id, nodeDBDiscoverFindFails))) -} - -// updateFindFails updates the number of findnode failures since bonding. -func (db *nodeDB) updateFindFails(id NodeID, fails int) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverFindFails), int64(fails)) -} - -// querySeeds retrieves random nodes to be used as potential seed nodes -// for bootstrapping. -func (db *nodeDB) querySeeds(n int, maxAge time.Duration) []*Node { - var ( - now = time.Now() - nodes = make([]*Node, 0, n) - it = db.lvl.NewIterator(nil, nil) - id NodeID - ) - defer it.Release() - -seek: - for seeks := 0; len(nodes) < n && seeks < n*5; seeks++ { - // Seek to a random entry. The first byte is incremented by a - // random amount each time in order to increase the likelihood - // of hitting all existing nodes in very small databases. - ctr := id[0] - rand.Read(id[:]) - id[0] = ctr + id[0]%16 - it.Seek(makeKey(id, nodeDBDiscoverRoot)) - - n := nextNode(it) - if n == nil { - id[0] = 0 - continue seek // iterator exhausted - } - if n.ID == db.self { - continue seek - } - if now.Sub(db.lastPong(n.ID)) > maxAge { - continue seek - } - for i := range nodes { - if nodes[i].ID == n.ID { - continue seek // duplicate - } - } - nodes = append(nodes, n) - } - return nodes -} - -func (db *nodeDB) fetchTopicRegTickets(id NodeID) (issued, used uint32) { - key := makeKey(id, nodeDBTopicRegTickets) - blob, _ := db.lvl.Get(key, nil) - if len(blob) != 8 { - return 0, 0 - } - issued = binary.BigEndian.Uint32(blob[0:4]) - used = binary.BigEndian.Uint32(blob[4:8]) - return -} - -func (db *nodeDB) updateTopicRegTickets(id NodeID, issued, used uint32) error { - key := makeKey(id, nodeDBTopicRegTickets) - blob := make([]byte, 8) - binary.BigEndian.PutUint32(blob[0:4], issued) - binary.BigEndian.PutUint32(blob[4:8], used) - return db.lvl.Put(key, blob, nil) -} - -// reads the next node record from the iterator, skipping over other -// database entries. -func nextNode(it iterator.Iterator) *Node { - for end := false; !end; end = !it.Next() { - id, field := splitKey(it.Key()) - if field != nodeDBDiscoverRoot { - continue - } - var n Node - if err := rlp.DecodeBytes(it.Value(), &n); err != nil { - log.Warn(fmt.Sprintf("invalid node %x: %v", id, err)) - continue - } - return &n - } - return nil -} - -// close flushes and closes the database files. -func (db *nodeDB) close() { - close(db.quit) - db.lvl.Close() -} diff --git a/p2p/discv5/database_test.go b/p2p/discv5/database_test.go deleted file mode 100644 index 2b86dc9cec..0000000000 --- a/p2p/discv5/database_test.go +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "bytes" - "io/ioutil" - "net" - "os" - "path/filepath" - "reflect" - "testing" - "time" -) - -var nodeDBKeyTests = []struct { - id NodeID - field string - key []byte -}{ - { - id: NodeID{}, - field: "version", - key: []byte{0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e}, // field - }, - { - id: MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - field: ":discover", - key: []byte{0x6e, 0x3a, // prefix - 0x1d, 0xd9, 0xd6, 0x5c, 0x45, 0x52, 0xb5, 0xeb, // node id - 0x43, 0xd5, 0xad, 0x55, 0xa2, 0xee, 0x3f, 0x56, // - 0xc6, 0xcb, 0xc1, 0xc6, 0x4a, 0x5c, 0x8d, 0x65, // - 0x9f, 0x51, 0xfc, 0xd5, 0x1b, 0xac, 0xe2, 0x43, // - 0x51, 0x23, 0x2b, 0x8d, 0x78, 0x21, 0x61, 0x7d, // - 0x2b, 0x29, 0xb5, 0x4b, 0x81, 0xcd, 0xef, 0xb9, // - 0xb3, 0xe9, 0xc3, 0x7d, 0x7f, 0xd5, 0xf6, 0x32, // - 0x70, 0xbc, 0xc9, 0xe1, 0xa6, 0xf6, 0xa4, 0x39, // - 0x3a, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, // field - }, - }, -} - -func TestNodeDBKeys(t *testing.T) { - for i, tt := range nodeDBKeyTests { - if key := makeKey(tt.id, tt.field); !bytes.Equal(key, tt.key) { - t.Errorf("make test %d: key mismatch: have 0x%x, want 0x%x", i, key, tt.key) - } - id, field := splitKey(tt.key) - if !bytes.Equal(id[:], tt.id[:]) { - t.Errorf("split test %d: id mismatch: have 0x%x, want 0x%x", i, id, tt.id) - } - if field != tt.field { - t.Errorf("split test %d: field mismatch: have 0x%x, want 0x%x", i, field, tt.field) - } - } -} - -var nodeDBInt64Tests = []struct { - key []byte - value int64 -}{ - {key: []byte{0x01}, value: 1}, - {key: []byte{0x02}, value: 2}, - {key: []byte{0x03}, value: 3}, -} - -func TestNodeDBInt64(t *testing.T) { - db, _ := newNodeDB("", Version, NodeID{}) - defer db.close() - - tests := nodeDBInt64Tests - for i := 0; i < len(tests); i++ { - // Insert the next value - if err := db.storeInt64(tests[i].key, tests[i].value); err != nil { - t.Errorf("test %d: failed to store value: %v", i, err) - } - // Check all existing and non existing values - for j := 0; j < len(tests); j++ { - num := db.fetchInt64(tests[j].key) - switch { - case j <= i && num != tests[j].value: - t.Errorf("test %d, item %d: value mismatch: have %v, want %v", i, j, num, tests[j].value) - case j > i && num != 0: - t.Errorf("test %d, item %d: value mismatch: have %v, want %v", i, j, num, 0) - } - } - } -} - -func TestNodeDBFetchStore(t *testing.T) { - node := NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{192, 168, 0, 1}, - 30303, - 30303, - ) - inst := time.Now() - num := 314 - - db, _ := newNodeDB("", Version, NodeID{}) - defer db.close() - - // Check fetch/store operations on a node ping object - if stored := db.lastPing(node.ID); stored.Unix() != 0 { - t.Errorf("ping: non-existing object: %v", stored) - } - if err := db.updateLastPing(node.ID, inst); err != nil { - t.Errorf("ping: failed to update: %v", err) - } - if stored := db.lastPing(node.ID); stored.Unix() != inst.Unix() { - t.Errorf("ping: value mismatch: have %v, want %v", stored, inst) - } - // Check fetch/store operations on a node pong object - if stored := db.lastPong(node.ID); stored.Unix() != 0 { - t.Errorf("pong: non-existing object: %v", stored) - } - if err := db.updateLastPong(node.ID, inst); err != nil { - t.Errorf("pong: failed to update: %v", err) - } - if stored := db.lastPong(node.ID); stored.Unix() != inst.Unix() { - t.Errorf("pong: value mismatch: have %v, want %v", stored, inst) - } - // Check fetch/store operations on a node findnode-failure object - if stored := db.findFails(node.ID); stored != 0 { - t.Errorf("find-node fails: non-existing object: %v", stored) - } - if err := db.updateFindFails(node.ID, num); err != nil { - t.Errorf("find-node fails: failed to update: %v", err) - } - if stored := db.findFails(node.ID); stored != num { - t.Errorf("find-node fails: value mismatch: have %v, want %v", stored, num) - } - // Check fetch/store operations on an actual node object - if stored := db.node(node.ID); stored != nil { - t.Errorf("node: non-existing object: %v", stored) - } - if err := db.updateNode(node); err != nil { - t.Errorf("node: failed to update: %v", err) - } - if stored := db.node(node.ID); stored == nil { - t.Errorf("node: not found") - } else if !reflect.DeepEqual(stored, node) { - t.Errorf("node: data mismatch: have %v, want %v", stored, node) - } -} - -var nodeDBSeedQueryNodes = []struct { - node *Node - pong time.Time -}{ - // This one should not be in the result set because its last - // pong time is too far in the past. - { - node: NewNode( - MustHexID("0x84d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 3}, - 30303, - 30303, - ), - pong: time.Now().Add(-3 * time.Hour), - }, - // This one shouldn't be in the result set because its - // nodeID is the local node's ID. - { - node: NewNode( - MustHexID("0x57d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 3}, - 30303, - 30303, - ), - pong: time.Now().Add(-4 * time.Second), - }, - - // These should be in the result set. - { - node: NewNode( - MustHexID("0x22d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 1}, - 30303, - 30303, - ), - pong: time.Now().Add(-2 * time.Second), - }, - { - node: NewNode( - MustHexID("0x44d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 2}, - 30303, - 30303, - ), - pong: time.Now().Add(-3 * time.Second), - }, - { - node: NewNode( - MustHexID("0xe2d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 3}, - 30303, - 30303, - ), - pong: time.Now().Add(-1 * time.Second), - }, -} - -func TestNodeDBSeedQuery(t *testing.T) { - db, _ := newNodeDB("", Version, nodeDBSeedQueryNodes[1].node.ID) - defer db.close() - - // Insert a batch of nodes for querying - for i, seed := range nodeDBSeedQueryNodes { - if err := db.updateNode(seed.node); err != nil { - t.Fatalf("node %d: failed to insert: %v", i, err) - } - if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil { - t.Fatalf("node %d: failed to insert lastPong: %v", i, err) - } - } - - // Retrieve the entire batch and check for duplicates - seeds := db.querySeeds(len(nodeDBSeedQueryNodes)*2, time.Hour) - have := make(map[NodeID]struct{}) - for _, seed := range seeds { - have[seed.ID] = struct{}{} - } - want := make(map[NodeID]struct{}) - for _, seed := range nodeDBSeedQueryNodes[2:] { - want[seed.node.ID] = struct{}{} - } - if len(seeds) != len(want) { - t.Errorf("seed count mismatch: have %v, want %v", len(seeds), len(want)) - } - for id := range have { - if _, ok := want[id]; !ok { - t.Errorf("extra seed: %v", id) - } - } - for id := range want { - if _, ok := have[id]; !ok { - t.Errorf("missing seed: %v", id) - } - } -} - -func TestNodeDBPersistency(t *testing.T) { - root, err := ioutil.TempDir("", "nodedb-") - if err != nil { - t.Fatalf("failed to create temporary data folder: %v", err) - } - defer os.RemoveAll(root) - - var ( - testKey = []byte("somekey") - testInt = int64(314) - ) - - // Create a persistent database and store some values - db, err := newNodeDB(filepath.Join(root, "database"), Version, NodeID{}) - if err != nil { - t.Fatalf("failed to create persistent database: %v", err) - } - if err := db.storeInt64(testKey, testInt); err != nil { - t.Fatalf("failed to store value: %v.", err) - } - db.close() - - // Reopen the database and check the value - db, err = newNodeDB(filepath.Join(root, "database"), Version, NodeID{}) - if err != nil { - t.Fatalf("failed to open persistent database: %v", err) - } - if val := db.fetchInt64(testKey); val != testInt { - t.Fatalf("value mismatch: have %v, want %v", val, testInt) - } - db.close() - - // Change the database version and check flush - db, err = newNodeDB(filepath.Join(root, "database"), Version+1, NodeID{}) - if err != nil { - t.Fatalf("failed to open persistent database: %v", err) - } - if val := db.fetchInt64(testKey); val != 0 { - t.Fatalf("value mismatch: have %v, want %v", val, 0) - } - db.close() -} - -var nodeDBExpirationNodes = []struct { - node *Node - pong time.Time - exp bool -}{ - { - node: NewNode( - MustHexID("0x01d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 1}, - 30303, - 30303, - ), - pong: time.Now().Add(-nodeDBNodeExpiration + time.Minute), - exp: false, - }, { - node: NewNode( - MustHexID("0x02d9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{127, 0, 0, 2}, - 30303, - 30303, - ), - pong: time.Now().Add(-nodeDBNodeExpiration - time.Minute), - exp: true, - }, -} - -func TestNodeDBExpiration(t *testing.T) { - db, _ := newNodeDB("", Version, NodeID{}) - defer db.close() - - // Add all the test nodes and set their last pong time - for i, seed := range nodeDBExpirationNodes { - if err := db.updateNode(seed.node); err != nil { - t.Fatalf("node %d: failed to insert: %v", i, err) - } - if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil { - t.Fatalf("node %d: failed to update pong: %v", i, err) - } - } - // Expire some of them, and check the rest - if err := db.expireNodes(); err != nil { - t.Fatalf("failed to expire nodes: %v", err) - } - for i, seed := range nodeDBExpirationNodes { - node := db.node(seed.node.ID) - if (node == nil && !seed.exp) || (node != nil && seed.exp) { - t.Errorf("node %d: expiration mismatch: have %v, want %v", i, node, seed.exp) - } - } -} - -func TestNodeDBSelfExpiration(t *testing.T) { - // Find a node in the tests that shouldn't expire, and assign it as self - var self NodeID - for _, node := range nodeDBExpirationNodes { - if !node.exp { - self = node.node.ID - break - } - } - db, _ := newNodeDB("", Version, self) - defer db.close() - - // Add all the test nodes and set their last pong time - for i, seed := range nodeDBExpirationNodes { - if err := db.updateNode(seed.node); err != nil { - t.Fatalf("node %d: failed to insert: %v", i, err) - } - if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil { - t.Fatalf("node %d: failed to update pong: %v", i, err) - } - } - // Expire the nodes and make sure self has been evacuated too - if err := db.expireNodes(); err != nil { - t.Fatalf("failed to expire nodes: %v", err) - } - node := db.node(self) - if node != nil { - t.Errorf("self not evacuated") - } -} diff --git a/p2p/discv5/metrics.go b/p2p/discv5/metrics.go deleted file mode 100644 index e68d53c13c..0000000000 --- a/p2p/discv5/metrics.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import "github.com/ethereum/go-ethereum/metrics" - -var ( - ingressTrafficMeter = metrics.NewRegisteredMeter("discv5/InboundTraffic", nil) - egressTrafficMeter = metrics.NewRegisteredMeter("discv5/OutboundTraffic", nil) -) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go deleted file mode 100644 index 53e00a3881..0000000000 --- a/p2p/discv5/net.go +++ /dev/null @@ -1,1269 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "bytes" - "crypto/ecdsa" - "errors" - "fmt" - "net" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/netutil" - "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" -) - -var ( - errInvalidEvent = errors.New("invalid in current state") - errNoQuery = errors.New("no pending query") -) - -const ( - autoRefreshInterval = 1 * time.Hour - bucketRefreshInterval = 1 * time.Minute - seedCount = 30 - seedMaxAge = 5 * 24 * time.Hour - lowPort = 1024 -) - -const testTopic = "foo" - -const ( - printTestImgLogs = false -) - -// Network manages the table and all protocol interaction. -type Network struct { - db *nodeDB // database of known nodes - conn transport - netrestrict *netutil.Netlist - - closed chan struct{} // closed when loop is done - closeReq chan struct{} // 'request to close' - refreshReq chan []*Node // lookups ask for refresh on this channel - refreshResp chan (<-chan struct{}) // ...and get the channel to block on from this one - read chan ingressPacket // ingress packets arrive here - timeout chan timeoutEvent - queryReq chan *findnodeQuery // lookups submit findnode queries on this channel - tableOpReq chan func() - tableOpResp chan struct{} - topicRegisterReq chan topicRegisterReq - topicSearchReq chan topicSearchReq - - // State of the main loop. - tab *Table - topictab *topicTable - ticketStore *ticketStore - nursery []*Node - nodes map[NodeID]*Node // tracks active nodes with state != known - timeoutTimers map[timeoutEvent]*time.Timer -} - -// transport is implemented by the UDP transport. -// it is an interface so we can test without opening lots of UDP -// sockets and without generating a private key. -type transport interface { - sendPing(remote *Node, remoteAddr *net.UDPAddr, topics []Topic) (hash []byte) - sendNeighbours(remote *Node, nodes []*Node) - sendFindnodeHash(remote *Node, target common.Hash) - sendTopicRegister(remote *Node, topics []Topic, topicIdx int, pong []byte) - sendTopicNodes(remote *Node, queryHash common.Hash, nodes []*Node) - - send(remote *Node, ptype nodeEvent, p interface{}) (hash []byte) - - localAddr() *net.UDPAddr - Close() -} - -type findnodeQuery struct { - remote *Node - target common.Hash - reply chan<- []*Node -} - -type topicRegisterReq struct { - add bool - topic Topic -} - -type topicSearchReq struct { - topic Topic - found chan<- *Node - lookup chan<- bool - delay time.Duration -} - -type topicSearchResult struct { - target lookupInfo - nodes []*Node -} - -type timeoutEvent struct { - ev nodeEvent - node *Node -} - -func newNetwork(conn transport, ourPubkey ecdsa.PublicKey, dbPath string, netrestrict *netutil.Netlist) (*Network, error) { - ourID := PubkeyID(&ourPubkey) - - var db *nodeDB - if dbPath != "" { - var err error - if db, err = newNodeDB(dbPath, Version, ourID); err != nil { - return nil, err - } - } - - tab := newTable(ourID, conn.localAddr()) - net := &Network{ - db: db, - conn: conn, - netrestrict: netrestrict, - tab: tab, - topictab: newTopicTable(db, tab.self), - ticketStore: newTicketStore(), - refreshReq: make(chan []*Node), - refreshResp: make(chan (<-chan struct{})), - closed: make(chan struct{}), - closeReq: make(chan struct{}), - read: make(chan ingressPacket, 100), - timeout: make(chan timeoutEvent), - timeoutTimers: make(map[timeoutEvent]*time.Timer), - tableOpReq: make(chan func()), - tableOpResp: make(chan struct{}), - queryReq: make(chan *findnodeQuery), - topicRegisterReq: make(chan topicRegisterReq), - topicSearchReq: make(chan topicSearchReq), - nodes: make(map[NodeID]*Node), - } - go net.loop() - return net, nil -} - -// Close terminates the network listener and flushes the node database. -func (net *Network) Close() { - net.conn.Close() - select { - case <-net.closed: - case net.closeReq <- struct{}{}: - <-net.closed - } -} - -// Self returns the local node. -// The returned node should not be modified by the caller. -func (net *Network) Self() *Node { - return net.tab.self -} - -// ReadRandomNodes fills the given slice with random nodes from the -// table. It will not write the same node more than once. The nodes in -// the slice are copies and can be modified by the caller. -func (net *Network) ReadRandomNodes(buf []*Node) (n int) { - net.reqTableOp(func() { n = net.tab.readRandomNodes(buf) }) - return n -} - -// SetFallbackNodes sets the initial points of contact. These nodes -// are used to connect to the network if the table is empty and there -// are no known nodes in the database. -func (net *Network) SetFallbackNodes(nodes []*Node) error { - nursery := make([]*Node, 0, len(nodes)) - for _, n := range nodes { - if err := n.validateComplete(); err != nil { - return fmt.Errorf("bad bootstrap/fallback node %q (%v)", n, err) - } - // Recompute cpy.sha because the node might not have been - // created by NewNode or ParseNode. - cpy := *n - cpy.sha = crypto.Keccak256Hash(n.ID[:]) - nursery = append(nursery, &cpy) - } - net.reqRefresh(nursery) - return nil -} - -// Resolve searches for a specific node with the given ID. -// It returns nil if the node could not be found. -func (net *Network) Resolve(targetID NodeID) *Node { - result := net.lookup(crypto.Keccak256Hash(targetID[:]), true) - for _, n := range result { - if n.ID == targetID { - return n - } - } - return nil -} - -// Lookup performs a network search for nodes close -// to the given target. It approaches the target by querying -// nodes that are closer to it on each iteration. -// The given target does not need to be an actual node -// identifier. -// -// The local node may be included in the result. -func (net *Network) Lookup(targetID NodeID) []*Node { - return net.lookup(crypto.Keccak256Hash(targetID[:]), false) -} - -func (net *Network) lookup(target common.Hash, stopOnMatch bool) []*Node { - var ( - asked = make(map[NodeID]bool) - seen = make(map[NodeID]bool) - reply = make(chan []*Node, alpha) - result = nodesByDistance{target: target} - pendingQueries = 0 - ) - // Get initial answers from the local node. - result.push(net.tab.self, bucketSize) - for { - // Ask the α closest nodes that we haven't asked yet. - for i := 0; i < len(result.entries) && pendingQueries < alpha; i++ { - n := result.entries[i] - if !asked[n.ID] { - asked[n.ID] = true - pendingQueries++ - net.reqQueryFindnode(n, target, reply) - } - } - if pendingQueries == 0 { - // We have asked all closest nodes, stop the search. - break - } - // Wait for the next reply. - select { - case nodes := <-reply: - for _, n := range nodes { - if n != nil && !seen[n.ID] { - seen[n.ID] = true - result.push(n, bucketSize) - if stopOnMatch && n.sha == target { - return result.entries - } - } - } - pendingQueries-- - case <-time.After(respTimeout): - // forget all pending requests, start new ones - pendingQueries = 0 - reply = make(chan []*Node, alpha) - } - } - return result.entries -} - -func (net *Network) RegisterTopic(topic Topic, stop <-chan struct{}) { - select { - case net.topicRegisterReq <- topicRegisterReq{true, topic}: - case <-net.closed: - return - } - select { - case <-net.closed: - case <-stop: - select { - case net.topicRegisterReq <- topicRegisterReq{false, topic}: - case <-net.closed: - } - } -} - -func (net *Network) SearchTopic(topic Topic, setPeriod <-chan time.Duration, found chan<- *Node, lookup chan<- bool) { - for { - select { - case <-net.closed: - return - case delay, ok := <-setPeriod: - select { - case net.topicSearchReq <- topicSearchReq{topic: topic, found: found, lookup: lookup, delay: delay}: - case <-net.closed: - return - } - if !ok { - return - } - } - } -} - -func (net *Network) reqRefresh(nursery []*Node) <-chan struct{} { - select { - case net.refreshReq <- nursery: - return <-net.refreshResp - case <-net.closed: - return net.closed - } -} - -func (net *Network) reqQueryFindnode(n *Node, target common.Hash, reply chan []*Node) bool { - q := &findnodeQuery{remote: n, target: target, reply: reply} - select { - case net.queryReq <- q: - return true - case <-net.closed: - return false - } -} - -func (net *Network) reqReadPacket(pkt ingressPacket) { - select { - case net.read <- pkt: - case <-net.closed: - } -} - -func (net *Network) reqTableOp(f func()) (called bool) { - select { - case net.tableOpReq <- f: - <-net.tableOpResp - return true - case <-net.closed: - return false - } -} - -// TODO: external address handling. - -type topicSearchInfo struct { - lookupChn chan<- bool - period time.Duration -} - -const maxSearchCount = 5 - -func (net *Network) loop() { - var ( - refreshTimer = time.NewTicker(autoRefreshInterval) - bucketRefreshTimer = time.NewTimer(bucketRefreshInterval) - refreshDone chan struct{} // closed when the 'refresh' lookup has ended - ) - defer refreshTimer.Stop() - defer bucketRefreshTimer.Stop() - - // Tracking the next ticket to register. - var ( - nextTicket *ticketRef - nextRegisterTimer *time.Timer - nextRegisterTime <-chan time.Time - ) - defer func() { - if nextRegisterTimer != nil { - nextRegisterTimer.Stop() - } - }() - resetNextTicket := func() { - ticket, timeout := net.ticketStore.nextFilteredTicket() - if nextTicket != ticket { - nextTicket = ticket - if nextRegisterTimer != nil { - nextRegisterTimer.Stop() - nextRegisterTime = nil - } - if ticket != nil { - nextRegisterTimer = time.NewTimer(timeout) - nextRegisterTime = nextRegisterTimer.C - } - } - } - - // Tracking registration and search lookups. - var ( - topicRegisterLookupTarget lookupInfo - topicRegisterLookupDone chan []*Node - topicRegisterLookupTick = time.NewTimer(0) - searchReqWhenRefreshDone []topicSearchReq - searchInfo = make(map[Topic]topicSearchInfo) - activeSearchCount int - ) - defer topicRegisterLookupTick.Stop() - topicSearchLookupDone := make(chan topicSearchResult, 100) - topicSearch := make(chan Topic, 100) - <-topicRegisterLookupTick.C - - statsDump := time.NewTicker(10 * time.Second) - defer statsDump.Stop() - -loop: - for { - resetNextTicket() - - select { - case <-net.closeReq: - log.Trace("<-net.closeReq") - break loop - - // Ingress packet handling. - case pkt := <-net.read: - //fmt.Println("read", pkt.ev) - log.Trace("<-net.read") - n := net.internNode(&pkt) - prestate := n.state - status := "ok" - if err := net.handle(n, pkt.ev, &pkt); err != nil { - status = err.Error() - } - log.Trace("", "msg", log.Lazy{Fn: func() string { - return fmt.Sprintf("<<< (%d) %v from %x@%v: %v -> %v (%v)", - net.tab.count, pkt.ev, pkt.remoteID[:8], pkt.remoteAddr, prestate, n.state, status) - }}) - // TODO: persist state if n.state goes >= known, delete if it goes <= known - - // State transition timeouts. - case timeout := <-net.timeout: - log.Trace("<-net.timeout") - if net.timeoutTimers[timeout] == nil { - // Stale timer (was aborted). - continue - } - delete(net.timeoutTimers, timeout) - prestate := timeout.node.state - status := "ok" - if err := net.handle(timeout.node, timeout.ev, nil); err != nil { - status = err.Error() - } - log.Trace("", "msg", log.Lazy{Fn: func() string { - return fmt.Sprintf("--- (%d) %v for %x@%v: %v -> %v (%v)", - net.tab.count, timeout.ev, timeout.node.ID[:8], timeout.node.addr(), prestate, timeout.node.state, status) - }}) - - // Querying. - case q := <-net.queryReq: - log.Trace("<-net.queryReq") - if !q.start(net) { - q.remote.deferQuery(q) - } - - // Interacting with the table. - case f := <-net.tableOpReq: - log.Trace("<-net.tableOpReq") - f() - net.tableOpResp <- struct{}{} - - // Topic registration stuff. - case req := <-net.topicRegisterReq: - log.Trace("<-net.topicRegisterReq") - if !req.add { - net.ticketStore.removeRegisterTopic(req.topic) - continue - } - net.ticketStore.addTopic(req.topic, true) - // If we're currently waiting idle (nothing to look up), give the ticket store a - // chance to start it sooner. This should speed up convergence of the radius - // determination for new topics. - // if topicRegisterLookupDone == nil { - if topicRegisterLookupTarget.target == (common.Hash{}) { - log.Trace("topicRegisterLookupTarget == null") - if topicRegisterLookupTick.Stop() { - <-topicRegisterLookupTick.C - } - target, delay := net.ticketStore.nextRegisterLookup() - topicRegisterLookupTarget = target - topicRegisterLookupTick.Reset(delay) - } - - case nodes := <-topicRegisterLookupDone: - log.Trace("<-topicRegisterLookupDone") - net.ticketStore.registerLookupDone(topicRegisterLookupTarget, nodes, func(n *Node) []byte { - net.ping(n, n.addr()) - return n.pingEcho - }) - target, delay := net.ticketStore.nextRegisterLookup() - topicRegisterLookupTarget = target - topicRegisterLookupTick.Reset(delay) - topicRegisterLookupDone = nil - - case <-topicRegisterLookupTick.C: - log.Trace("<-topicRegisterLookupTick") - if (topicRegisterLookupTarget.target == common.Hash{}) { - target, delay := net.ticketStore.nextRegisterLookup() - topicRegisterLookupTarget = target - topicRegisterLookupTick.Reset(delay) - topicRegisterLookupDone = nil - } else { - topicRegisterLookupDone = make(chan []*Node) - target := topicRegisterLookupTarget.target - go func() { topicRegisterLookupDone <- net.lookup(target, false) }() - } - - case <-nextRegisterTime: - log.Trace("<-nextRegisterTime") - net.ticketStore.ticketRegistered(*nextTicket) - //fmt.Println("sendTopicRegister", nextTicket.t.node.addr().String(), nextTicket.t.topics, nextTicket.idx, nextTicket.t.pong) - net.conn.sendTopicRegister(nextTicket.t.node, nextTicket.t.topics, nextTicket.idx, nextTicket.t.pong) - - case req := <-net.topicSearchReq: - if refreshDone == nil { - log.Trace("<-net.topicSearchReq") - info, ok := searchInfo[req.topic] - if ok { - if req.delay == time.Duration(0) { - delete(searchInfo, req.topic) - net.ticketStore.removeSearchTopic(req.topic) - } else { - info.period = req.delay - searchInfo[req.topic] = info - } - continue - } - if req.delay != time.Duration(0) { - var info topicSearchInfo - info.period = req.delay - info.lookupChn = req.lookup - searchInfo[req.topic] = info - net.ticketStore.addSearchTopic(req.topic, req.found) - topicSearch <- req.topic - } - } else { - searchReqWhenRefreshDone = append(searchReqWhenRefreshDone, req) - } - - case topic := <-topicSearch: - if activeSearchCount < maxSearchCount { - activeSearchCount++ - target := net.ticketStore.nextSearchLookup(topic) - go func() { - nodes := net.lookup(target.target, false) - topicSearchLookupDone <- topicSearchResult{target: target, nodes: nodes} - }() - } - period := searchInfo[topic].period - if period != time.Duration(0) { - go func() { - time.Sleep(period) - topicSearch <- topic - }() - } - - case res := <-topicSearchLookupDone: - activeSearchCount-- - if lookupChn := searchInfo[res.target.topic].lookupChn; lookupChn != nil { - lookupChn <- net.ticketStore.radius[res.target.topic].converged - } - net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte { - if n.state != nil && n.state.canQuery { - return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration - } - if n.state == unknown { - net.ping(n, n.addr()) - } - return nil - }) - - case <-statsDump.C: - log.Trace("<-statsDump.C") - /*r, ok := net.ticketStore.radius[testTopic] - if !ok { - fmt.Printf("(%x) no radius @ %v\n", net.tab.self.ID[:8], time.Now()) - } else { - topics := len(net.ticketStore.tickets) - tickets := len(net.ticketStore.nodes) - rad := r.radius / (maxRadius/10000+1) - fmt.Printf("(%x) topics:%d radius:%d tickets:%d @ %v\n", net.tab.self.ID[:8], topics, rad, tickets, time.Now()) - }*/ - - tm := mclock.Now() - for topic, r := range net.ticketStore.radius { - if printTestImgLogs { - rad := r.radius / (maxRadius/1000000 + 1) - minrad := r.minRadius / (maxRadius/1000000 + 1) - fmt.Printf("*R %d %v %016x %v\n", tm/1000000, topic, net.tab.self.sha[:8], rad) - fmt.Printf("*MR %d %v %016x %v\n", tm/1000000, topic, net.tab.self.sha[:8], minrad) - } - } - for topic, t := range net.topictab.topics { - wp := t.wcl.nextWaitPeriod(tm) - if printTestImgLogs { - fmt.Printf("*W %d %v %016x %d\n", tm/1000000, topic, net.tab.self.sha[:8], wp/1000000) - } - } - - // Periodic / lookup-initiated bucket refresh. - case <-refreshTimer.C: - log.Trace("<-refreshTimer.C") - // TODO: ideally we would start the refresh timer after - // fallback nodes have been set for the first time. - if refreshDone == nil { - refreshDone = make(chan struct{}) - net.refresh(refreshDone) - } - case <-bucketRefreshTimer.C: - target := net.tab.chooseBucketRefreshTarget() - go func() { - net.lookup(target, false) - bucketRefreshTimer.Reset(bucketRefreshInterval) - }() - case newNursery := <-net.refreshReq: - log.Trace("<-net.refreshReq") - if newNursery != nil { - net.nursery = newNursery - } - if refreshDone == nil { - refreshDone = make(chan struct{}) - net.refresh(refreshDone) - } - net.refreshResp <- refreshDone - case <-refreshDone: - log.Trace("<-net.refreshDone", "table size", net.tab.count) - if net.tab.count != 0 { - refreshDone = nil - list := searchReqWhenRefreshDone - searchReqWhenRefreshDone = nil - go func() { - for _, req := range list { - net.topicSearchReq <- req - } - }() - } else { - refreshDone = make(chan struct{}) - net.refresh(refreshDone) - } - } - } - log.Trace("loop stopped") - - log.Debug("shutting down") - if net.conn != nil { - net.conn.Close() - } - // TODO: wait for pending refresh. - // if refreshDone != nil { - // <-refreshResults - // } - // Cancel all pending timeouts. - for _, timer := range net.timeoutTimers { - timer.Stop() - } - if net.db != nil { - net.db.close() - } - close(net.closed) -} - -// Everything below runs on the Network.loop goroutine -// and can modify Node, Table and Network at any time without locking. - -func (net *Network) refresh(done chan<- struct{}) { - var seeds []*Node - if net.db != nil { - seeds = net.db.querySeeds(seedCount, seedMaxAge) - } - if len(seeds) == 0 { - seeds = net.nursery - } - if len(seeds) == 0 { - log.Trace("no seed nodes found") - time.AfterFunc(time.Second*10, func() { close(done) }) - return - } - for _, n := range seeds { - log.Debug("", "msg", log.Lazy{Fn: func() string { - var age string - if net.db != nil { - age = time.Since(net.db.lastPong(n.ID)).String() - } else { - age = "unknown" - } - return fmt.Sprintf("seed node (age %s): %v", age, n) - }}) - n = net.internNodeFromDB(n) - if n.state == unknown { - net.transition(n, verifyinit) - } - // Force-add the seed node so Lookup does something. - // It will be deleted again if verification fails. - net.tab.add(n) - } - // Start self lookup to fill up the buckets. - go func() { - net.Lookup(net.tab.self.ID) - close(done) - }() -} - -// Node Interning. - -func (net *Network) internNode(pkt *ingressPacket) *Node { - if n := net.nodes[pkt.remoteID]; n != nil { - n.IP = pkt.remoteAddr.IP - n.UDP = uint16(pkt.remoteAddr.Port) - n.TCP = uint16(pkt.remoteAddr.Port) - return n - } - n := NewNode(pkt.remoteID, pkt.remoteAddr.IP, uint16(pkt.remoteAddr.Port), uint16(pkt.remoteAddr.Port)) - n.state = unknown - net.nodes[pkt.remoteID] = n - return n -} - -func (net *Network) internNodeFromDB(dbn *Node) *Node { - if n := net.nodes[dbn.ID]; n != nil { - return n - } - n := NewNode(dbn.ID, dbn.IP, dbn.UDP, dbn.TCP) - n.state = unknown - net.nodes[n.ID] = n - return n -} - -func (net *Network) internNodeFromNeighbours(sender *net.UDPAddr, rn rpcNode) (n *Node, err error) { - if rn.ID == net.tab.self.ID { - return nil, errors.New("is self") - } - if rn.UDP <= lowPort { - return nil, errors.New("low port") - } - n = net.nodes[rn.ID] - if n == nil { - // We haven't seen this node before. - n, err = nodeFromRPC(sender, rn) - if net.netrestrict != nil && !net.netrestrict.Contains(n.IP) { - return n, errors.New("not contained in netrestrict whitelist") - } - if err == nil { - n.state = unknown - net.nodes[n.ID] = n - } - return n, err - } - if !n.IP.Equal(rn.IP) || n.UDP != rn.UDP || n.TCP != rn.TCP { - if n.state == known { - // reject address change if node is known by us - err = fmt.Errorf("metadata mismatch: got %v, want %v", rn, n) - } else { - // accept otherwise; this will be handled nicer with signed ENRs - n.IP = rn.IP - n.UDP = rn.UDP - n.TCP = rn.TCP - } - } - return n, err -} - -// nodeNetGuts is embedded in Node and contains fields. -type nodeNetGuts struct { - // This is a cached copy of sha3(ID) which is used for node - // distance calculations. This is part of Node in order to make it - // possible to write tests that need a node at a certain distance. - // In those tests, the content of sha will not actually correspond - // with ID. - sha common.Hash - - // State machine fields. Access to these fields - // is restricted to the Network.loop goroutine. - state *nodeState - pingEcho []byte // hash of last ping sent by us - pingTopics []Topic // topic set sent by us in last ping - deferredQueries []*findnodeQuery // queries that can't be sent yet - pendingNeighbours *findnodeQuery // current query, waiting for reply - queryTimeouts int -} - -func (n *nodeNetGuts) deferQuery(q *findnodeQuery) { - n.deferredQueries = append(n.deferredQueries, q) -} - -func (n *nodeNetGuts) startNextQuery(net *Network) { - if len(n.deferredQueries) == 0 { - return - } - nextq := n.deferredQueries[0] - if nextq.start(net) { - n.deferredQueries = append(n.deferredQueries[:0], n.deferredQueries[1:]...) - } -} - -func (q *findnodeQuery) start(net *Network) bool { - // Satisfy queries against the local node directly. - if q.remote == net.tab.self { - closest := net.tab.closest(q.target, bucketSize) - q.reply <- closest.entries - return true - } - if q.remote.state.canQuery && q.remote.pendingNeighbours == nil { - net.conn.sendFindnodeHash(q.remote, q.target) - net.timedEvent(respTimeout, q.remote, neighboursTimeout) - q.remote.pendingNeighbours = q - return true - } - // If the node is not known yet, it won't accept queries. - // Initiate the transition to known. - // The request will be sent later when the node reaches known state. - if q.remote.state == unknown { - net.transition(q.remote, verifyinit) - } - return false -} - -// Node Events (the input to the state machine). - -type nodeEvent uint - -//go:generate stringer -type=nodeEvent - -const ( - - // Packet type events. - // These correspond to packet types in the UDP protocol. - pingPacket = iota + 1 - pongPacket - findnodePacket - neighborsPacket - findnodeHashPacket - topicRegisterPacket - topicQueryPacket - topicNodesPacket - - // Non-packet events. - // Event values in this category are allocated outside - // the packet type range (packet types are encoded as a single byte). - pongTimeout nodeEvent = iota + 256 - pingTimeout - neighboursTimeout -) - -// Node State Machine. - -type nodeState struct { - name string - handle func(*Network, *Node, nodeEvent, *ingressPacket) (next *nodeState, err error) - enter func(*Network, *Node) - canQuery bool -} - -func (s *nodeState) String() string { - return s.name -} - -var ( - unknown *nodeState - verifyinit *nodeState - verifywait *nodeState - remoteverifywait *nodeState - known *nodeState - contested *nodeState - unresponsive *nodeState -) - -func init() { - unknown = &nodeState{ - name: "unknown", - enter: func(net *Network, n *Node) { - net.tab.delete(n) - n.pingEcho = nil - // Abort active queries. - for _, q := range n.deferredQueries { - q.reply <- nil - } - n.deferredQueries = nil - if n.pendingNeighbours != nil { - n.pendingNeighbours.reply <- nil - n.pendingNeighbours = nil - } - n.queryTimeouts = 0 - }, - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pingPacket: - net.handlePing(n, pkt) - net.ping(n, pkt.remoteAddr) - return verifywait, nil - default: - return unknown, errInvalidEvent - } - }, - } - - verifyinit = &nodeState{ - name: "verifyinit", - enter: func(net *Network, n *Node) { - net.ping(n, n.addr()) - }, - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pingPacket: - net.handlePing(n, pkt) - return verifywait, nil - case pongPacket: - err := net.handleKnownPong(n, pkt) - return remoteverifywait, err - case pongTimeout: - return unknown, nil - default: - return verifyinit, errInvalidEvent - } - }, - } - - verifywait = &nodeState{ - name: "verifywait", - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pingPacket: - net.handlePing(n, pkt) - return verifywait, nil - case pongPacket: - err := net.handleKnownPong(n, pkt) - return known, err - case pongTimeout: - return unknown, nil - default: - return verifywait, errInvalidEvent - } - }, - } - - remoteverifywait = &nodeState{ - name: "remoteverifywait", - enter: func(net *Network, n *Node) { - net.timedEvent(respTimeout, n, pingTimeout) - }, - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pingPacket: - net.handlePing(n, pkt) - return remoteverifywait, nil - case pingTimeout: - return known, nil - default: - return remoteverifywait, errInvalidEvent - } - }, - } - - known = &nodeState{ - name: "known", - canQuery: true, - enter: func(net *Network, n *Node) { - n.queryTimeouts = 0 - n.startNextQuery(net) - // Insert into the table and start revalidation of the last node - // in the bucket if it is full. - last := net.tab.add(n) - if last != nil && last.state == known { - // TODO: do this asynchronously - net.transition(last, contested) - } - }, - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pingPacket: - net.handlePing(n, pkt) - return known, nil - case pongPacket: - err := net.handleKnownPong(n, pkt) - return known, err - default: - return net.handleQueryEvent(n, ev, pkt) - } - }, - } - - contested = &nodeState{ - name: "contested", - canQuery: true, - enter: func(net *Network, n *Node) { - net.ping(n, n.addr()) - }, - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pongPacket: - // Node is still alive. - err := net.handleKnownPong(n, pkt) - return known, err - case pongTimeout: - net.tab.deleteReplace(n) - return unresponsive, nil - case pingPacket: - net.handlePing(n, pkt) - return contested, nil - default: - return net.handleQueryEvent(n, ev, pkt) - } - }, - } - - unresponsive = &nodeState{ - name: "unresponsive", - canQuery: true, - handle: func(net *Network, n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case pingPacket: - net.handlePing(n, pkt) - return known, nil - case pongPacket: - err := net.handleKnownPong(n, pkt) - return known, err - default: - return net.handleQueryEvent(n, ev, pkt) - } - }, - } -} - -// handle processes packets sent by n and events related to n. -func (net *Network) handle(n *Node, ev nodeEvent, pkt *ingressPacket) error { - //fmt.Println("handle", n.addr().String(), n.state, ev) - if pkt != nil { - if err := net.checkPacket(n, ev, pkt); err != nil { - //fmt.Println("check err:", err) - return err - } - // Start the background expiration goroutine after the first - // successful communication. Subsequent calls have no effect if it - // is already running. We do this here instead of somewhere else - // so that the search for seed nodes also considers older nodes - // that would otherwise be removed by the expirer. - if net.db != nil { - net.db.ensureExpirer() - } - } - if ev == pongTimeout { - n.pingEcho = nil // clean up if pongtimeout - } - if n.state == nil { - n.state = unknown //??? - } - next, err := n.state.handle(net, n, ev, pkt) - net.transition(n, next) - //fmt.Println("new state:", n.state) - return err -} - -func (net *Network) checkPacket(n *Node, ev nodeEvent, pkt *ingressPacket) error { - // Replay prevention checks. - switch ev { - case pingPacket, findnodeHashPacket, neighborsPacket: - // TODO: check date is > last date seen - // TODO: check ping version - case pongPacket: - if !bytes.Equal(pkt.data.(*pong).ReplyTok, n.pingEcho) { - // fmt.Println("pong reply token mismatch") - return fmt.Errorf("pong reply token mismatch") - } - n.pingEcho = nil - } - // Address validation. - // TODO: Ideally we would do the following: - // - reject all packets with wrong address except ping. - // - for ping with new address, transition to verifywait but keep the - // previous node (with old address) around. if the new one reaches known, - // swap it out. - return nil -} - -func (net *Network) transition(n *Node, next *nodeState) { - if n.state != next { - n.state = next - if next.enter != nil { - next.enter(net, n) - } - } - - // TODO: persist/unpersist node -} - -func (net *Network) timedEvent(d time.Duration, n *Node, ev nodeEvent) { - timeout := timeoutEvent{ev, n} - net.timeoutTimers[timeout] = time.AfterFunc(d, func() { - select { - case net.timeout <- timeout: - case <-net.closed: - } - }) -} - -func (net *Network) abortTimedEvent(n *Node, ev nodeEvent) { - timer := net.timeoutTimers[timeoutEvent{ev, n}] - if timer != nil { - timer.Stop() - delete(net.timeoutTimers, timeoutEvent{ev, n}) - } -} - -func (net *Network) ping(n *Node, addr *net.UDPAddr) { - //fmt.Println("ping", n.addr().String(), n.ID.String(), n.sha.Hex()) - if n.pingEcho != nil || n.ID == net.tab.self.ID { - //fmt.Println(" not sent") - return - } - log.Trace("Pinging remote node", "node", n.ID) - n.pingTopics = net.ticketStore.regTopicSet() - n.pingEcho = net.conn.sendPing(n, addr, n.pingTopics) - net.timedEvent(respTimeout, n, pongTimeout) -} - -func (net *Network) handlePing(n *Node, pkt *ingressPacket) { - log.Trace("Handling remote ping", "node", n.ID) - ping := pkt.data.(*ping) - n.TCP = ping.From.TCP - t := net.topictab.getTicket(n, ping.Topics) - - pong := &pong{ - To: makeEndpoint(n.addr(), n.TCP), // TODO: maybe use known TCP port from DB - ReplyTok: pkt.hash, - Expiration: uint64(time.Now().Add(expiration).Unix()), - } - ticketToPong(t, pong) - net.conn.send(n, pongPacket, pong) -} - -func (net *Network) handleKnownPong(n *Node, pkt *ingressPacket) error { - log.Trace("Handling known pong", "node", n.ID) - net.abortTimedEvent(n, pongTimeout) - now := mclock.Now() - ticket, err := pongToTicket(now, n.pingTopics, n, pkt) - if err == nil { - // fmt.Printf("(%x) ticket: %+v\n", net.tab.self.ID[:8], pkt.data) - net.ticketStore.addTicket(now, pkt.data.(*pong).ReplyTok, ticket) - } else { - log.Trace("Failed to convert pong to ticket", "err", err) - } - n.pingEcho = nil - n.pingTopics = nil - return err -} - -func (net *Network) handleQueryEvent(n *Node, ev nodeEvent, pkt *ingressPacket) (*nodeState, error) { - switch ev { - case findnodePacket: - target := crypto.Keccak256Hash(pkt.data.(*findnode).Target[:]) - results := net.tab.closest(target, bucketSize).entries - net.conn.sendNeighbours(n, results) - return n.state, nil - case neighborsPacket: - err := net.handleNeighboursPacket(n, pkt) - return n.state, err - case neighboursTimeout: - if n.pendingNeighbours != nil { - n.pendingNeighbours.reply <- nil - n.pendingNeighbours = nil - } - n.queryTimeouts++ - if n.queryTimeouts > maxFindnodeFailures && n.state == known { - return contested, errors.New("too many timeouts") - } - return n.state, nil - - // v5 - - case findnodeHashPacket: - results := net.tab.closest(pkt.data.(*findnodeHash).Target, bucketSize).entries - net.conn.sendNeighbours(n, results) - return n.state, nil - case topicRegisterPacket: - //fmt.Println("got topicRegisterPacket") - regdata := pkt.data.(*topicRegister) - pong, err := net.checkTopicRegister(regdata) - if err != nil { - //fmt.Println(err) - return n.state, fmt.Errorf("bad waiting ticket: %v", err) - } - net.topictab.useTicket(n, pong.TicketSerial, regdata.Topics, int(regdata.Idx), pong.Expiration, pong.WaitPeriods) - return n.state, nil - case topicQueryPacket: - // TODO: handle expiration - topic := pkt.data.(*topicQuery).Topic - results := net.topictab.getEntries(topic) - if _, ok := net.ticketStore.tickets[topic]; ok { - results = append(results, net.tab.self) // we're not registering in our own table but if we're advertising, return ourselves too - } - if len(results) > 10 { - results = results[:10] - } - var hash common.Hash - copy(hash[:], pkt.hash) - net.conn.sendTopicNodes(n, hash, results) - return n.state, nil - case topicNodesPacket: - p := pkt.data.(*topicNodes) - if net.ticketStore.gotTopicNodes(n, p.Echo, p.Nodes) { - n.queryTimeouts++ - if n.queryTimeouts > maxFindnodeFailures && n.state == known { - return contested, errors.New("too many timeouts") - } - } - return n.state, nil - - default: - return n.state, errInvalidEvent - } -} - -func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) { - var pongpkt ingressPacket - if err := decodePacket(data.Pong, &pongpkt); err != nil { - return nil, err - } - if pongpkt.ev != pongPacket { - return nil, errors.New("is not pong packet") - } - if pongpkt.remoteID != net.tab.self.ID { - return nil, errors.New("not signed by us") - } - // check that we previously authorised all topics - // that the other side is trying to register. - if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash { - return nil, errors.New("topic hash mismatch") - } - if data.Idx >= uint(len(data.Topics)) { - return nil, errors.New("topic index out of range") - } - return pongpkt.data.(*pong), nil -} - -func rlpHash(x interface{}) (h common.Hash) { - hw := sha3.NewLegacyKeccak256() - rlp.Encode(hw, x) - hw.Sum(h[:0]) - return h -} - -func (net *Network) handleNeighboursPacket(n *Node, pkt *ingressPacket) error { - if n.pendingNeighbours == nil { - return errNoQuery - } - net.abortTimedEvent(n, neighboursTimeout) - - req := pkt.data.(*neighbors) - nodes := make([]*Node, len(req.Nodes)) - for i, rn := range req.Nodes { - nn, err := net.internNodeFromNeighbours(pkt.remoteAddr, rn) - if err != nil { - log.Debug(fmt.Sprintf("invalid neighbour (%v) from %x@%v: %v", rn.IP, n.ID[:8], pkt.remoteAddr, err)) - continue - } - nodes[i] = nn - // Start validation of query results immediately. - // This fills the table quickly. - // TODO: generates way too many packets, maybe do it via queue. - if nn.state == unknown { - net.transition(nn, verifyinit) - } - } - // TODO: don't ignore second packet - n.pendingNeighbours.reply <- nodes - n.pendingNeighbours = nil - // Now that this query is done, start the next one. - n.startNextQuery(net) - return nil -} diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go deleted file mode 100644 index 29321bc86f..0000000000 --- a/p2p/discv5/net_test.go +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "net" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" -) - -func TestNetwork_Lookup(t *testing.T) { - key, _ := crypto.GenerateKey() - network, err := newNetwork(lookupTestnet, key.PublicKey, "", nil) - if err != nil { - t.Fatal(err) - } - lookupTestnet.net = network - defer network.Close() - - // lookup on empty table returns no nodes - // if results := network.Lookup(lookupTestnet.target, false); len(results) > 0 { - // t.Fatalf("lookup on empty table returned %d results: %#v", len(results), results) - // } - // seed table with initial node (otherwise lookup will terminate immediately) - seeds := []*Node{NewNode(lookupTestnet.dists[256][0], net.IP{10, 0, 2, 99}, lowPort+256, 999)} - if err := network.SetFallbackNodes(seeds); err != nil { - t.Fatal(err) - } - time.Sleep(3 * time.Second) - - results := network.Lookup(lookupTestnet.target) - t.Logf("results:") - for _, e := range results { - t.Logf(" ld=%d, %x", logdist(lookupTestnet.targetSha, e.sha), e.sha[:]) - } - if len(results) != bucketSize { - t.Errorf("wrong number of results: got %d, want %d", len(results), bucketSize) - } - if hasDuplicates(results) { - t.Errorf("result set contains duplicate entries") - } - if !sortedByDistanceTo(lookupTestnet.targetSha, results) { - t.Errorf("result set not sorted by distance to target") - } - // TODO: check result nodes are actually closest -} - -// This is the test network for the Lookup test. -// The nodes were obtained by running testnet.mine with a random NodeID as target. -var lookupTestnet = &preminedTestnet{ - target: MustHexID("166aea4f556532c6d34e8b740e5d314af7e9ac0ca79833bd751d6b665f12dfd38ec563c363b32f02aef4a80b44fd3def94612d497b99cb5f17fd24de454927ec"), - targetSha: common.Hash{0x5c, 0x94, 0x4e, 0xe5, 0x1c, 0x5a, 0xe9, 0xf7, 0x2a, 0x95, 0xec, 0xcb, 0x8a, 0xed, 0x3, 0x74, 0xee, 0xcb, 0x51, 0x19, 0xd7, 0x20, 0xcb, 0xea, 0x68, 0x13, 0xe8, 0xe0, 0xd6, 0xad, 0x92, 0x61}, - dists: [257][]NodeID{ - 240: { - MustHexID("2001ad5e3e80c71b952161bc0186731cf5ffe942d24a79230a0555802296238e57ea7a32f5b6f18564eadc1c65389448481f8c9338df0a3dbd18f708cbc2cbcb"), - MustHexID("6ba3f4f57d084b6bf94cc4555b8c657e4a8ac7b7baf23c6874efc21dd1e4f56b7eb2721e07f5242d2f1d8381fc8cae535e860197c69236798ba1ad231b105794"), - }, - 244: { - MustHexID("696ba1f0a9d55c59246f776600542a9e6432490f0cd78f8bb55a196918df2081a9b521c3c3ba48e465a75c10768807717f8f689b0b4adce00e1c75737552a178"), - }, - 246: { - MustHexID("d6d32178bdc38416f46ffb8b3ec9e4cb2cfff8d04dd7e4311a70e403cb62b10be1b447311b60b4f9ee221a8131fc2cbd45b96dd80deba68a949d467241facfa8"), - MustHexID("3ea3d04a43a3dfb5ac11cffc2319248cf41b6279659393c2f55b8a0a5fc9d12581a9d97ef5d8ff9b5abf3321a290e8f63a4f785f450dc8a672aba3ba2ff4fdab"), - MustHexID("2fc897f05ae585553e5c014effd3078f84f37f9333afacffb109f00ca8e7a3373de810a3946be971cbccdfd40249f9fe7f322118ea459ac71acca85a1ef8b7f4"), - }, - 247: { - MustHexID("3155e1427f85f10a5c9a7755877748041af1bcd8d474ec065eb33df57a97babf54bfd2103575fa829115d224c523596b401065a97f74010610fce76382c0bf32"), - MustHexID("312c55512422cf9b8a4097e9a6ad79402e87a15ae909a4bfefa22398f03d20951933beea1e4dfa6f968212385e829f04c2d314fc2d4e255e0d3bc08792b069db"), - MustHexID("38643200b172dcfef857492156971f0e6aa2c538d8b74010f8e140811d53b98c765dd2d96126051913f44582e8c199ad7c6d6819e9a56483f637feaac9448aac"), - MustHexID("8dcab8618c3253b558d459da53bd8fa68935a719aff8b811197101a4b2b47dd2d47295286fc00cc081bb542d760717d1bdd6bec2c37cd72eca367d6dd3b9df73"), - MustHexID("8b58c6073dd98bbad4e310b97186c8f822d3a5c7d57af40e2136e88e315afd115edb27d2d0685a908cfe5aa49d0debdda6e6e63972691d6bd8c5af2d771dd2a9"), - MustHexID("2cbb718b7dc682da19652e7d9eb4fefaf7b7147d82c1c2b6805edf77b85e29fde9f6da195741467ff2638dc62c8d3e014ea5686693c15ed0080b6de90354c137"), - MustHexID("e84027696d3f12f2de30a9311afea8fbd313c2360daff52bb5fc8c7094d5295758bec3134e4eef24e4cdf377b40da344993284628a7a346eba94f74160998feb"), - MustHexID("f1357a4f04f9d33753a57c0b65ba20a5d8777abbffd04e906014491c9103fb08590e45548d37aa4bd70965e2e81ddba94f31860348df01469eec8c1829200a68"), - MustHexID("4ab0a75941b12892369b4490a1928c8ca52a9ad6d3dffbd1d8c0b907bc200fe74c022d011ec39b64808a39c0ca41f1d3254386c3e7733e7044c44259486461b6"), - MustHexID("d45150a72dc74388773e68e03133a3b5f51447fe91837d566706b3c035ee4b56f160c878c6273394daee7f56cc398985269052f22f75a8057df2fe6172765354"), - }, - 248: { - MustHexID("6aadfce366a189bab08ac84721567483202c86590642ea6d6a14f37ca78d82bdb6509eb7b8b2f6f63c78ae3ae1d8837c89509e41497d719b23ad53dd81574afa"), - MustHexID("a605ecfd6069a4cf4cf7f5840e5bc0ce10d23a3ac59e2aaa70c6afd5637359d2519b4524f56fc2ca180cdbebe54262f720ccaae8c1b28fd553c485675831624d"), - MustHexID("29701451cb9448ca33fc33680b44b840d815be90146eb521641efbffed0859c154e8892d3906eae9934bfacee72cd1d2fa9dd050fd18888eea49da155ab0efd2"), - MustHexID("3ed426322dee7572b08592e1e079f8b6c6b30e10e6243edd144a6a48fdbdb83df73a6e41b1143722cb82604f2203a32758610b5d9544f44a1a7921ba001528c1"), - MustHexID("b2e2a2b7fdd363572a3256e75435fab1da3b16f7891a8bd2015f30995dae665d7eabfd194d87d99d5df628b4bbc7b04e5b492c596422dd8272746c7a1b0b8e4f"), - MustHexID("0c69c9756162c593e85615b814ce57a2a8ca2df6c690b9c4e4602731b61e1531a3bbe3f7114271554427ffabea80ad8f36fa95a49fa77b675ae182c6ccac1728"), - MustHexID("8d28be21d5a97b0876442fa4f5e5387f5bf3faad0b6f13b8607b64d6e448c0991ca28dd7fe2f64eb8eadd7150bff5d5666aa6ed868b84c71311f4ba9a38569dd"), - MustHexID("2c677e1c64b9c9df6359348a7f5f33dc79e22f0177042486d125f8b6ca7f0dc756b1f672aceee5f1746bcff80aaf6f92a8dc0c9fbeb259b3fa0da060de5ab7e8"), - MustHexID("3994880f94a8678f0cd247a43f474a8af375d2a072128da1ad6cae84a244105ff85e94fc7d8496f639468de7ee998908a91c7e33ef7585fff92e984b210941a1"), - MustHexID("b45a9153c08d002a48090d15d61a7c7dad8c2af85d4ff5bd36ce23a9a11e0709bf8d56614c7b193bc028c16cbf7f20dfbcc751328b64a924995d47b41e452422"), - MustHexID("057ab3a9e53c7a84b0f3fc586117a525cdd18e313f52a67bf31798d48078e325abe5cfee3f6c2533230cb37d0549289d692a29dd400e899b8552d4b928f6f907"), - MustHexID("0ddf663d308791eb92e6bd88a2f8cb45e4f4f35bb16708a0e6ff7f1362aa6a73fedd0a1b1557fb3365e38e1b79d6918e2fae2788728b70c9ab6b51a3b94a4338"), - MustHexID("f637e07ff50cc1e3731735841c4798411059f2023abcf3885674f3e8032531b0edca50fd715df6feb489b6177c345374d64f4b07d257a7745de393a107b013a5"), - MustHexID("e24ec7c6eec094f63c7b3239f56d311ec5a3e45bc4e622a1095a65b95eea6fe13e29f3b6b7a2cbfe40906e3989f17ac834c3102dd0cadaaa26e16ee06d782b72"), - MustHexID("b76ea1a6fd6506ef6e3506a4f1f60ed6287fff8114af6141b2ff13e61242331b54082b023cfea5b3083354a4fb3f9eb8be01fb4a518f579e731a5d0707291a6b"), - MustHexID("9b53a37950ca8890ee349b325032d7b672cab7eced178d3060137b24ef6b92a43977922d5bdfb4a3409a2d80128e02f795f9dae6d7d99973ad0e23a2afb8442f"), - }, - 249: { - MustHexID("675ae65567c3c72c50c73bc0fd4f61f202ea5f93346ca57b551de3411ccc614fad61cb9035493af47615311b9d44ee7a161972ee4d77c28fe1ec029d01434e6a"), - MustHexID("8eb81408389da88536ae5800392b16ef5109d7ea132c18e9a82928047ecdb502693f6e4a4cdd18b54296caf561db937185731456c456c98bfe7de0baf0eaa495"), - MustHexID("2adba8b1612a541771cb93a726a38a4b88e97b18eced2593eb7daf82f05a5321ca94a72cc780c306ff21e551a932fc2c6d791e4681907b5ceab7f084c3fa2944"), - MustHexID("b1b4bfbda514d9b8f35b1c28961da5d5216fe50548f4066f69af3b7666a3b2e06eac646735e963e5c8f8138a2fb95af15b13b23ff00c6986eccc0efaa8ee6fb4"), - MustHexID("d2139281b289ad0e4d7b4243c4364f5c51aac8b60f4806135de06b12b5b369c9e43a6eb494eab860d115c15c6fbb8c5a1b0e382972e0e460af395b8385363de7"), - MustHexID("4a693df4b8fc5bdc7cec342c3ed2e228d7c5b4ab7321ddaa6cccbeb45b05a9f1d95766b4002e6d4791c2deacb8a667aadea6a700da28a3eea810a30395701bbc"), - MustHexID("ab41611195ec3c62bb8cd762ee19fb182d194fd141f4a66780efbef4b07ce916246c022b841237a3a6b512a93431157edd221e854ed2a259b72e9c5351f44d0c"), - MustHexID("68e8e26099030d10c3c703ae7045c0a48061fb88058d853b3e67880014c449d4311014da99d617d3150a20f1a3da5e34bf0f14f1c51fe4dd9d58afd222823176"), - MustHexID("3fbcacf546fb129cd70fc48de3b593ba99d3c473798bc309292aca280320e0eacc04442c914cad5c4cf6950345ba79b0d51302df88285d4e83ee3fe41339eee7"), - MustHexID("1d4a623659f7c8f80b6c3939596afdf42e78f892f682c768ad36eb7bfba402dbf97aea3a268f3badd8fe7636be216edf3d67ee1e08789ebbc7be625056bd7109"), - MustHexID("a283c474ab09da02bbc96b16317241d0627646fcc427d1fe790b76a7bf1989ced90f92101a973047ae9940c92720dffbac8eff21df8cae468a50f72f9e159417"), - MustHexID("dbf7e5ad7f87c3dfecae65d87c3039e14ed0bdc56caf00ce81931073e2e16719d746295512ff7937a15c3b03603e7c41a4f9df94fcd37bb200dd8f332767e9cb"), - MustHexID("caaa070a26692f64fc77f30d7b5ae980d419b4393a0f442b1c821ef58c0862898b0d22f74a4f8c5d83069493e3ec0b92f17dc1fe6e4cd437c1ec25039e7ce839"), - MustHexID("874cc8d1213beb65c4e0e1de38ef5d8165235893ac74ab5ea937c885eaab25c8d79dad0456e9fd3e9450626cac7e107b004478fb59842f067857f39a47cee695"), - MustHexID("d94193f236105010972f5df1b7818b55846592a0445b9cdc4eaed811b8c4c0f7c27dc8cc9837a4774656d6b34682d6d329d42b6ebb55da1d475c2474dc3dfdf4"), - MustHexID("edd9af6aded4094e9785637c28fccbd3980cbe28e2eb9a411048a23c2ace4bd6b0b7088a7817997b49a3dd05fc6929ca6c7abbb69438dbdabe65e971d2a794b2"), - }, - 250: { - MustHexID("53a5bd1215d4ab709ae8fdc2ced50bba320bced78bd9c5dc92947fb402250c914891786db0978c898c058493f86fc68b1c5de8a5cb36336150ac7a88655b6c39"), - MustHexID("b7f79e3ab59f79262623c9ccefc8f01d682323aee56ffbe295437487e9d5acaf556a9c92e1f1c6a9601f2b9eb6b027ae1aeaebac71d61b9b78e88676efd3e1a3"), - MustHexID("d374bf7e8d7ffff69cc00bebff38ef5bc1dcb0a8d51c1a3d70e61ac6b2e2d6617109254b0ac224354dfbf79009fe4239e09020c483cc60c071e00b9238684f30"), - MustHexID("1e1eac1c9add703eb252eb991594f8f5a173255d526a855fab24ae57dc277e055bc3c7a7ae0b45d437c4f47a72d97eb7b126f2ba344ba6c0e14b2c6f27d4b1e6"), - MustHexID("ae28953f63d4bc4e706712a59319c111f5ff8f312584f65d7436b4cd3d14b217b958f8486bad666b4481fe879019fb1f767cf15b3e3e2711efc33b56d460448a"), - MustHexID("934bb1edf9c7a318b82306aca67feb3d6b434421fa275d694f0b4927afd8b1d3935b727fd4ff6e3d012e0c82f1824385174e8c6450ade59c2a43281a4b3446b6"), - MustHexID("9eef3f28f70ce19637519a0916555bf76d26de31312ac656cf9d3e379899ea44e4dd7ffcce923b4f3563f8a00489a34bd6936db0cbb4c959d32c49f017e07d05"), - MustHexID("82200872e8f871c48f1fad13daec6478298099b591bb3dbc4ef6890aa28ebee5860d07d70be62f4c0af85085a90ae8179ee8f937cf37915c67ea73e704b03ee7"), - MustHexID("6c75a5834a08476b7fc37ff3dc2011dc3ea3b36524bad7a6d319b18878fad813c0ba76d1f4555cacd3890c865438c21f0e0aed1f80e0a157e642124c69f43a11"), - MustHexID("995b873742206cb02b736e73a88580c2aacb0bd4a3c97a647b647bcab3f5e03c0e0736520a8b3600da09edf4248991fb01091ec7ff3ec7cdc8a1beae011e7aae"), - MustHexID("c773a056594b5cdef2e850d30891ff0e927c3b1b9c35cd8e8d53a1017001e237468e1ece3ae33d612ca3e6abb0a9169aa352e9dcda358e5af2ad982b577447db"), - MustHexID("2b46a5f6923f475c6be99ec6d134437a6d11f6bb4b4ac6bcd94572fa1092639d1c08aeefcb51f0912f0a060f71d4f38ee4da70ecc16010b05dd4a674aab14c3a"), - MustHexID("af6ab501366debbaa0d22e20e9688f32ef6b3b644440580fd78de4fe0e99e2a16eb5636bbae0d1c259df8ddda77b35b9a35cbc36137473e9c68fbc9d203ba842"), - MustHexID("c9f6f2dd1a941926f03f770695bda289859e85fabaf94baaae20b93e5015dc014ba41150176a36a1884adb52f405194693e63b0c464a6891cc9cc1c80d450326"), - MustHexID("5b116f0751526868a909b61a30b0c5282c37df6925cc03ddea556ef0d0602a9595fd6c14d371f8ed7d45d89918a032dcd22be4342a8793d88fdbeb3ca3d75bd7"), - MustHexID("50f3222fb6b82481c7c813b2172e1daea43e2710a443b9c2a57a12bd160dd37e20f87aa968c82ad639af6972185609d47036c0d93b4b7269b74ebd7073221c10"), - }, - 251: { - MustHexID("9b8f702a62d1bee67bedfeb102eca7f37fa1713e310f0d6651cc0c33ea7c5477575289ccd463e5a2574a00a676a1fdce05658ba447bb9d2827f0ba47b947e894"), - MustHexID("b97532eb83054ed054b4abdf413bb30c00e4205545c93521554dbe77faa3cfaa5bd31ef466a107b0b34a71ec97214c0c83919720142cddac93aa7a3e928d4708"), - MustHexID("2f7a5e952bfb67f2f90b8441b5fadc9ee13b1dcde3afeeb3dd64bf937f86663cc5c55d1fa83952b5422763c7df1b7f2794b751c6be316ebc0beb4942e65ab8c1"), - MustHexID("42c7483781727051a0b3660f14faf39e0d33de5e643702ae933837d036508ab856ce7eec8ec89c4929a4901256e5233a3d847d5d4893f91bcf21835a9a880fee"), - MustHexID("873bae27bf1dc854408fba94046a53ab0c965cebe1e4e12290806fc62b88deb1f4a47f9e18f78fc0e7913a0c6e42ac4d0fc3a20cea6bc65f0c8a0ca90b67521e"), - MustHexID("a7e3a370bbd761d413f8d209e85886f68bf73d5c3089b2dc6fa42aab1ecb5162635497eed95dee2417f3c9c74a3e76319625c48ead2e963c7de877cd4551f347"), - MustHexID("528597534776a40df2addaaea15b6ff832ce36b9748a265768368f657e76d58569d9f30dbb91e91cf0ae7efe8f402f17aa0ae15f5c55051ba03ba830287f4c42"), - MustHexID("461d8bd4f13c3c09031fdb84f104ed737a52f630261463ce0bdb5704259bab4b737dda688285b8444dbecaecad7f50f835190b38684ced5e90c54219e5adf1bc"), - MustHexID("6ec50c0be3fd232737090fc0111caaf0bb6b18f72be453428087a11a97fd6b52db0344acbf789a689bd4f5f50f79017ea784f8fd6fe723ad6ae675b9e3b13e21"), - MustHexID("12fc5e2f77a83fdcc727b79d8ae7fe6a516881138d3011847ee136b400fed7cfba1f53fd7a9730253c7aa4f39abeacd04f138417ba7fcb0f36cccc3514e0dab6"), - MustHexID("4fdbe75914ccd0bce02101606a1ccf3657ec963e3b3c20239d5fec87673fe446d649b4f15f1fe1a40e6cfbd446dda2d31d40bb602b1093b8fcd5f139ba0eb46a"), - MustHexID("3753668a0f6281e425ea69b52cb2d17ab97afbe6eb84cf5d25425bc5e53009388857640668fadd7c110721e6047c9697803bd8a6487b43bb343bfa32ebf24039"), - MustHexID("2e81b16346637dec4410fd88e527346145b9c0a849dbf2628049ac7dae016c8f4305649d5659ec77f1e8a0fac0db457b6080547226f06283598e3740ad94849a"), - MustHexID("802c3cc27f91c89213223d758f8d2ecd41135b357b6d698f24d811cdf113033a81c38e0bdff574a5c005b00a8c193dc2531f8c1fa05fa60acf0ab6f2858af09f"), - MustHexID("fcc9a2e1ac3667026ff16192876d1813bb75abdbf39b929a92863012fe8b1d890badea7a0de36274d5c1eb1e8f975785532c50d80fd44b1a4b692f437303393f"), - MustHexID("6d8b3efb461151dd4f6de809b62726f5b89e9b38e9ba1391967f61cde844f7528fecf821b74049207cee5a527096b31f3ad623928cd3ce51d926fa345a6b2951"), - }, - 252: { - MustHexID("f1ae93157cc48c2075dd5868fbf523e79e06caf4b8198f352f6e526680b78ff4227263de92612f7d63472bd09367bb92a636fff16fe46ccf41614f7a72495c2a"), - MustHexID("587f482d111b239c27c0cb89b51dd5d574db8efd8de14a2e6a1400c54d4567e77c65f89c1da52841212080b91604104768350276b6682f2f961cdaf4039581c7"), - MustHexID("e3f88274d35cefdaabdf205afe0e80e936cc982b8e3e47a84ce664c413b29016a4fb4f3a3ebae0a2f79671f8323661ed462bf4390af94c424dc8ace0c301b90f"), - MustHexID("0ddc736077da9a12ba410dc5ea63cbcbe7659dd08596485b2bff3435221f82c10d263efd9af938e128464be64a178b7cd22e19f400d5802f4c9df54bf89f2619"), - MustHexID("784aa34d833c6ce63fcc1279630113c3272e82c4ae8c126c5a52a88ac461b6baeed4244e607b05dc14e5b2f41c70a273c3804dea237f14f7a1e546f6d1309d14"), - MustHexID("f253a2c354ee0e27cfcae786d726753d4ad24be6516b279a936195a487de4a59dbc296accf20463749ff55293263ed8c1b6365eecb248d44e75e9741c0d18205"), - MustHexID("a1910b80357b3ad9b4593e0628922939614dc9056a5fbf477279c8b2c1d0b4b31d89a0c09d0d41f795271d14d3360ef08a3f821e65e7e1f56c07a36afe49c7c5"), - MustHexID("f1168552c2efe541160f0909b0b4a9d6aeedcf595cdf0e9b165c97e3e197471a1ee6320e93389edfba28af6eaf10de98597ad56e7ab1b504ed762451996c3b98"), - MustHexID("b0c8e5d2c8634a7930e1a6fd082e448c6cf9d2d8b7293558b59238815a4df926c286bf297d2049f14e8296a6eb3256af614ec1812c4f2bbe807673b58bf14c8c"), - MustHexID("0fb346076396a38badc342df3679b55bd7f40a609ab103411fe45082c01f12ea016729e95914b2b5540e987ff5c9b133e85862648e7f36abdfd23100d248d234"), - MustHexID("f736e0cc83417feaa280d9483f5d4d72d1b036cd0c6d9cbdeb8ac35ceb2604780de46dddaa32a378474e1d5ccdf79b373331c30c7911ade2ae32f98832e5de1f"), - MustHexID("8b02991457602f42b38b342d3f2259ae4100c354b3843885f7e4e07bd644f64dab94bb7f38a3915f8b7f11d8e3f81c28e07a0078cf79d7397e38a7b7e0c857e2"), - MustHexID("9221d9f04a8a184993d12baa91116692bb685f887671302999d69300ad103eb2d2c75a09d8979404c6dd28f12362f58a1a43619c493d9108fd47588a23ce5824"), - MustHexID("652797801744dada833fff207d67484742eea6835d695925f3e618d71b68ec3c65bdd85b4302b2cdcb835ad3f94fd00d8da07e570b41bc0d2bcf69a8de1b3284"), - MustHexID("d84f06fe64debc4cd0625e36d19b99014b6218375262cc2209202bdbafd7dffcc4e34ce6398e182e02fd8faeed622c3e175545864902dfd3d1ac57647cddf4c6"), - MustHexID("d0ed87b294f38f1d741eb601020eeec30ac16331d05880fe27868f1e454446de367d7457b41c79e202eaf9525b029e4f1d7e17d85a55f83a557c005c68d7328a"), - }, - 253: { - MustHexID("ad4485e386e3cc7c7310366a7c38fb810b8896c0d52e55944bfd320ca294e7912d6c53c0a0cf85e7ce226e92491d60430e86f8f15cda0161ed71893fb4a9e3a1"), - MustHexID("36d0e7e5b7734f98c6183eeeb8ac5130a85e910a925311a19c4941b1290f945d4fc3996b12ef4966960b6fa0fb29b1604f83a0f81bd5fd6398d2e1a22e46af0c"), - MustHexID("7d307d8acb4a561afa23bdf0bd945d35c90245e26345ec3a1f9f7df354222a7cdcb81339c9ed6744526c27a1a0c8d10857e98df942fa433602facac71ac68a31"), - MustHexID("d97bf55f88c83fae36232661af115d66ca600fc4bd6d1fb35ff9bb4dad674c02cf8c8d05f317525b5522250db58bb1ecafb7157392bf5aa61b178c61f098d995"), - MustHexID("7045d678f1f9eb7a4613764d17bd5698796494d0bf977b16f2dbc272b8a0f7858a60805c022fc3d1fe4f31c37e63cdaca0416c0d053ef48a815f8b19121605e0"), - MustHexID("14e1f21418d445748de2a95cd9a8c3b15b506f86a0acabd8af44bb968ce39885b19c8822af61b3dd58a34d1f265baec30e3ae56149dc7d2aa4a538f7319f69c8"), - MustHexID("b9453d78281b66a4eac95a1546017111eaaa5f92a65d0de10b1122940e92b319728a24edf4dec6acc412321b1c95266d39c7b3a5d265c629c3e49a65fb022c09"), - MustHexID("e8a49248419e3824a00d86af422f22f7366e2d4922b304b7169937616a01d9d6fa5abf5cc01061a352dc866f48e1fa2240dbb453d872b1d7be62bdfc1d5e248c"), - MustHexID("bebcff24b52362f30e0589ee573ce2d86f073d58d18e6852a592fa86ceb1a6c9b96d7fb9ec7ed1ed98a51b6743039e780279f6bb49d0a04327ac7a182d9a56f6"), - MustHexID("d0835e5a4291db249b8d2fca9f503049988180c7d247bedaa2cf3a1bad0a76709360a85d4f9a1423b2cbc82bb4d94b47c0cde20afc430224834c49fe312a9ae3"), - MustHexID("6b087fe2a2da5e4f0b0f4777598a4a7fb66bf77dbd5bfc44e8a7eaa432ab585a6e226891f56a7d4f5ed11a7c57b90f1661bba1059590ca4267a35801c2802913"), - MustHexID("d901e5bde52d1a0f4ddf010a686a53974cdae4ebe5c6551b3c37d6b6d635d38d5b0e5f80bc0186a2c7809dbf3a42870dd09643e68d32db896c6da8ba734579e7"), - MustHexID("96419fb80efae4b674402bb969ebaab86c1274f29a83a311e24516d36cdf148fe21754d46c97688cdd7468f24c08b13e4727c29263393638a3b37b99ff60ebca"), - MustHexID("7b9c1889ae916a5d5abcdfb0aaedcc9c6f9eb1c1a4f68d0c2d034fe79ac610ce917c3abc670744150fa891bfcd8ab14fed6983fca964de920aa393fa7b326748"), - MustHexID("7a369b2b8962cc4c65900be046482fbf7c14f98a135bbbae25152c82ad168fb2097b3d1429197cf46d3ce9fdeb64808f908a489cc6019725db040060fdfe5405"), - MustHexID("47bcae48288da5ecc7f5058dfa07cf14d89d06d6e449cb946e237aa6652ea050d9f5a24a65efdc0013ccf232bf88670979eddef249b054f63f38da9d7796dbd8"), - }, - 254: { - MustHexID("099739d7abc8abd38ecc7a816c521a1168a4dbd359fa7212a5123ab583ffa1cf485a5fed219575d6475dbcdd541638b2d3631a6c7fce7474e7fe3cba1d4d5853"), - MustHexID("c2b01603b088a7182d0cf7ef29fb2b04c70acb320fccf78526bf9472e10c74ee70b3fcfa6f4b11d167bd7d3bc4d936b660f2c9bff934793d97cb21750e7c3d31"), - MustHexID("20e4d8f45f2f863e94b45548c1ef22a11f7d36f263e4f8623761e05a64c4572379b000a52211751e2561b0f14f4fc92dd4130410c8ccc71eb4f0e95a700d4ca9"), - MustHexID("27f4a16cc085e72d86e25c98bd2eca173eaaee7565c78ec5a52e9e12b2211f35de81b5b45e9195de2ebfe29106742c59112b951a04eb7ae48822911fc1f9389e"), - MustHexID("55db5ee7d98e7f0b1c3b9d5be6f2bc619a1b86c3cdd513160ad4dcf267037a5fffad527ac15d50aeb32c59c13d1d4c1e567ebbf4de0d25236130c8361f9aac63"), - MustHexID("883df308b0130fc928a8559fe50667a0fff80493bc09685d18213b2db241a3ad11310ed86b0ef662b3ce21fc3d9aa7f3fc24b8d9afe17c7407e9afd3345ae548"), - MustHexID("c7af968cc9bc8200c3ee1a387405f7563be1dce6710a3439f42ea40657d0eae9d2b3c16c42d779605351fcdece4da637b9804e60ca08cfb89aec32c197beffa6"), - MustHexID("3e66f2b788e3ff1d04106b80597915cd7afa06c405a7ae026556b6e583dca8e05cfbab5039bb9a1b5d06083ffe8de5780b1775550e7218f5e98624bf7af9a0a8"), - MustHexID("4fc7f53764de3337fdaec0a711d35d3a923e72fa65025444d12230b3552ed43d9b2d1ad08ccb11f2d50c58809e6dd74dde910e195294fca3b47ae5a3967cc479"), - MustHexID("bafdfdcf6ccaa989436752fa97c77477b6baa7deb374b16c095492c529eb133e8e2f99e1977012b64767b9d34b2cf6d2048ed489bd822b5139b523f6a423167b"), - MustHexID("7f5d78008a4312fe059104ce80202c82b8915c2eb4411c6b812b16f7642e57c00f2c9425121f5cbac4257fe0b3e81ef5dea97ea2dbaa98f6a8b6fd4d1e5980bb"), - MustHexID("598c37fe78f922751a052f463aeb0cb0bc7f52b7c2a4cf2da72ec0931c7c32175d4165d0f8998f7320e87324ac3311c03f9382a5385c55f0407b7a66b2acd864"), - MustHexID("f758c4136e1c148777a7f3275a76e2db0b2b04066fd738554ec398c1c6cc9fb47e14a3b4c87bd47deaeab3ffd2110514c3855685a374794daff87b605b27ee2e"), - MustHexID("0307bb9e4fd865a49dcf1fe4333d1b944547db650ab580af0b33e53c4fef6c789531110fac801bbcbce21fc4d6f61b6d5b24abdf5b22e3030646d579f6dca9c2"), - MustHexID("82504b6eb49bb2c0f91a7006ce9cefdbaf6df38706198502c2e06601091fc9dc91e4f15db3410d45c6af355bc270b0f268d3dff560f956985c7332d4b10bd1ed"), - MustHexID("b39b5b677b45944ceebe76e76d1f051de2f2a0ec7b0d650da52135743e66a9a5dba45f638258f9a7545d9a790c7fe6d3fdf82c25425c7887323e45d27d06c057"), - }, - 255: { - MustHexID("5c4d58d46e055dd1f093f81ee60a675e1f02f54da6206720adee4dccef9b67a31efc5c2a2949c31a04ee31beadc79aba10da31440a1f9ff2a24093c63c36d784"), - MustHexID("ea72161ffdd4b1e124c7b93b0684805f4c4b58d617ed498b37a145c670dbc2e04976f8785583d9c805ffbf343c31d492d79f841652bbbd01b61ed85640b23495"), - MustHexID("51caa1d93352d47a8e531692a3612adac1e8ac68d0a200d086c1c57ae1e1a91aa285ab242e8c52ef9d7afe374c9485b122ae815f1707b875569d0433c1c3ce85"), - MustHexID("c08397d5751b47bd3da044b908be0fb0e510d3149574dff7aeab33749b023bb171b5769990fe17469dbebc100bc150e798aeda426a2dcc766699a225fddd75c6"), - MustHexID("0222c1c194b749736e593f937fad67ee348ac57287a15c7e42877aa38a9b87732a408bca370f812efd0eedbff13e6d5b854bf3ba1dec431a796ed47f32552b09"), - MustHexID("03d859cd46ef02d9bfad5268461a6955426845eef4126de6be0fa4e8d7e0727ba2385b78f1a883a8239e95ebb814f2af8379632c7d5b100688eebc5841209582"), - MustHexID("64d5004b7e043c39ff0bd10cb20094c287721d5251715884c280a612b494b3e9e1c64ba6f67614994c7d969a0d0c0295d107d53fc225d47c44c4b82852d6f960"), - MustHexID("b0a5eefb2dab6f786670f35bf9641eefe6dd87fd3f1362bcab4aaa792903500ab23d88fae68411372e0813b057535a601d46e454323745a948017f6063a47b1f"), - MustHexID("0cc6df0a3433d448b5684d2a3ffa9d1a825388177a18f44ad0008c7bd7702f1ec0fc38b83506f7de689c3b6ecb552599927e29699eed6bb867ff08f80068b287"), - MustHexID("50772f7b8c03a4e153355fbbf79c8a80cf32af656ff0c7873c99911099d04a0dae0674706c357e0145ad017a0ade65e6052cb1b0d574fcd6f67da3eee0ace66b"), - MustHexID("1ae37829c9ef41f8b508b82259ebac76b1ed900d7a45c08b7970f25d2d48ddd1829e2f11423a18749940b6dab8598c6e416cef0efd47e46e51f29a0bc65b37cd"), - MustHexID("ba973cab31c2af091fc1644a93527d62b2394999e2b6ccbf158dd5ab9796a43d408786f1803ef4e29debfeb62fce2b6caa5ab2b24d1549c822a11c40c2856665"), - MustHexID("bc413ad270dd6ea25bddba78f3298b03b8ba6f8608ac03d06007d4116fa78ef5a0cfe8c80155089382fc7a193243ee5500082660cb5d7793f60f2d7d18650964"), - MustHexID("5a6a9ef07634d9eec3baa87c997b529b92652afa11473dfee41ef7037d5c06e0ddb9fe842364462d79dd31cff8a59a1b8d5bc2b810dea1d4cbbd3beb80ecec83"), - MustHexID("f492c6ee2696d5f682f7f537757e52744c2ae560f1090a07024609e903d334e9e174fc01609c5a229ddbcac36c9d21adaf6457dab38a25bfd44f2f0ee4277998"), - MustHexID("459e4db99298cb0467a90acee6888b08bb857450deac11015cced5104853be5adce5b69c740968bc7f931495d671a70cad9f48546d7cd203357fe9af0e8d2164"), - }, - 256: { - MustHexID("a8593af8a4aef7b806b5197612017951bac8845a1917ca9a6a15dd6086d608505144990b245785c4cd2d67a295701c7aac2aa18823fb0033987284b019656268"), - MustHexID("d2eebef914928c3aad77fc1b2a495f52d2294acf5edaa7d8a530b540f094b861a68fe8348a46a7c302f08ab609d85912a4968eacfea0740847b29421b4795d9e"), - MustHexID("b14bfcb31495f32b650b63cf7d08492e3e29071fdc73cf2da0da48d4b191a70ba1a65f42ad8c343206101f00f8a48e8db4b08bf3f622c0853e7323b250835b91"), - MustHexID("7feaee0d818c03eb30e4e0bf03ade0f3c21ca38e938a761aa1781cf70bda8cc5cd631a6cc53dd44f1d4a6d3e2dae6513c6c66ee50cb2f0e9ad6f7e319b309fd9"), - MustHexID("4ca3b657b139311db8d583c25dd5963005e46689e1317620496cc64129c7f3e52870820e0ec7941d28809311df6db8a2867bbd4f235b4248af24d7a9c22d1232"), - MustHexID("1181defb1d16851d42dd951d84424d6bd1479137f587fa184d5a8152be6b6b16ed08bcdb2c2ed8539bcde98c80c432875f9f724737c316a2bd385a39d3cab1d8"), - MustHexID("d9dd818769fa0c3ec9f553c759b92476f082817252a04a47dc1777740b1731d280058c66f982812f173a294acf4944a85ba08346e2de153ba3ba41ce8a62cb64"), - MustHexID("bd7c4f8a9e770aa915c771b15e107ca123d838762da0d3ffc53aa6b53e9cd076cffc534ec4d2e4c334c683f1f5ea72e0e123f6c261915ed5b58ac1b59f003d88"), - MustHexID("3dd5739c73649d510456a70e9d6b46a855864a4a3f744e088fd8c8da11b18e4c9b5f2d7da50b1c147b2bae5ca9609ae01f7a3cdea9dce34f80a91d29cd82f918"), - MustHexID("f0d7df1efc439b4bcc0b762118c1cfa99b2a6143a9f4b10e3c9465125f4c9fca4ab88a2504169bbcad65492cf2f50da9dd5d077c39574a944f94d8246529066b"), - MustHexID("dd598b9ba441448e5fb1a6ec6c5f5aa9605bad6e223297c729b1705d11d05f6bfd3d41988b694681ae69bb03b9a08bff4beab5596503d12a39bffb5cd6e94c7c"), - MustHexID("3fce284ac97e567aebae681b15b7a2b6df9d873945536335883e4bbc26460c064370537f323fd1ada828ea43154992d14ac0cec0940a2bd2a3f42ec156d60c83"), - MustHexID("7c8dfa8c1311cb14fb29a8ac11bca23ecc115e56d9fcf7b7ac1db9066aa4eb39f8b1dabf46e192a65be95ebfb4e839b5ab4533fef414921825e996b210dd53bd"), - MustHexID("cafa6934f82120456620573d7f801390ed5e16ed619613a37e409e44ab355ef755e83565a913b48a9466db786f8d4fbd590bfec474c2524d4a2608d4eafd6abd"), - MustHexID("9d16600d0dd310d77045769fed2cb427f32db88cd57d86e49390c2ba8a9698cfa856f775be2013237226e7bf47b248871cf865d23015937d1edeb20db5e3e760"), - MustHexID("17be6b6ba54199b1d80eff866d348ea11d8a4b341d63ad9a6681d3ef8a43853ac564d153eb2a8737f0afc9ab320f6f95c55aa11aaa13bbb1ff422fd16bdf8188"), - }, - }, -} - -type preminedTestnet struct { - target NodeID - targetSha common.Hash // sha3(target) - dists [hashBits + 1][]NodeID - net *Network -} - -func (tn *preminedTestnet) sendFindnodeHash(to *Node, target common.Hash) { - // current log distance is encoded in port number - // fmt.Println("findnode query at dist", toaddr.Port) - if to.UDP <= lowPort { - panic("query to node at or below distance 0") - } - next := to.UDP - 1 - var result []rpcNode - for i, id := range tn.dists[to.UDP-lowPort] { - result = append(result, nodeToRPC(NewNode(id, net.ParseIP("10.0.2.99"), next, uint16(i)+1+lowPort))) - } - injectResponse(tn.net, to, neighborsPacket, &neighbors{Nodes: result}) -} - -func (tn *preminedTestnet) sendPing(to *Node, addr *net.UDPAddr, topics []Topic) []byte { - injectResponse(tn.net, to, pongPacket, &pong{ReplyTok: []byte{1}}) - return []byte{1} -} - -func (tn *preminedTestnet) send(to *Node, ptype nodeEvent, data interface{}) (hash []byte) { - switch ptype { - case pingPacket: - injectResponse(tn.net, to, pongPacket, &pong{ReplyTok: []byte{1}}) - case pongPacket: - // ignored - case findnodeHashPacket: - // current log distance is encoded in port number - // fmt.Println("findnode query at dist", toaddr.Port-lowPort) - if to.UDP <= lowPort { - panic("query to node at or below distance 0") - } - next := to.UDP - 1 - var result []rpcNode - for i, id := range tn.dists[to.UDP-lowPort] { - result = append(result, nodeToRPC(NewNode(id, net.ParseIP("10.0.2.99"), next, uint16(i)+1+lowPort))) - } - injectResponse(tn.net, to, neighborsPacket, &neighbors{Nodes: result}) - default: - panic("send(" + ptype.String() + ")") - } - return []byte{2} -} - -func (tn *preminedTestnet) sendNeighbours(to *Node, nodes []*Node) { - panic("sendNeighbours called") -} - -func (tn *preminedTestnet) sendTopicNodes(to *Node, queryHash common.Hash, nodes []*Node) { - panic("sendTopicNodes called") -} - -func (tn *preminedTestnet) sendTopicRegister(to *Node, topics []Topic, idx int, pong []byte) { - panic("sendTopicRegister called") -} - -func (*preminedTestnet) Close() {} - -func (*preminedTestnet) localAddr() *net.UDPAddr { - return &net.UDPAddr{IP: net.ParseIP("10.0.1.1"), Port: 40000} -} - -func injectResponse(net *Network, from *Node, ev nodeEvent, packet interface{}) { - go net.reqReadPacket(ingressPacket{remoteID: from.ID, remoteAddr: from.addr(), ev: ev, data: packet}) -} diff --git a/p2p/discv5/node.go b/p2p/discv5/node.go deleted file mode 100644 index 44d3025b70..0000000000 --- a/p2p/discv5/node.go +++ /dev/null @@ -1,413 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "encoding/hex" - "errors" - "fmt" - "math/big" - "math/rand" - "net" - "net/url" - "regexp" - "strconv" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" -) - -// Node represents a host on the network. -// The public fields of Node may not be modified. -type Node struct { - IP net.IP // len 4 for IPv4 or 16 for IPv6 - UDP, TCP uint16 // port numbers - ID NodeID // the node's public key - - // Network-related fields are contained in nodeNetGuts. - // These fields are not supposed to be used off the - // Network.loop goroutine. - nodeNetGuts -} - -// NewNode creates a new node. It is mostly meant to be used for -// testing purposes. -func NewNode(id NodeID, ip net.IP, udpPort, tcpPort uint16) *Node { - if ipv4 := ip.To4(); ipv4 != nil { - ip = ipv4 - } - return &Node{ - IP: ip, - UDP: udpPort, - TCP: tcpPort, - ID: id, - nodeNetGuts: nodeNetGuts{sha: crypto.Keccak256Hash(id[:])}, - } -} - -func (n *Node) addr() *net.UDPAddr { - return &net.UDPAddr{IP: n.IP, Port: int(n.UDP)} -} - -// Incomplete returns true for nodes with no IP address. -func (n *Node) Incomplete() bool { - return n.IP == nil -} - -// checks whether n is a valid complete node. -func (n *Node) validateComplete() error { - if n.Incomplete() { - return errors.New("incomplete node") - } - if n.UDP == 0 { - return errors.New("missing UDP port") - } - if n.TCP == 0 { - return errors.New("missing TCP port") - } - if n.IP.IsMulticast() || n.IP.IsUnspecified() { - return errors.New("invalid IP (multicast/unspecified)") - } - _, err := n.ID.Pubkey() // validate the key (on curve, etc.) - return err -} - -// The string representation of a Node is a URL. -// Please see ParseNode for a description of the format. -func (n *Node) String() string { - u := url.URL{Scheme: "enode"} - if n.Incomplete() { - u.Host = fmt.Sprintf("%x", n.ID[:]) - } else { - addr := net.TCPAddr{IP: n.IP, Port: int(n.TCP)} - u.User = url.User(fmt.Sprintf("%x", n.ID[:])) - u.Host = addr.String() - if n.UDP != n.TCP { - u.RawQuery = "discport=" + strconv.Itoa(int(n.UDP)) - } - } - return u.String() -} - -var incompleteNodeURL = regexp.MustCompile("(?i)^(?:enode://)?([0-9a-f]+)$") - -// ParseNode parses a node designator. -// -// There are two basic forms of node designators -// - incomplete nodes, which only have the public key (node ID) -// - complete nodes, which contain the public key and IP/Port information -// -// For incomplete nodes, the designator must look like one of these -// -// enode:// -// -// -// For complete nodes, the node ID is encoded in the username portion -// of the URL, separated from the host by an @ sign. The hostname can -// only be given as an IP address, DNS domain names are not allowed. -// The port in the host name section is the TCP listening port. If the -// TCP and UDP (discovery) ports differ, the UDP port is specified as -// query parameter "discport". -// -// In the following example, the node URL describes -// a node with IP address 10.3.58.6, TCP listening port 30303 -// and UDP discovery port 30301. -// -// enode://@10.3.58.6:30303?discport=30301 -func ParseNode(rawurl string) (*Node, error) { - if m := incompleteNodeURL.FindStringSubmatch(rawurl); m != nil { - id, err := HexID(m[1]) - if err != nil { - return nil, fmt.Errorf("invalid node ID (%v)", err) - } - return NewNode(id, nil, 0, 0), nil - } - return parseComplete(rawurl) -} - -func parseComplete(rawurl string) (*Node, error) { - var ( - id NodeID - ip net.IP - tcpPort, udpPort uint64 - ) - u, err := url.Parse(rawurl) - if err != nil { - return nil, err - } - if u.Scheme != "enode" { - return nil, errors.New("invalid URL scheme, want \"enode\"") - } - // Parse the Node ID from the user portion. - if u.User == nil { - return nil, errors.New("does not contain node ID") - } - if id, err = HexID(u.User.String()); err != nil { - return nil, fmt.Errorf("invalid node ID (%v)", err) - } - // Parse the IP address. - host, port, err := net.SplitHostPort(u.Host) - if err != nil { - return nil, fmt.Errorf("invalid host: %v", err) - } - if ip = net.ParseIP(host); ip == nil { - return nil, errors.New("invalid IP address") - } - // Ensure the IP is 4 bytes long for IPv4 addresses. - if ipv4 := ip.To4(); ipv4 != nil { - ip = ipv4 - } - // Parse the port numbers. - if tcpPort, err = strconv.ParseUint(port, 10, 16); err != nil { - return nil, errors.New("invalid port") - } - udpPort = tcpPort - qv := u.Query() - if qv.Get("discport") != "" { - udpPort, err = strconv.ParseUint(qv.Get("discport"), 10, 16) - if err != nil { - return nil, errors.New("invalid discport in query") - } - } - return NewNode(id, ip, uint16(udpPort), uint16(tcpPort)), nil -} - -// MustParseNode parses a node URL. It panics if the URL is not valid. -func MustParseNode(rawurl string) *Node { - n, err := ParseNode(rawurl) - if err != nil { - panic("invalid node URL: " + err.Error()) - } - return n -} - -// MarshalText implements encoding.TextMarshaler. -func (n *Node) MarshalText() ([]byte, error) { - return []byte(n.String()), nil -} - -// UnmarshalText implements encoding.TextUnmarshaler. -func (n *Node) UnmarshalText(text []byte) error { - dec, err := ParseNode(string(text)) - if err == nil { - *n = *dec - } - return err -} - -// type nodeQueue []*Node -// -// // pushNew adds n to the end if it is not present. -// func (nl *nodeList) appendNew(n *Node) { -// for _, entry := range n { -// if entry == n { -// return -// } -// } -// *nq = append(*nq, n) -// } -// -// // popRandom removes a random node. Nodes closer to -// // to the head of the beginning of the have a slightly higher probability. -// func (nl *nodeList) popRandom() *Node { -// ix := rand.Intn(len(*nq)) -// //TODO: probability as mentioned above. -// nl.removeIndex(ix) -// } -// -// func (nl *nodeList) removeIndex(i int) *Node { -// slice = *nl -// if len(*slice) <= i { -// return nil -// } -// *nl = append(slice[:i], slice[i+1:]...) -// } - -const nodeIDBits = 512 - -// NodeID is a unique identifier for each node. -// The node identifier is a marshaled elliptic curve public key. -type NodeID [nodeIDBits / 8]byte - -// NodeID prints as a long hexadecimal number. -func (n NodeID) String() string { - return fmt.Sprintf("%x", n[:]) -} - -// The Go syntax representation of a NodeID is a call to HexID. -func (n NodeID) GoString() string { - return fmt.Sprintf("discover.HexID(\"%x\")", n[:]) -} - -// TerminalString returns a shortened hex string for terminal logging. -func (n NodeID) TerminalString() string { - return hex.EncodeToString(n[:8]) -} - -// HexID converts a hex string to a NodeID. -// The string may be prefixed with 0x. -func HexID(in string) (NodeID, error) { - var id NodeID - b, err := hex.DecodeString(strings.TrimPrefix(in, "0x")) - if err != nil { - return id, err - } else if len(b) != len(id) { - return id, fmt.Errorf("wrong length, want %d hex chars", len(id)*2) - } - copy(id[:], b) - return id, nil -} - -// MustHexID converts a hex string to a NodeID. -// It panics if the string is not a valid NodeID. -func MustHexID(in string) NodeID { - id, err := HexID(in) - if err != nil { - panic(err) - } - return id -} - -// PubkeyID returns a marshaled representation of the given public key. -func PubkeyID(pub *ecdsa.PublicKey) NodeID { - var id NodeID - pbytes := elliptic.Marshal(pub.Curve, pub.X, pub.Y) - if len(pbytes)-1 != len(id) { - panic(fmt.Errorf("need %d bit pubkey, got %d bits", (len(id)+1)*8, len(pbytes))) - } - copy(id[:], pbytes[1:]) - return id -} - -// Pubkey returns the public key represented by the node ID. -// It returns an error if the ID is not a point on the curve. -func (n NodeID) Pubkey() (*ecdsa.PublicKey, error) { - p := &ecdsa.PublicKey{Curve: crypto.S256(), X: new(big.Int), Y: new(big.Int)} - half := len(n) / 2 - p.X.SetBytes(n[:half]) - p.Y.SetBytes(n[half:]) - if !p.Curve.IsOnCurve(p.X, p.Y) { - return nil, errors.New("id is invalid secp256k1 curve point") - } - return p, nil -} - -// recoverNodeID computes the public key used to sign the -// given hash from the signature. -func recoverNodeID(hash, sig []byte) (id NodeID, err error) { - pubkey, err := crypto.Ecrecover(hash, sig) - if err != nil { - return id, err - } - if len(pubkey)-1 != len(id) { - return id, fmt.Errorf("recovered pubkey has %d bits, want %d bits", len(pubkey)*8, (len(id)+1)*8) - } - for i := range id { - id[i] = pubkey[i+1] - } - return id, nil -} - -// distcmp compares the distances a->target and b->target. -// Returns -1 if a is closer to target, 1 if b is closer to target -// and 0 if they are equal. -func distcmp(target, a, b common.Hash) int { - for i := range target { - da := a[i] ^ target[i] - db := b[i] ^ target[i] - if da > db { - return 1 - } else if da < db { - return -1 - } - } - return 0 -} - -// table of leading zero counts for bytes [0..255] -var lzcount = [256]int{ - 8, 7, 6, 6, 5, 5, 5, 5, - 4, 4, 4, 4, 4, 4, 4, 4, - 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, - 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, -} - -// logdist returns the logarithmic distance between a and b, log2(a ^ b). -func logdist(a, b common.Hash) int { - lz := 0 - for i := range a { - x := a[i] ^ b[i] - if x == 0 { - lz += 8 - } else { - lz += lzcount[x] - break - } - } - return len(a)*8 - lz -} - -// hashAtDistance returns a random hash such that logdist(a, b) == n -func hashAtDistance(a common.Hash, n int) (b common.Hash) { - if n == 0 { - return a - } - // flip bit at position n, fill the rest with random bits - b = a - pos := len(a) - n/8 - 1 - bit := byte(0x01) << (byte(n%8) - 1) - if bit == 0 { - pos++ - bit = 0x80 - } - b[pos] = a[pos]&^bit | ^a[pos]&bit // TODO: randomize end bits - for i := pos + 1; i < len(a); i++ { - b[i] = byte(rand.Intn(255)) - } - return b -} diff --git a/p2p/discv5/node_test.go b/p2p/discv5/node_test.go deleted file mode 100644 index 4e0fdbe3db..0000000000 --- a/p2p/discv5/node_test.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "fmt" - "math/big" - "math/rand" - "net" - "reflect" - "strings" - "testing" - "testing/quick" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" -) - -func ExampleNewNode() { - id := MustHexID("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439") - - // Complete nodes contain UDP and TCP endpoints: - n1 := NewNode(id, net.ParseIP("2001:db8:3c4d:15::abcd:ef12"), 52150, 30303) - fmt.Println("n1:", n1) - fmt.Println("n1.Incomplete() ->", n1.Incomplete()) - - // An incomplete node can be created by passing zero values - // for all parameters except id. - n2 := NewNode(id, nil, 0, 0) - fmt.Println("n2:", n2) - fmt.Println("n2.Incomplete() ->", n2.Incomplete()) - - // Output: - // n1: enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@[2001:db8:3c4d:15::abcd:ef12]:30303?discport=52150 - // n1.Incomplete() -> false - // n2: enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439 - // n2.Incomplete() -> true -} - -var parseNodeTests = []struct { - rawurl string - wantError string - wantResult *Node -}{ - { - rawurl: "http://foobar", - wantError: `invalid URL scheme, want "enode"`, - }, - { - rawurl: "enode://01010101@123.124.125.126:3", - wantError: `invalid node ID (wrong length, want 128 hex chars)`, - }, - // Complete nodes with IP address. - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@hostname:3", - wantError: `invalid IP address`, - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@127.0.0.1:foo", - wantError: `invalid port`, - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@127.0.0.1:3?discport=foo", - wantError: `invalid discport in query`, - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@127.0.0.1:52150", - wantResult: NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{0x7f, 0x0, 0x0, 0x1}, - 52150, - 52150, - ), - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@[::]:52150", - wantResult: NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.ParseIP("::"), - 52150, - 52150, - ), - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@[2001:db8:3c4d:15::abcd:ef12]:52150", - wantResult: NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.ParseIP("2001:db8:3c4d:15::abcd:ef12"), - 52150, - 52150, - ), - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@127.0.0.1:52150?discport=22334", - wantResult: NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - net.IP{0x7f, 0x0, 0x0, 0x1}, - 22334, - 52150, - ), - }, - // Incomplete nodes with no address. - { - rawurl: "1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439", - wantResult: NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - nil, 0, 0, - ), - }, - { - rawurl: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439", - wantResult: NewNode( - MustHexID("0x1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), - nil, 0, 0, - ), - }, - // Invalid URLs - { - rawurl: "01010101", - wantError: `invalid node ID (wrong length, want 128 hex chars)`, - }, - { - rawurl: "enode://01010101", - wantError: `invalid node ID (wrong length, want 128 hex chars)`, - }, - { - // This test checks that errors from url.Parse are handled. - rawurl: "://foo", - wantError: `missing protocol scheme`, - }, -} - -func TestParseNode(t *testing.T) { - for _, test := range parseNodeTests { - n, err := ParseNode(test.rawurl) - if test.wantError != "" { - if err == nil { - t.Errorf("test %q:\n got nil error, expected %#q", test.rawurl, test.wantError) - continue - } else if !strings.Contains(err.Error(), test.wantError) { - t.Errorf("test %q:\n got error %#q, expected %#q", test.rawurl, err.Error(), test.wantError) - continue - } - } else { - if err != nil { - t.Errorf("test %q:\n unexpected error: %v", test.rawurl, err) - continue - } - if !reflect.DeepEqual(n, test.wantResult) { - t.Errorf("test %q:\n result mismatch:\ngot: %#v, want: %#v", test.rawurl, n, test.wantResult) - } - } - } -} - -func TestNodeString(t *testing.T) { - for i, test := range parseNodeTests { - if test.wantError == "" && strings.HasPrefix(test.rawurl, "enode://") { - str := test.wantResult.String() - if str != test.rawurl { - t.Errorf("test %d: Node.String() mismatch:\ngot: %s\nwant: %s", i, str, test.rawurl) - } - } - } -} - -func TestHexID(t *testing.T) { - ref := NodeID{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 106, 217, 182, 31, 165, 174, 1, 67, 7, 235, 220, 150, 66, 83, 173, 205, 159, 44, 10, 57, 42, 161, 26, 188} - id1 := MustHexID("0x000000000000000000000000000000000000000000000000000000000000000000000000000000806ad9b61fa5ae014307ebdc964253adcd9f2c0a392aa11abc") - id2 := MustHexID("000000000000000000000000000000000000000000000000000000000000000000000000000000806ad9b61fa5ae014307ebdc964253adcd9f2c0a392aa11abc") - - if id1 != ref { - t.Errorf("wrong id1\ngot %v\nwant %v", id1[:], ref[:]) - } - if id2 != ref { - t.Errorf("wrong id2\ngot %v\nwant %v", id2[:], ref[:]) - } -} - -func TestNodeID_recover(t *testing.T) { - prv := newkey() - hash := make([]byte, 32) - sig, err := crypto.Sign(hash, prv) - if err != nil { - t.Fatalf("signing error: %v", err) - } - - pub := PubkeyID(&prv.PublicKey) - recpub, err := recoverNodeID(hash, sig) - if err != nil { - t.Fatalf("recovery error: %v", err) - } - if pub != recpub { - t.Errorf("recovered wrong pubkey:\ngot: %v\nwant: %v", recpub, pub) - } - - ecdsa, err := pub.Pubkey() - if err != nil { - t.Errorf("Pubkey error: %v", err) - } - if !reflect.DeepEqual(ecdsa, &prv.PublicKey) { - t.Errorf("Pubkey mismatch:\n got: %#v\n want: %#v", ecdsa, &prv.PublicKey) - } -} - -func TestNodeID_pubkeyBad(t *testing.T) { - ecdsa, err := NodeID{}.Pubkey() - if err == nil { - t.Error("expected error for zero ID") - } - if ecdsa != nil { - t.Error("expected nil result") - } -} - -func TestNodeID_distcmp(t *testing.T) { - distcmpBig := func(target, a, b common.Hash) int { - tbig := new(big.Int).SetBytes(target[:]) - abig := new(big.Int).SetBytes(a[:]) - bbig := new(big.Int).SetBytes(b[:]) - return new(big.Int).Xor(tbig, abig).Cmp(new(big.Int).Xor(tbig, bbig)) - } - if err := quick.CheckEqual(distcmp, distcmpBig, quickcfg()); err != nil { - t.Error(err) - } -} - -// the random tests is likely to miss the case where they're equal. -func TestNodeID_distcmpEqual(t *testing.T) { - base := common.Hash{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} - x := common.Hash{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} - if distcmp(base, x, x) != 0 { - t.Errorf("distcmp(base, x, x) != 0") - } -} - -func TestNodeID_logdist(t *testing.T) { - logdistBig := func(a, b common.Hash) int { - abig, bbig := new(big.Int).SetBytes(a[:]), new(big.Int).SetBytes(b[:]) - return new(big.Int).Xor(abig, bbig).BitLen() - } - if err := quick.CheckEqual(logdist, logdistBig, quickcfg()); err != nil { - t.Error(err) - } -} - -// the random tests is likely to miss the case where they're equal. -func TestNodeID_logdistEqual(t *testing.T) { - x := common.Hash{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} - if logdist(x, x) != 0 { - t.Errorf("logdist(x, x) != 0") - } -} - -func TestNodeID_hashAtDistance(t *testing.T) { - // we don't use quick.Check here because its output isn't - // very helpful when the test fails. - cfg := quickcfg() - for i := 0; i < cfg.MaxCount; i++ { - a := gen(common.Hash{}, cfg.Rand).(common.Hash) - dist := cfg.Rand.Intn(len(common.Hash{}) * 8) - result := hashAtDistance(a, dist) - actualdist := logdist(result, a) - - if dist != actualdist { - t.Log("a: ", a) - t.Log("result:", result) - t.Fatalf("#%d: distance of result is %d, want %d", i, actualdist, dist) - } - } -} - -func quickcfg() *quick.Config { - return &quick.Config{ - MaxCount: 5000, - Rand: rand.New(rand.NewSource(time.Now().Unix())), - } -} - -// TODO: The Generate method can be dropped when we require Go >= 1.5 -// because testing/quick learned to generate arrays in 1.5. - -func (NodeID) Generate(rand *rand.Rand, size int) reflect.Value { - var id NodeID - m := rand.Intn(len(id)) - for i := len(id) - 1; i > m; i-- { - id[i] = byte(rand.Uint32()) - } - return reflect.ValueOf(id) -} diff --git a/p2p/discv5/nodeevent_string.go b/p2p/discv5/nodeevent_string.go deleted file mode 100644 index 38c1993bac..0000000000 --- a/p2p/discv5/nodeevent_string.go +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by "stringer -type=nodeEvent"; DO NOT EDIT. - -package discv5 - -import "strconv" - -const _nodeEvent_name = "pongTimeoutpingTimeoutneighboursTimeout" - -var _nodeEvent_index = [...]uint8{0, 11, 22, 39} - -func (i nodeEvent) String() string { - i -= 264 - if i >= nodeEvent(len(_nodeEvent_index)-1) { - return "nodeEvent(" + strconv.FormatInt(int64(i+264), 10) + ")" - } - return _nodeEvent_name[_nodeEvent_index[i]:_nodeEvent_index[i+1]] -} diff --git a/p2p/discv5/sim_run_test.go b/p2p/discv5/sim_run_test.go deleted file mode 100644 index bded0cc023..0000000000 --- a/p2p/discv5/sim_run_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "bufio" - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - "os" - "os/exec" - "runtime" - "strings" - "testing" -) - -func getnacl() (string, error) { - switch runtime.GOARCH { - case "amd64": - _, err := exec.LookPath("sel_ldr_x86_64") - return "amd64p32", err - case "i386": - _, err := exec.LookPath("sel_ldr_i386") - return "i386", err - default: - return "", errors.New("nacl is not supported on " + runtime.GOARCH) - } -} - -// runWithPlaygroundTime executes the caller -// in the NaCl sandbox with faketime enabled. -// -// This function must be called from a Test* function -// and the caller must skip the actual test when isHost is true. -func runWithPlaygroundTime(t *testing.T) (isHost bool) { - if runtime.GOOS == "nacl" { - return false - } - - // Get the caller. - callerPC, _, _, ok := runtime.Caller(1) - if !ok { - panic("can't get caller") - } - callerFunc := runtime.FuncForPC(callerPC) - if callerFunc == nil { - panic("can't get caller") - } - callerName := callerFunc.Name()[strings.LastIndexByte(callerFunc.Name(), '.')+1:] - if !strings.HasPrefix(callerName, "Test") { - panic("must be called from witin a Test* function") - } - testPattern := "^" + callerName + "$" - - // Unfortunately runtime.faketime (playground time mode) only works on NaCl. The NaCl - // SDK must be installed and linked into PATH for this to work. - arch, err := getnacl() - if err != nil { - t.Skip(err) - } - - // Compile and run the calling test using NaCl. - // The extra tag ensures that the TestMain function in sim_main_test.go is used. - cmd := exec.Command("go", "test", "-v", "-tags", "faketime_simulation", "-timeout", "100h", "-run", testPattern, ".") - cmd.Env = append([]string{"GOOS=nacl", "GOARCH=" + arch}, os.Environ()...) - stdout, _ := cmd.StdoutPipe() - stderr, _ := cmd.StderrPipe() - go skipPlaygroundOutputHeaders(os.Stdout, stdout) - go skipPlaygroundOutputHeaders(os.Stderr, stderr) - if err := cmd.Run(); err != nil { - t.Error(err) - } - - // Ensure that the test function doesn't run in the (non-NaCl) host process. - return true -} - -func skipPlaygroundOutputHeaders(out io.Writer, in io.Reader) { - // Additional output can be printed without the headers - // before the NaCl binary starts running (e.g. compiler error messages). - bufin := bufio.NewReader(in) - output, err := bufin.ReadBytes(0) - output = bytes.TrimSuffix(output, []byte{0}) - if len(output) > 0 { - out.Write(output) - } - if err != nil { - return - } - bufin.UnreadByte() - - // Playback header: 0 0 P B <8-byte time> <4-byte data length> - head := make([]byte, 4+8+4) - for { - if _, err := io.ReadFull(bufin, head); err != nil { - if err != io.EOF { - fmt.Fprintln(out, "read error:", err) - } - return - } - if !bytes.HasPrefix(head, []byte{0x00, 0x00, 'P', 'B'}) { - fmt.Fprintf(out, "expected playback header, got %q\n", head) - io.Copy(out, bufin) - return - } - // Copy data until next header. - size := binary.BigEndian.Uint32(head[12:]) - io.CopyN(out, bufin, int64(size)) - } -} diff --git a/p2p/discv5/sim_test.go b/p2p/discv5/sim_test.go deleted file mode 100644 index 3d1e610d3a..0000000000 --- a/p2p/discv5/sim_test.go +++ /dev/null @@ -1,432 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "crypto/ecdsa" - "encoding/binary" - "fmt" - "math/rand" - "net" - "strconv" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" -) - -// In this test, nodes try to randomly resolve each other. -func TestSimRandomResolve(t *testing.T) { - t.Skip("boring") - if runWithPlaygroundTime(t) { - return - } - - sim := newSimulation() - bootnode := sim.launchNode(false) - - // A new node joins every 10s. - launcher := time.NewTicker(10 * time.Second) - defer launcher.Stop() - go func() { - for range launcher.C { - net := sim.launchNode(false) - go randomResolves(t, sim, net) - if err := net.SetFallbackNodes([]*Node{bootnode.Self()}); err != nil { - panic(err) - } - t.Logf("launched @ %v: %x\n", time.Now(), net.Self().ID[:16]) - } - }() - - time.Sleep(3 * time.Hour) - sim.shutdown() - sim.printStats() -} - -func TestSimTopics(t *testing.T) { - t.Skip("NaCl test") - if runWithPlaygroundTime(t) { - return - } - sim := newSimulation() - bootnode := sim.launchNode(false) - - go func() { - nets := make([]*Network, 1024) - for i := range nets { - net := sim.launchNode(false) - nets[i] = net - if err := net.SetFallbackNodes([]*Node{bootnode.Self()}); err != nil { - panic(err) - } - time.Sleep(time.Second * 5) - } - - for i, net := range nets { - if i < 256 { - stop := make(chan struct{}) - go net.RegisterTopic(testTopic, stop) - go func() { - //time.Sleep(time.Second * 36000) - time.Sleep(time.Second * 40000) - close(stop) - }() - time.Sleep(time.Millisecond * 100) - } - // time.Sleep(time.Second * 10) - //time.Sleep(time.Second) - /*if i%500 == 499 { - time.Sleep(time.Second * 9501) - } else { - time.Sleep(time.Second) - }*/ - } - }() - - // A new node joins every 10s. - /* launcher := time.NewTicker(5 * time.Second) - cnt := 0 - var printNet *Network - go func() { - for range launcher.C { - cnt++ - if cnt <= 1000 { - log := false //(cnt == 500) - net := sim.launchNode(log) - if log { - printNet = net - } - if cnt > 500 { - go net.RegisterTopic(testTopic, nil) - } - if err := net.SetFallbackNodes([]*Node{bootnode.Self()}); err != nil { - panic(err) - } - } - //fmt.Printf("launched @ %v: %x\n", time.Now(), net.Self().ID[:16]) - } - }() - */ - time.Sleep(55000 * time.Second) - //launcher.Stop() - sim.shutdown() - //sim.printStats() - //printNet.log.printLogs() -} - -/*func testHierarchicalTopics(i int) []Topic { - digits := strconv.FormatInt(int64(256+i/4), 4) - res := make([]Topic, 5) - for i, _ := range res { - res[i] = Topic("foo" + digits[1:i+1]) - } - return res -}*/ - -func testHierarchicalTopics(i int) []Topic { - digits := strconv.FormatInt(int64(128+i/8), 2) - res := make([]Topic, 8) - for i := range res { - res[i] = Topic("foo" + digits[1:i+1]) - } - return res -} - -func TestSimTopicHierarchy(t *testing.T) { - t.Skip("NaCl test") - if runWithPlaygroundTime(t) { - return - } - sim := newSimulation() - bootnode := sim.launchNode(false) - - go func() { - nets := make([]*Network, 1024) - for i := range nets { - net := sim.launchNode(false) - nets[i] = net - if err := net.SetFallbackNodes([]*Node{bootnode.Self()}); err != nil { - panic(err) - } - time.Sleep(time.Second * 5) - } - - stop := make(chan struct{}) - for i, net := range nets { - //if i < 256 { - for _, topic := range testHierarchicalTopics(i)[:5] { - //fmt.Println("reg", topic) - go net.RegisterTopic(topic, stop) - } - time.Sleep(time.Millisecond * 100) - //} - } - time.Sleep(time.Second * 90000) - close(stop) - }() - - time.Sleep(100000 * time.Second) - sim.shutdown() -} - -func randomResolves(t *testing.T, s *simulation, net *Network) { - randtime := func() time.Duration { - return time.Duration(rand.Intn(50)+20) * time.Second - } - lookup := func(target NodeID) bool { - result := net.Resolve(target) - return result != nil && result.ID == target - } - - timer := time.NewTimer(randtime()) - defer timer.Stop() - for { - select { - case <-timer.C: - target := s.randomNode().Self().ID - if !lookup(target) { - t.Errorf("node %x: target %x not found", net.Self().ID[:8], target[:8]) - } - timer.Reset(randtime()) - case <-net.closed: - return - } - } -} - -type simulation struct { - mu sync.RWMutex - nodes map[NodeID]*Network - nodectr uint32 -} - -func newSimulation() *simulation { - return &simulation{nodes: make(map[NodeID]*Network)} -} - -func (s *simulation) shutdown() { - s.mu.RLock() - alive := make([]*Network, 0, len(s.nodes)) - for _, n := range s.nodes { - alive = append(alive, n) - } - defer s.mu.RUnlock() - - for _, n := range alive { - n.Close() - } -} - -func (s *simulation) printStats() { - s.mu.Lock() - defer s.mu.Unlock() - fmt.Println("node counter:", s.nodectr) - fmt.Println("alive nodes:", len(s.nodes)) - - // for _, n := range s.nodes { - // fmt.Printf("%x\n", n.tab.self.ID[:8]) - // transport := n.conn.(*simTransport) - // fmt.Println(" joined:", transport.joinTime) - // fmt.Println(" sends:", transport.hashctr) - // fmt.Println(" table size:", n.tab.count) - // } - - /*for _, n := range s.nodes { - fmt.Println() - fmt.Printf("*** Node %x\n", n.tab.self.ID[:8]) - n.log.printLogs() - }*/ - -} - -func (s *simulation) randomNode() *Network { - s.mu.Lock() - defer s.mu.Unlock() - - n := rand.Intn(len(s.nodes)) - for _, net := range s.nodes { - if n == 0 { - return net - } - n-- - } - return nil -} - -func (s *simulation) launchNode(log bool) *Network { - var ( - num = s.nodectr - key = newkey() - id = PubkeyID(&key.PublicKey) - ip = make(net.IP, 4) - ) - s.nodectr++ - binary.BigEndian.PutUint32(ip, num) - ip[0] = 10 - addr := &net.UDPAddr{IP: ip, Port: 30303} - - transport := &simTransport{joinTime: time.Now(), sender: id, senderAddr: addr, sim: s, priv: key} - net, err := newNetwork(transport, key.PublicKey, "", nil) - if err != nil { - panic("cannot launch new node: " + err.Error()) - } - - s.mu.Lock() - s.nodes[id] = net - s.mu.Unlock() - - return net -} - -type simTransport struct { - joinTime time.Time - sender NodeID - senderAddr *net.UDPAddr - sim *simulation - hashctr uint64 - priv *ecdsa.PrivateKey -} - -func (st *simTransport) localAddr() *net.UDPAddr { - return st.senderAddr -} - -func (st *simTransport) Close() {} - -func (st *simTransport) send(remote *Node, ptype nodeEvent, data interface{}) (hash []byte) { - hash = st.nextHash() - var raw []byte - if ptype == pongPacket { - var err error - raw, _, err = encodePacket(st.priv, byte(ptype), data) - if err != nil { - panic(err) - } - } - - st.sendPacket(remote.ID, ingressPacket{ - remoteID: st.sender, - remoteAddr: st.senderAddr, - hash: hash, - ev: ptype, - data: data, - rawData: raw, - }) - return hash -} - -func (st *simTransport) sendPing(remote *Node, remoteAddr *net.UDPAddr, topics []Topic) []byte { - hash := st.nextHash() - st.sendPacket(remote.ID, ingressPacket{ - remoteID: st.sender, - remoteAddr: st.senderAddr, - hash: hash, - ev: pingPacket, - data: &ping{ - Version: 4, - From: rpcEndpoint{IP: st.senderAddr.IP, UDP: uint16(st.senderAddr.Port), TCP: 30303}, - To: rpcEndpoint{IP: remoteAddr.IP, UDP: uint16(remoteAddr.Port), TCP: 30303}, - Expiration: uint64(time.Now().Unix() + int64(expiration)), - Topics: topics, - }, - }) - return hash -} - -func (st *simTransport) sendFindnodeHash(remote *Node, target common.Hash) { - st.sendPacket(remote.ID, ingressPacket{ - remoteID: st.sender, - remoteAddr: st.senderAddr, - hash: st.nextHash(), - ev: findnodeHashPacket, - data: &findnodeHash{ - Target: target, - Expiration: uint64(time.Now().Unix() + int64(expiration)), - }, - }) -} - -func (st *simTransport) sendTopicRegister(remote *Node, topics []Topic, idx int, pong []byte) { - //fmt.Println("send", topics, pong) - st.sendPacket(remote.ID, ingressPacket{ - remoteID: st.sender, - remoteAddr: st.senderAddr, - hash: st.nextHash(), - ev: topicRegisterPacket, - data: &topicRegister{ - Topics: topics, - Idx: uint(idx), - Pong: pong, - }, - }) -} - -func (st *simTransport) sendTopicNodes(remote *Node, queryHash common.Hash, nodes []*Node) { - rnodes := make([]rpcNode, len(nodes)) - for i := range nodes { - rnodes[i] = nodeToRPC(nodes[i]) - } - st.sendPacket(remote.ID, ingressPacket{ - remoteID: st.sender, - remoteAddr: st.senderAddr, - hash: st.nextHash(), - ev: topicNodesPacket, - data: &topicNodes{Echo: queryHash, Nodes: rnodes}, - }) -} - -func (st *simTransport) sendNeighbours(remote *Node, nodes []*Node) { - // TODO: send multiple packets - rnodes := make([]rpcNode, len(nodes)) - for i := range nodes { - rnodes[i] = nodeToRPC(nodes[i]) - } - st.sendPacket(remote.ID, ingressPacket{ - remoteID: st.sender, - remoteAddr: st.senderAddr, - hash: st.nextHash(), - ev: neighborsPacket, - data: &neighbors{ - Nodes: rnodes, - Expiration: uint64(time.Now().Unix() + int64(expiration)), - }, - }) -} - -func (st *simTransport) nextHash() []byte { - v := atomic.AddUint64(&st.hashctr, 1) - var hash common.Hash - binary.BigEndian.PutUint64(hash[:], v) - return hash[:] -} - -const packetLoss = 0 // 1/1000 - -func (st *simTransport) sendPacket(remote NodeID, p ingressPacket) { - if rand.Int31n(1000) >= packetLoss { - st.sim.mu.RLock() - recipient := st.sim.nodes[remote] - st.sim.mu.RUnlock() - - time.AfterFunc(200*time.Millisecond, func() { - recipient.reqReadPacket(p) - }) - } -} diff --git a/p2p/discv5/sim_testmain_test.go b/p2p/discv5/sim_testmain_test.go deleted file mode 100644 index 77e751c419..0000000000 --- a/p2p/discv5/sim_testmain_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build go1.4,nacl,faketime_simulation - -package discv5 - -import ( - "os" - "runtime" - "testing" - "unsafe" -) - -// Enable fake time mode in the runtime, like on the go playground. -// There is a slight chance that this won't work because some go code -// might have executed before the variable is set. - -//go:linkname faketime runtime.faketime -var faketime = 1 - -func TestMain(m *testing.M) { - // We need to use unsafe somehow in order to get access to go:linkname. - _ = unsafe.Sizeof(0) - - // Run the actual test. runWithPlaygroundTime ensures that the only test - // that runs is the one calling it. - runtime.GOMAXPROCS(8) - os.Exit(m.Run()) -} diff --git a/p2p/discv5/table.go b/p2p/discv5/table.go deleted file mode 100644 index 64c3ecd1c7..0000000000 --- a/p2p/discv5/table.go +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package discv5 is a prototype implementation of Discvery v5. -// Deprecated: do not use this package. -package discv5 - -import ( - "crypto/rand" - "encoding/binary" - "fmt" - "net" - "sort" - - "github.com/ethereum/go-ethereum/common" -) - -const ( - alpha = 3 // Kademlia concurrency factor - bucketSize = 16 // Kademlia bucket size - hashBits = len(common.Hash{}) * 8 - nBuckets = hashBits + 1 // Number of buckets - - maxFindnodeFailures = 5 -) - -type Table struct { - count int // number of nodes - buckets [nBuckets]*bucket // index of known nodes by distance - nodeAddedHook func(*Node) // for testing - self *Node // metadata of the local node -} - -// bucket contains nodes, ordered by their last activity. the entry -// that was most recently active is the first element in entries. -type bucket struct { - entries []*Node - replacements []*Node -} - -func newTable(ourID NodeID, ourAddr *net.UDPAddr) *Table { - self := NewNode(ourID, ourAddr.IP, uint16(ourAddr.Port), uint16(ourAddr.Port)) - tab := &Table{self: self} - for i := range tab.buckets { - tab.buckets[i] = new(bucket) - } - return tab -} - -const printTable = false - -// chooseBucketRefreshTarget selects random refresh targets to keep all Kademlia -// buckets filled with live connections and keep the network topology healthy. -// This requires selecting addresses closer to our own with a higher probability -// in order to refresh closer buckets too. -// -// This algorithm approximates the distance distribution of existing nodes in the -// table by selecting a random node from the table and selecting a target address -// with a distance less than twice of that of the selected node. -// This algorithm will be improved later to specifically target the least recently -// used buckets. -func (tab *Table) chooseBucketRefreshTarget() common.Hash { - entries := 0 - if printTable { - fmt.Println() - } - for i, b := range &tab.buckets { - entries += len(b.entries) - if printTable { - for _, e := range b.entries { - fmt.Println(i, e.state, e.addr().String(), e.ID.String(), e.sha.Hex()) - } - } - } - - prefix := binary.BigEndian.Uint64(tab.self.sha[0:8]) - dist := ^uint64(0) - entry := int(randUint(uint32(entries + 1))) - for _, b := range &tab.buckets { - if entry < len(b.entries) { - n := b.entries[entry] - dist = binary.BigEndian.Uint64(n.sha[0:8]) ^ prefix - break - } - entry -= len(b.entries) - } - - ddist := ^uint64(0) - if dist+dist > dist { - ddist = dist - } - targetPrefix := prefix ^ randUint64n(ddist) - - var target common.Hash - binary.BigEndian.PutUint64(target[0:8], targetPrefix) - rand.Read(target[8:]) - return target -} - -// readRandomNodes fills the given slice with random nodes from the -// table. It will not write the same node more than once. The nodes in -// the slice are copies and can be modified by the caller. -func (tab *Table) readRandomNodes(buf []*Node) (n int) { - // TODO: tree-based buckets would help here - // Find all non-empty buckets and get a fresh slice of their entries. - var buckets [][]*Node - for _, b := range &tab.buckets { - if len(b.entries) > 0 { - buckets = append(buckets, b.entries) - } - } - if len(buckets) == 0 { - return 0 - } - // Shuffle the buckets. - for i := uint32(len(buckets)) - 1; i > 0; i-- { - j := randUint(i) - buckets[i], buckets[j] = buckets[j], buckets[i] - } - // Move head of each bucket into buf, removing buckets that become empty. - var i, j int - for ; i < len(buf); i, j = i+1, (j+1)%len(buckets) { - b := buckets[j] - buf[i] = &(*b[0]) - buckets[j] = b[1:] - if len(b) == 1 { - buckets = append(buckets[:j], buckets[j+1:]...) - } - if len(buckets) == 0 { - break - } - } - return i + 1 -} - -func randUint(max uint32) uint32 { - if max < 2 { - return 0 - } - var b [4]byte - rand.Read(b[:]) - return binary.BigEndian.Uint32(b[:]) % max -} - -func randUint64n(max uint64) uint64 { - if max < 2 { - return 0 - } - var b [8]byte - rand.Read(b[:]) - return binary.BigEndian.Uint64(b[:]) % max -} - -// closest returns the n nodes in the table that are closest to the -// given id. The caller must hold tab.mutex. -func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { - // This is a very wasteful way to find the closest nodes but - // obviously correct. I believe that tree-based buckets would make - // this easier to implement efficiently. - close := &nodesByDistance{target: target} - for _, b := range &tab.buckets { - for _, n := range b.entries { - close.push(n, nresults) - } - } - return close -} - -// add attempts to add the given node its corresponding bucket. If the -// bucket has space available, adding the node succeeds immediately. -// Otherwise, the node is added to the replacement cache for the bucket. -func (tab *Table) add(n *Node) (contested *Node) { - //fmt.Println("add", n.addr().String(), n.ID.String(), n.sha.Hex()) - if n.ID == tab.self.ID { - return - } - b := tab.buckets[logdist(tab.self.sha, n.sha)] - switch { - case b.bump(n): - // n exists in b. - return nil - case len(b.entries) < bucketSize: - // b has space available. - b.addFront(n) - tab.count++ - if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) - } - return nil - default: - // b has no space left, add to replacement cache - // and revalidate the last entry. - // TODO: drop previous node - b.replacements = append(b.replacements, n) - if len(b.replacements) > bucketSize { - copy(b.replacements, b.replacements[1:]) - b.replacements = b.replacements[:len(b.replacements)-1] - } - return b.entries[len(b.entries)-1] - } -} - -// stuff adds nodes the table to the end of their corresponding bucket -// if the bucket is not full. -func (tab *Table) stuff(nodes []*Node) { -outer: - for _, n := range nodes { - if n.ID == tab.self.ID { - continue // don't add self - } - bucket := tab.buckets[logdist(tab.self.sha, n.sha)] - for i := range bucket.entries { - if bucket.entries[i].ID == n.ID { - continue outer // already in bucket - } - } - if len(bucket.entries) < bucketSize { - bucket.entries = append(bucket.entries, n) - tab.count++ - if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) - } - } - } -} - -// delete removes an entry from the node table (used to evacuate -// failed/non-bonded discovery peers). -func (tab *Table) delete(node *Node) { - //fmt.Println("delete", node.addr().String(), node.ID.String(), node.sha.Hex()) - bucket := tab.buckets[logdist(tab.self.sha, node.sha)] - for i := range bucket.entries { - if bucket.entries[i].ID == node.ID { - bucket.entries = append(bucket.entries[:i], bucket.entries[i+1:]...) - tab.count-- - return - } - } -} - -func (tab *Table) deleteReplace(node *Node) { - b := tab.buckets[logdist(tab.self.sha, node.sha)] - i := 0 - for i < len(b.entries) { - if b.entries[i].ID == node.ID { - b.entries = append(b.entries[:i], b.entries[i+1:]...) - tab.count-- - } else { - i++ - } - } - // refill from replacement cache - // TODO: maybe use random index - if len(b.entries) < bucketSize && len(b.replacements) > 0 { - ri := len(b.replacements) - 1 - b.addFront(b.replacements[ri]) - tab.count++ - b.replacements[ri] = nil - b.replacements = b.replacements[:ri] - } -} - -func (b *bucket) addFront(n *Node) { - b.entries = append(b.entries, nil) - copy(b.entries[1:], b.entries) - b.entries[0] = n -} - -func (b *bucket) bump(n *Node) bool { - for i := range b.entries { - if b.entries[i].ID == n.ID { - // move it to the front - copy(b.entries[1:], b.entries[:i]) - b.entries[0] = n - return true - } - } - return false -} - -// nodesByDistance is a list of nodes, ordered by -// distance to target. -type nodesByDistance struct { - entries []*Node - target common.Hash -} - -// push adds the given node to the list, keeping the total size below maxElems. -func (h *nodesByDistance) push(n *Node, maxElems int) { - ix := sort.Search(len(h.entries), func(i int) bool { - return distcmp(h.target, h.entries[i].sha, n.sha) > 0 - }) - if len(h.entries) < maxElems { - h.entries = append(h.entries, n) - } - if ix == len(h.entries) { - // farther away than all nodes we already have. - // if there was room for it, the node is now the last element. - } else { - // slide existing entries down to make room - // this will overwrite the entry we just appended. - copy(h.entries[ix+1:], h.entries[ix:]) - h.entries[ix] = n - } -} diff --git a/p2p/discv5/table_test.go b/p2p/discv5/table_test.go deleted file mode 100644 index 872a4f6836..0000000000 --- a/p2p/discv5/table_test.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "crypto/ecdsa" - "fmt" - "math/rand" - - "net" - "reflect" - "testing" - "testing/quick" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" -) - -func TestBucket_bumpNoDuplicates(t *testing.T) { - t.Parallel() - cfg := &quick.Config{ - MaxCount: 1000, - Rand: rand.New(rand.NewSource(time.Now().Unix())), - Values: func(args []reflect.Value, rand *rand.Rand) { - // generate a random list of nodes. this will be the content of the bucket. - n := rand.Intn(bucketSize-1) + 1 - nodes := make([]*Node, n) - for i := range nodes { - nodes[i] = nodeAtDistance(common.Hash{}, 200) - } - args[0] = reflect.ValueOf(nodes) - // generate random bump positions. - bumps := make([]int, rand.Intn(100)) - for i := range bumps { - bumps[i] = rand.Intn(len(nodes)) - } - args[1] = reflect.ValueOf(bumps) - }, - } - - prop := func(nodes []*Node, bumps []int) (ok bool) { - b := &bucket{entries: make([]*Node, len(nodes))} - copy(b.entries, nodes) - for i, pos := range bumps { - b.bump(b.entries[pos]) - if hasDuplicates(b.entries) { - t.Logf("bucket has duplicates after %d/%d bumps:", i+1, len(bumps)) - for _, n := range b.entries { - t.Logf(" %p", n) - } - return false - } - } - return true - } - if err := quick.Check(prop, cfg); err != nil { - t.Error(err) - } -} - -// nodeAtDistance creates a node for which logdist(base, n.sha) == ld. -// The node's ID does not correspond to n.sha. -func nodeAtDistance(base common.Hash, ld int) (n *Node) { - n = new(Node) - n.sha = hashAtDistance(base, ld) - copy(n.ID[:], n.sha[:]) // ensure the node still has a unique ID - return n -} - -func TestTable_closest(t *testing.T) { - t.Parallel() - - test := func(test *closeTest) bool { - // for any node table, Target and N - tab := newTable(test.Self, &net.UDPAddr{}) - tab.stuff(test.All) - - // check that doClosest(Target, N) returns nodes - result := tab.closest(test.Target, test.N).entries - if hasDuplicates(result) { - t.Errorf("result contains duplicates") - return false - } - if !sortedByDistanceTo(test.Target, result) { - t.Errorf("result is not sorted by distance to target") - return false - } - - // check that the number of results is min(N, tablen) - wantN := test.N - if tab.count < test.N { - wantN = tab.count - } - if len(result) != wantN { - t.Errorf("wrong number of nodes: got %d, want %d", len(result), wantN) - return false - } else if len(result) == 0 { - return true // no need to check distance - } - - // check that the result nodes have minimum distance to target. - for _, b := range tab.buckets { - for _, n := range b.entries { - if contains(result, n.ID) { - continue // don't run the check below for nodes in result - } - farthestResult := result[len(result)-1].sha - if distcmp(test.Target, n.sha, farthestResult) < 0 { - t.Errorf("table contains node that is closer to target but it's not in result") - t.Logf(" Target: %v", test.Target) - t.Logf(" Farthest Result: %v", farthestResult) - t.Logf(" ID: %v", n.ID) - return false - } - } - } - return true - } - if err := quick.Check(test, quickcfg()); err != nil { - t.Error(err) - } -} - -func TestTable_ReadRandomNodesGetAll(t *testing.T) { - cfg := &quick.Config{ - MaxCount: 200, - Rand: rand.New(rand.NewSource(time.Now().Unix())), - Values: func(args []reflect.Value, rand *rand.Rand) { - args[0] = reflect.ValueOf(make([]*Node, rand.Intn(1000))) - }, - } - test := func(buf []*Node) bool { - tab := newTable(NodeID{}, &net.UDPAddr{}) - for i := 0; i < len(buf); i++ { - ld := cfg.Rand.Intn(len(tab.buckets)) - tab.stuff([]*Node{nodeAtDistance(tab.self.sha, ld)}) - } - gotN := tab.readRandomNodes(buf) - if gotN != tab.count { - t.Errorf("wrong number of nodes, got %d, want %d", gotN, tab.count) - return false - } - if hasDuplicates(buf[:gotN]) { - t.Errorf("result contains duplicates") - return false - } - return true - } - if err := quick.Check(test, cfg); err != nil { - t.Error(err) - } -} - -type closeTest struct { - Self NodeID - Target common.Hash - All []*Node - N int -} - -func (*closeTest) Generate(rand *rand.Rand, size int) reflect.Value { - t := &closeTest{ - Self: gen(NodeID{}, rand).(NodeID), - Target: gen(common.Hash{}, rand).(common.Hash), - N: rand.Intn(bucketSize), - } - for _, id := range gen([]NodeID{}, rand).([]NodeID) { - t.All = append(t.All, &Node{ID: id}) - } - return reflect.ValueOf(t) -} - -func hasDuplicates(slice []*Node) bool { - seen := make(map[NodeID]bool) - for i, e := range slice { - if e == nil { - panic(fmt.Sprintf("nil *Node at %d", i)) - } - if seen[e.ID] { - return true - } - seen[e.ID] = true - } - return false -} - -func sortedByDistanceTo(distbase common.Hash, slice []*Node) bool { - var last common.Hash - for i, e := range slice { - if i > 0 && distcmp(distbase, e.sha, last) < 0 { - return false - } - last = e.sha - } - return true -} - -func contains(ns []*Node, id NodeID) bool { - for _, n := range ns { - if n.ID == id { - return true - } - } - return false -} - -// gen wraps quick.Value so it's easier to use. -// it generates a random value of the given value's type. -func gen(typ interface{}, rand *rand.Rand) interface{} { - v, ok := quick.Value(reflect.TypeOf(typ), rand) - if !ok { - panic(fmt.Sprintf("couldn't generate random value of type %T", typ)) - } - return v.Interface() -} - -func newkey() *ecdsa.PrivateKey { - key, err := crypto.GenerateKey() - if err != nil { - panic("couldn't generate key: " + err.Error()) - } - return key -} diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go deleted file mode 100644 index c5e3d6c08f..0000000000 --- a/p2p/discv5/ticket.go +++ /dev/null @@ -1,884 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" -) - -const ( - ticketTimeBucketLen = time.Minute - collectFrequency = time.Second * 30 - registerFrequency = time.Second * 60 - maxCollectDebt = 10 - maxRegisterDebt = 5 - keepTicketConst = time.Minute * 10 - keepTicketExp = time.Minute * 5 - targetWaitTime = time.Minute * 10 - topicQueryTimeout = time.Second * 5 - topicQueryResend = time.Minute - // topic radius detection - maxRadius = 0xffffffffffffffff - radiusTC = time.Minute * 20 - radiusBucketsPerBit = 8 - minSlope = 1 - minPeakSize = 40 - maxNoAdjust = 20 - lookupWidth = 8 - minRightSum = 20 - searchForceQuery = 4 -) - -// timeBucket represents absolute monotonic time in minutes. -// It is used as the index into the per-topic ticket buckets. -type timeBucket int - -type ticket struct { - topics []Topic - regTime []mclock.AbsTime // Per-topic local absolute time when the ticket can be used. - - // The serial number that was issued by the server. - serial uint32 - // Used by registrar, tracks absolute time when the ticket was created. - issueTime mclock.AbsTime - - // Fields used only by registrants - node *Node // the registrar node that signed this ticket - refCnt int // tracks number of topics that will be registered using this ticket - pong []byte // encoded pong packet signed by the registrar -} - -// ticketRef refers to a single topic in a ticket. -type ticketRef struct { - t *ticket - idx int // index of the topic in t.topics and t.regTime -} - -func (ref ticketRef) topic() Topic { - return ref.t.topics[ref.idx] -} - -func (ref ticketRef) topicRegTime() mclock.AbsTime { - return ref.t.regTime[ref.idx] -} - -func pongToTicket(localTime mclock.AbsTime, topics []Topic, node *Node, p *ingressPacket) (*ticket, error) { - wps := p.data.(*pong).WaitPeriods - if len(topics) != len(wps) { - return nil, fmt.Errorf("bad wait period list: got %d values, want %d", len(topics), len(wps)) - } - if rlpHash(topics) != p.data.(*pong).TopicHash { - return nil, fmt.Errorf("bad topic hash") - } - t := &ticket{ - issueTime: localTime, - node: node, - topics: topics, - pong: p.rawData, - regTime: make([]mclock.AbsTime, len(wps)), - } - // Convert wait periods to local absolute time. - for i, wp := range wps { - t.regTime[i] = localTime + mclock.AbsTime(time.Second*time.Duration(wp)) - } - return t, nil -} - -func ticketToPong(t *ticket, pong *pong) { - pong.Expiration = uint64(t.issueTime / mclock.AbsTime(time.Second)) - pong.TopicHash = rlpHash(t.topics) - pong.TicketSerial = t.serial - pong.WaitPeriods = make([]uint32, len(t.regTime)) - for i, regTime := range t.regTime { - pong.WaitPeriods[i] = uint32(time.Duration(regTime-t.issueTime) / time.Second) - } -} - -type ticketStore struct { - // radius detector and target address generator - // exists for both searched and registered topics - radius map[Topic]*topicRadius - - // Contains buckets (for each absolute minute) of tickets - // that can be used in that minute. - // This is only set if the topic is being registered. - tickets map[Topic]*topicTickets - - regQueue []Topic // Topic registration queue for round robin attempts - regSet map[Topic]struct{} // Topic registration queue contents for fast filling - - nodes map[*Node]*ticket - nodeLastReq map[*Node]reqInfo - - lastBucketFetched timeBucket - nextTicketCached *ticketRef - - searchTopicMap map[Topic]searchTopic - nextTopicQueryCleanup mclock.AbsTime - queriesSent map[*Node]map[common.Hash]sentQuery -} - -type searchTopic struct { - foundChn chan<- *Node -} - -type sentQuery struct { - sent mclock.AbsTime - lookup lookupInfo -} - -type topicTickets struct { - buckets map[timeBucket][]ticketRef - nextLookup mclock.AbsTime - nextReg mclock.AbsTime -} - -func newTicketStore() *ticketStore { - return &ticketStore{ - radius: make(map[Topic]*topicRadius), - tickets: make(map[Topic]*topicTickets), - regSet: make(map[Topic]struct{}), - nodes: make(map[*Node]*ticket), - nodeLastReq: make(map[*Node]reqInfo), - searchTopicMap: make(map[Topic]searchTopic), - queriesSent: make(map[*Node]map[common.Hash]sentQuery), - } -} - -// addTopic starts tracking a topic. If register is true, -// the local node will register the topic and tickets will be collected. -func (s *ticketStore) addTopic(topic Topic, register bool) { - log.Trace("Adding discovery topic", "topic", topic, "register", register) - if s.radius[topic] == nil { - s.radius[topic] = newTopicRadius(topic) - } - if register && s.tickets[topic] == nil { - s.tickets[topic] = &topicTickets{buckets: make(map[timeBucket][]ticketRef)} - } -} - -func (s *ticketStore) addSearchTopic(t Topic, foundChn chan<- *Node) { - s.addTopic(t, false) - if s.searchTopicMap[t].foundChn == nil { - s.searchTopicMap[t] = searchTopic{foundChn: foundChn} - } -} - -func (s *ticketStore) removeSearchTopic(t Topic) { - if st := s.searchTopicMap[t]; st.foundChn != nil { - delete(s.searchTopicMap, t) - } -} - -// removeRegisterTopic deletes all tickets for the given topic. -func (s *ticketStore) removeRegisterTopic(topic Topic) { - log.Trace("Removing discovery topic", "topic", topic) - if s.tickets[topic] == nil { - log.Warn("Removing non-existent discovery topic", "topic", topic) - return - } - for _, list := range s.tickets[topic].buckets { - for _, ref := range list { - ref.t.refCnt-- - if ref.t.refCnt == 0 { - delete(s.nodes, ref.t.node) - delete(s.nodeLastReq, ref.t.node) - } - } - } - delete(s.tickets, topic) -} - -func (s *ticketStore) regTopicSet() []Topic { - topics := make([]Topic, 0, len(s.tickets)) - for topic := range s.tickets { - topics = append(topics, topic) - } - return topics -} - -// nextRegisterLookup returns the target of the next lookup for ticket collection. -func (s *ticketStore) nextRegisterLookup() (lookupInfo, time.Duration) { - // Queue up any new topics (or discarded ones), preserving iteration order - for topic := range s.tickets { - if _, ok := s.regSet[topic]; !ok { - s.regQueue = append(s.regQueue, topic) - s.regSet[topic] = struct{}{} - } - } - // Iterate over the set of all topics and look up the next suitable one - for len(s.regQueue) > 0 { - // Fetch the next topic from the queue, and ensure it still exists - topic := s.regQueue[0] - s.regQueue = s.regQueue[1:] - delete(s.regSet, topic) - - if s.tickets[topic] == nil { - continue - } - // If the topic needs more tickets, return it - if s.tickets[topic].nextLookup < mclock.Now() { - next, delay := s.radius[topic].nextTarget(false), 100*time.Millisecond - log.Trace("Found discovery topic to register", "topic", topic, "target", next.target, "delay", delay) - return next, delay - } - } - // No registration topics found or all exhausted, sleep - delay := 40 * time.Second - log.Trace("No topic found to register", "delay", delay) - return lookupInfo{}, delay -} - -func (s *ticketStore) nextSearchLookup(topic Topic) lookupInfo { - tr := s.radius[topic] - target := tr.nextTarget(tr.radiusLookupCnt >= searchForceQuery) - if target.radiusLookup { - tr.radiusLookupCnt++ - } else { - tr.radiusLookupCnt = 0 - } - return target -} - -func (s *ticketStore) addTicketRef(r ticketRef) { - topic := r.t.topics[r.idx] - tickets := s.tickets[topic] - if tickets == nil { - log.Warn("Adding ticket to non-existent topic", "topic", topic) - return - } - bucket := timeBucket(r.t.regTime[r.idx] / mclock.AbsTime(ticketTimeBucketLen)) - tickets.buckets[bucket] = append(tickets.buckets[bucket], r) - r.t.refCnt++ - - min := mclock.Now() - mclock.AbsTime(collectFrequency)*maxCollectDebt - if tickets.nextLookup < min { - tickets.nextLookup = min - } - tickets.nextLookup += mclock.AbsTime(collectFrequency) - - //s.removeExcessTickets(topic) -} - -func (s *ticketStore) nextFilteredTicket() (*ticketRef, time.Duration) { - now := mclock.Now() - for { - ticket, wait := s.nextRegisterableTicket() - if ticket == nil { - return ticket, wait - } - log.Trace("Found discovery ticket to register", "node", ticket.t.node, "serial", ticket.t.serial, "wait", wait) - - regTime := now + mclock.AbsTime(wait) - topic := ticket.t.topics[ticket.idx] - if s.tickets[topic] != nil && regTime >= s.tickets[topic].nextReg { - return ticket, wait - } - s.removeTicketRef(*ticket) - } -} - -func (s *ticketStore) ticketRegistered(ref ticketRef) { - now := mclock.Now() - - topic := ref.t.topics[ref.idx] - tickets := s.tickets[topic] - min := now - mclock.AbsTime(registerFrequency)*maxRegisterDebt - if min > tickets.nextReg { - tickets.nextReg = min - } - tickets.nextReg += mclock.AbsTime(registerFrequency) - s.tickets[topic] = tickets - - s.removeTicketRef(ref) -} - -// nextRegisterableTicket returns the next ticket that can be used -// to register. -// -// If the returned wait time <= zero the ticket can be used. For a positive -// wait time, the caller should requery the next ticket later. -// -// A ticket can be returned more than once with <= zero wait time in case -// the ticket contains multiple topics. -func (s *ticketStore) nextRegisterableTicket() (*ticketRef, time.Duration) { - now := mclock.Now() - if s.nextTicketCached != nil { - return s.nextTicketCached, time.Duration(s.nextTicketCached.topicRegTime() - now) - } - - for bucket := s.lastBucketFetched; ; bucket++ { - var ( - empty = true // true if there are no tickets - nextTicket ticketRef // uninitialized if this bucket is empty - ) - for _, tickets := range s.tickets { - //s.removeExcessTickets(topic) - if len(tickets.buckets) != 0 { - empty = false - - list := tickets.buckets[bucket] - for _, ref := range list { - //debugLog(fmt.Sprintf(" nrt bucket = %d node = %x sn = %v wait = %v", bucket, ref.t.node.ID[:8], ref.t.serial, time.Duration(ref.topicRegTime()-now))) - if nextTicket.t == nil || ref.topicRegTime() < nextTicket.topicRegTime() { - nextTicket = ref - } - } - } - } - if empty { - return nil, 0 - } - if nextTicket.t != nil { - s.nextTicketCached = &nextTicket - return &nextTicket, time.Duration(nextTicket.topicRegTime() - now) - } - s.lastBucketFetched = bucket - } -} - -// removeTicket removes a ticket from the ticket store -func (s *ticketStore) removeTicketRef(ref ticketRef) { - log.Trace("Removing discovery ticket reference", "node", ref.t.node.ID, "serial", ref.t.serial) - - // Make nextRegisterableTicket return the next available ticket. - s.nextTicketCached = nil - - topic := ref.topic() - tickets := s.tickets[topic] - - if tickets == nil { - log.Trace("Removing tickets from unknown topic", "topic", topic) - return - } - bucket := timeBucket(ref.t.regTime[ref.idx] / mclock.AbsTime(ticketTimeBucketLen)) - list := tickets.buckets[bucket] - idx := -1 - for i, bt := range list { - if bt.t == ref.t { - idx = i - break - } - } - if idx == -1 { - panic(nil) - } - list = append(list[:idx], list[idx+1:]...) - if len(list) != 0 { - tickets.buckets[bucket] = list - } else { - delete(tickets.buckets, bucket) - } - ref.t.refCnt-- - if ref.t.refCnt == 0 { - delete(s.nodes, ref.t.node) - delete(s.nodeLastReq, ref.t.node) - } -} - -type lookupInfo struct { - target common.Hash - topic Topic - radiusLookup bool -} - -type reqInfo struct { - pingHash []byte - lookup lookupInfo - time mclock.AbsTime -} - -// returns -1 if not found -func (t *ticket) findIdx(topic Topic) int { - for i, tt := range t.topics { - if tt == topic { - return i - } - } - return -1 -} - -func (s *ticketStore) registerLookupDone(lookup lookupInfo, nodes []*Node, ping func(n *Node) []byte) { - now := mclock.Now() - for i, n := range nodes { - if i == 0 || (binary.BigEndian.Uint64(n.sha[:8])^binary.BigEndian.Uint64(lookup.target[:8])) < s.radius[lookup.topic].minRadius { - if lookup.radiusLookup { - if lastReq, ok := s.nodeLastReq[n]; !ok || time.Duration(now-lastReq.time) > radiusTC { - s.nodeLastReq[n] = reqInfo{pingHash: ping(n), lookup: lookup, time: now} - } - } else { - if s.nodes[n] == nil { - s.nodeLastReq[n] = reqInfo{pingHash: ping(n), lookup: lookup, time: now} - } - } - } - } -} - -func (s *ticketStore) searchLookupDone(lookup lookupInfo, nodes []*Node, query func(n *Node, topic Topic) []byte) { - now := mclock.Now() - for i, n := range nodes { - if i == 0 || (binary.BigEndian.Uint64(n.sha[:8])^binary.BigEndian.Uint64(lookup.target[:8])) < s.radius[lookup.topic].minRadius { - if lookup.radiusLookup { - if lastReq, ok := s.nodeLastReq[n]; !ok || time.Duration(now-lastReq.time) > radiusTC { - s.nodeLastReq[n] = reqInfo{pingHash: nil, lookup: lookup, time: now} - } - } // else { - if s.canQueryTopic(n, lookup.topic) { - hash := query(n, lookup.topic) - if hash != nil { - s.addTopicQuery(common.BytesToHash(hash), n, lookup) - } - } - //} - } - } -} - -func (s *ticketStore) adjustWithTicket(now mclock.AbsTime, targetHash common.Hash, t *ticket) { - for i, topic := range t.topics { - if tt, ok := s.radius[topic]; ok { - tt.adjustWithTicket(now, targetHash, ticketRef{t, i}) - } - } -} - -func (s *ticketStore) addTicket(localTime mclock.AbsTime, pingHash []byte, ticket *ticket) { - log.Trace("Adding discovery ticket", "node", ticket.node.ID, "serial", ticket.serial) - - lastReq, ok := s.nodeLastReq[ticket.node] - if !(ok && bytes.Equal(pingHash, lastReq.pingHash)) { - return - } - s.adjustWithTicket(localTime, lastReq.lookup.target, ticket) - - if lastReq.lookup.radiusLookup || s.nodes[ticket.node] != nil { - return - } - - topic := lastReq.lookup.topic - topicIdx := ticket.findIdx(topic) - if topicIdx == -1 { - return - } - - bucket := timeBucket(localTime / mclock.AbsTime(ticketTimeBucketLen)) - if s.lastBucketFetched == 0 || bucket < s.lastBucketFetched { - s.lastBucketFetched = bucket - } - - if _, ok := s.tickets[topic]; ok { - wait := ticket.regTime[topicIdx] - localTime - rnd := rand.ExpFloat64() - if rnd > 10 { - rnd = 10 - } - if float64(wait) < float64(keepTicketConst)+float64(keepTicketExp)*rnd { - // use the ticket to register this topic - //fmt.Println("addTicket", ticket.node.ID[:8], ticket.node.addr().String(), ticket.serial, ticket.pong) - s.addTicketRef(ticketRef{ticket, topicIdx}) - } - } - - if ticket.refCnt > 0 { - s.nextTicketCached = nil - s.nodes[ticket.node] = ticket - } -} - -func (s *ticketStore) canQueryTopic(node *Node, topic Topic) bool { - qq := s.queriesSent[node] - if qq != nil { - now := mclock.Now() - for _, sq := range qq { - if sq.lookup.topic == topic && sq.sent > now-mclock.AbsTime(topicQueryResend) { - return false - } - } - } - return true -} - -func (s *ticketStore) addTopicQuery(hash common.Hash, node *Node, lookup lookupInfo) { - now := mclock.Now() - qq := s.queriesSent[node] - if qq == nil { - qq = make(map[common.Hash]sentQuery) - s.queriesSent[node] = qq - } - qq[hash] = sentQuery{sent: now, lookup: lookup} - s.cleanupTopicQueries(now) -} - -func (s *ticketStore) cleanupTopicQueries(now mclock.AbsTime) { - if s.nextTopicQueryCleanup > now { - return - } - exp := now - mclock.AbsTime(topicQueryResend) - for n, qq := range s.queriesSent { - for h, q := range qq { - if q.sent < exp { - delete(qq, h) - } - } - if len(qq) == 0 { - delete(s.queriesSent, n) - } - } - s.nextTopicQueryCleanup = now + mclock.AbsTime(topicQueryTimeout) -} - -func (s *ticketStore) gotTopicNodes(from *Node, hash common.Hash, nodes []rpcNode) (timeout bool) { - now := mclock.Now() - //fmt.Println("got", from.addr().String(), hash, len(nodes)) - qq := s.queriesSent[from] - if qq == nil { - return true - } - q, ok := qq[hash] - if !ok || now > q.sent+mclock.AbsTime(topicQueryTimeout) { - return true - } - inside := float64(0) - if len(nodes) > 0 { - inside = 1 - } - s.radius[q.lookup.topic].adjust(now, q.lookup.target, from.sha, inside) - chn := s.searchTopicMap[q.lookup.topic].foundChn - if chn == nil { - //fmt.Println("no channel") - return false - } - for _, node := range nodes { - ip := node.IP - if ip.IsUnspecified() || ip.IsLoopback() { - ip = from.IP - } - n := NewNode(node.ID, ip, node.UDP, node.TCP) - select { - case chn <- n: - default: - return false - } - } - return false -} - -type topicRadius struct { - topic Topic - topicHashPrefix uint64 - radius, minRadius uint64 - buckets []topicRadiusBucket - converged bool - radiusLookupCnt int -} - -type topicRadiusEvent int - -const ( - trOutside topicRadiusEvent = iota - trInside - trNoAdjust - trCount -) - -type topicRadiusBucket struct { - weights [trCount]float64 - lastTime mclock.AbsTime - value float64 - lookupSent map[common.Hash]mclock.AbsTime -} - -func (b *topicRadiusBucket) update(now mclock.AbsTime) { - if now == b.lastTime { - return - } - exp := math.Exp(-float64(now-b.lastTime) / float64(radiusTC)) - for i, w := range b.weights { - b.weights[i] = w * exp - } - b.lastTime = now - - for target, tm := range b.lookupSent { - if now-tm > mclock.AbsTime(respTimeout) { - b.weights[trNoAdjust] += 1 - delete(b.lookupSent, target) - } - } -} - -func (b *topicRadiusBucket) adjust(now mclock.AbsTime, inside float64) { - b.update(now) - if inside <= 0 { - b.weights[trOutside] += 1 - } else { - if inside >= 1 { - b.weights[trInside] += 1 - } else { - b.weights[trInside] += inside - b.weights[trOutside] += 1 - inside - } - } -} - -func newTopicRadius(t Topic) *topicRadius { - topicHash := crypto.Keccak256Hash([]byte(t)) - topicHashPrefix := binary.BigEndian.Uint64(topicHash[0:8]) - - return &topicRadius{ - topic: t, - topicHashPrefix: topicHashPrefix, - radius: maxRadius, - minRadius: maxRadius, - } -} - -func (r *topicRadius) getBucketIdx(addrHash common.Hash) int { - prefix := binary.BigEndian.Uint64(addrHash[0:8]) - var log2 float64 - if prefix != r.topicHashPrefix { - log2 = math.Log2(float64(prefix ^ r.topicHashPrefix)) - } - bucket := int((64 - log2) * radiusBucketsPerBit) - max := 64*radiusBucketsPerBit - 1 - if bucket > max { - return max - } - if bucket < 0 { - return 0 - } - return bucket -} - -func (r *topicRadius) targetForBucket(bucket int) common.Hash { - min := math.Pow(2, 64-float64(bucket+1)/radiusBucketsPerBit) - max := math.Pow(2, 64-float64(bucket)/radiusBucketsPerBit) - a := uint64(min) - b := randUint64n(uint64(max - min)) - xor := a + b - if xor < a { - xor = ^uint64(0) - } - prefix := r.topicHashPrefix ^ xor - var target common.Hash - binary.BigEndian.PutUint64(target[0:8], prefix) - globalRandRead(target[8:]) - return target -} - -// package rand provides a Read function in Go 1.6 and later, but -// we can't use it yet because we still support Go 1.5. -func globalRandRead(b []byte) { - pos := 0 - val := 0 - for n := 0; n < len(b); n++ { - if pos == 0 { - val = rand.Int() - pos = 7 - } - b[n] = byte(val) - val >>= 8 - pos-- - } -} - -func (r *topicRadius) chooseLookupBucket(a, b int) int { - if a < 0 { - a = 0 - } - if a > b { - return -1 - } - c := 0 - for i := a; i <= b; i++ { - if i >= len(r.buckets) || r.buckets[i].weights[trNoAdjust] < maxNoAdjust { - c++ - } - } - if c == 0 { - return -1 - } - rnd := randUint(uint32(c)) - for i := a; i <= b; i++ { - if i >= len(r.buckets) || r.buckets[i].weights[trNoAdjust] < maxNoAdjust { - if rnd == 0 { - return i - } - rnd-- - } - } - panic(nil) // should never happen -} - -func (r *topicRadius) needMoreLookups(a, b int, maxValue float64) bool { - var max float64 - if a < 0 { - a = 0 - } - if b >= len(r.buckets) { - b = len(r.buckets) - 1 - if r.buckets[b].value > max { - max = r.buckets[b].value - } - } - if b >= a { - for i := a; i <= b; i++ { - if r.buckets[i].value > max { - max = r.buckets[i].value - } - } - } - return maxValue-max < minPeakSize -} - -func (r *topicRadius) recalcRadius() (radius uint64, radiusLookup int) { - maxBucket := 0 - maxValue := float64(0) - now := mclock.Now() - v := float64(0) - for i := range r.buckets { - r.buckets[i].update(now) - v += r.buckets[i].weights[trOutside] - r.buckets[i].weights[trInside] - r.buckets[i].value = v - //fmt.Printf("%v %v | ", v, r.buckets[i].weights[trNoAdjust]) - } - //fmt.Println() - slopeCross := -1 - for i, b := range r.buckets { - v := b.value - if v < float64(i)*minSlope { - slopeCross = i - break - } - if v > maxValue { - maxValue = v - maxBucket = i + 1 - } - } - - minRadBucket := len(r.buckets) - sum := float64(0) - for minRadBucket > 0 && sum < minRightSum { - minRadBucket-- - b := r.buckets[minRadBucket] - sum += b.weights[trInside] + b.weights[trOutside] - } - r.minRadius = uint64(math.Pow(2, 64-float64(minRadBucket)/radiusBucketsPerBit)) - - lookupLeft := -1 - if r.needMoreLookups(0, maxBucket-lookupWidth-1, maxValue) { - lookupLeft = r.chooseLookupBucket(maxBucket-lookupWidth, maxBucket-1) - } - lookupRight := -1 - if slopeCross != maxBucket && (minRadBucket <= maxBucket || r.needMoreLookups(maxBucket+lookupWidth, len(r.buckets)-1, maxValue)) { - for len(r.buckets) <= maxBucket+lookupWidth { - r.buckets = append(r.buckets, topicRadiusBucket{lookupSent: make(map[common.Hash]mclock.AbsTime)}) - } - lookupRight = r.chooseLookupBucket(maxBucket, maxBucket+lookupWidth-1) - } - if lookupLeft == -1 { - radiusLookup = lookupRight - } else { - if lookupRight == -1 { - radiusLookup = lookupLeft - } else { - if randUint(2) == 0 { - radiusLookup = lookupLeft - } else { - radiusLookup = lookupRight - } - } - } - - //fmt.Println("mb", maxBucket, "sc", slopeCross, "mrb", minRadBucket, "ll", lookupLeft, "lr", lookupRight, "mv", maxValue) - - if radiusLookup == -1 { - // no more radius lookups needed at the moment, return a radius - r.converged = true - rad := maxBucket - if minRadBucket < rad { - rad = minRadBucket - } - radius = ^uint64(0) - if rad > 0 { - radius = uint64(math.Pow(2, 64-float64(rad)/radiusBucketsPerBit)) - } - r.radius = radius - } - - return -} - -func (r *topicRadius) nextTarget(forceRegular bool) lookupInfo { - if !forceRegular { - _, radiusLookup := r.recalcRadius() - if radiusLookup != -1 { - target := r.targetForBucket(radiusLookup) - r.buckets[radiusLookup].lookupSent[target] = mclock.Now() - return lookupInfo{target: target, topic: r.topic, radiusLookup: true} - } - } - - radExt := r.radius / 2 - if radExt > maxRadius-r.radius { - radExt = maxRadius - r.radius - } - rnd := randUint64n(r.radius) + randUint64n(2*radExt) - if rnd > radExt { - rnd -= radExt - } else { - rnd = radExt - rnd - } - - prefix := r.topicHashPrefix ^ rnd - var target common.Hash - binary.BigEndian.PutUint64(target[0:8], prefix) - globalRandRead(target[8:]) - return lookupInfo{target: target, topic: r.topic, radiusLookup: false} -} - -func (r *topicRadius) adjustWithTicket(now mclock.AbsTime, targetHash common.Hash, t ticketRef) { - wait := t.t.regTime[t.idx] - t.t.issueTime - inside := float64(wait)/float64(targetWaitTime) - 0.5 - if inside > 1 { - inside = 1 - } - if inside < 0 { - inside = 0 - } - r.adjust(now, targetHash, t.t.node.sha, inside) -} - -func (r *topicRadius) adjust(now mclock.AbsTime, targetHash, addrHash common.Hash, inside float64) { - bucket := r.getBucketIdx(addrHash) - //fmt.Println("adjust", bucket, len(r.buckets), inside) - if bucket >= len(r.buckets) { - return - } - r.buckets[bucket].adjust(now, inside) - delete(r.buckets[bucket].lookupSent, targetHash) -} diff --git a/p2p/discv5/topic.go b/p2p/discv5/topic.go deleted file mode 100644 index 609a41297f..0000000000 --- a/p2p/discv5/topic.go +++ /dev/null @@ -1,407 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "container/heap" - "fmt" - "math" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/log" -) - -const ( - maxEntries = 10000 - maxEntriesPerTopic = 50 - - fallbackRegistrationExpiry = 1 * time.Hour -) - -type Topic string - -type topicEntry struct { - topic Topic - fifoIdx uint64 - node *Node - expire mclock.AbsTime -} - -type topicInfo struct { - entries map[uint64]*topicEntry - fifoHead, fifoTail uint64 - rqItem *topicRequestQueueItem - wcl waitControlLoop -} - -// removes tail element from the fifo -func (t *topicInfo) getFifoTail() *topicEntry { - for t.entries[t.fifoTail] == nil { - t.fifoTail++ - } - tail := t.entries[t.fifoTail] - t.fifoTail++ - return tail -} - -type nodeInfo struct { - entries map[Topic]*topicEntry - lastIssuedTicket, lastUsedTicket uint32 - // you can't register a ticket newer than lastUsedTicket before noRegUntil (absolute time) - noRegUntil mclock.AbsTime -} - -type topicTable struct { - db *nodeDB - self *Node - nodes map[*Node]*nodeInfo - topics map[Topic]*topicInfo - globalEntries uint64 - requested topicRequestQueue - requestCnt uint64 - lastGarbageCollection mclock.AbsTime -} - -func newTopicTable(db *nodeDB, self *Node) *topicTable { - if printTestImgLogs { - fmt.Printf("*N %016x\n", self.sha[:8]) - } - return &topicTable{ - db: db, - nodes: make(map[*Node]*nodeInfo), - topics: make(map[Topic]*topicInfo), - self: self, - } -} - -func (t *topicTable) getOrNewTopic(topic Topic) *topicInfo { - ti := t.topics[topic] - if ti == nil { - rqItem := &topicRequestQueueItem{ - topic: topic, - priority: t.requestCnt, - } - ti = &topicInfo{ - entries: make(map[uint64]*topicEntry), - rqItem: rqItem, - } - t.topics[topic] = ti - heap.Push(&t.requested, rqItem) - } - return ti -} - -func (t *topicTable) checkDeleteTopic(topic Topic) { - ti := t.topics[topic] - if ti == nil { - return - } - if len(ti.entries) == 0 && ti.wcl.hasMinimumWaitPeriod() { - delete(t.topics, topic) - heap.Remove(&t.requested, ti.rqItem.index) - } -} - -func (t *topicTable) getOrNewNode(node *Node) *nodeInfo { - n := t.nodes[node] - if n == nil { - //fmt.Printf("newNode %016x %016x\n", t.self.sha[:8], node.sha[:8]) - var issued, used uint32 - if t.db != nil { - issued, used = t.db.fetchTopicRegTickets(node.ID) - } - n = &nodeInfo{ - entries: make(map[Topic]*topicEntry), - lastIssuedTicket: issued, - lastUsedTicket: used, - } - t.nodes[node] = n - } - return n -} - -func (t *topicTable) checkDeleteNode(node *Node) { - if n, ok := t.nodes[node]; ok && len(n.entries) == 0 && n.noRegUntil < mclock.Now() { - //fmt.Printf("deleteNode %016x %016x\n", t.self.sha[:8], node.sha[:8]) - delete(t.nodes, node) - } -} - -func (t *topicTable) storeTicketCounters(node *Node) { - n := t.getOrNewNode(node) - if t.db != nil { - t.db.updateTopicRegTickets(node.ID, n.lastIssuedTicket, n.lastUsedTicket) - } -} - -func (t *topicTable) getEntries(topic Topic) []*Node { - t.collectGarbage() - - te := t.topics[topic] - if te == nil { - return nil - } - nodes := make([]*Node, len(te.entries)) - i := 0 - for _, e := range te.entries { - nodes[i] = e.node - i++ - } - t.requestCnt++ - t.requested.update(te.rqItem, t.requestCnt) - return nodes -} - -func (t *topicTable) addEntry(node *Node, topic Topic) { - n := t.getOrNewNode(node) - // clear previous entries by the same node - for _, e := range n.entries { - t.deleteEntry(e) - } - // *** - n = t.getOrNewNode(node) - - tm := mclock.Now() - te := t.getOrNewTopic(topic) - - if len(te.entries) == maxEntriesPerTopic { - t.deleteEntry(te.getFifoTail()) - } - - if t.globalEntries == maxEntries { - t.deleteEntry(t.leastRequested()) // not empty, no need to check for nil - } - - fifoIdx := te.fifoHead - te.fifoHead++ - entry := &topicEntry{ - topic: topic, - fifoIdx: fifoIdx, - node: node, - expire: tm + mclock.AbsTime(fallbackRegistrationExpiry), - } - if printTestImgLogs { - fmt.Printf("*+ %d %v %016x %016x\n", tm/1000000, topic, t.self.sha[:8], node.sha[:8]) - } - te.entries[fifoIdx] = entry - n.entries[topic] = entry - t.globalEntries++ - te.wcl.registered(tm) -} - -// removes least requested element from the fifo -func (t *topicTable) leastRequested() *topicEntry { - for t.requested.Len() > 0 && t.topics[t.requested[0].topic] == nil { - heap.Pop(&t.requested) - } - if t.requested.Len() == 0 { - return nil - } - return t.topics[t.requested[0].topic].getFifoTail() -} - -// entry should exist -func (t *topicTable) deleteEntry(e *topicEntry) { - if printTestImgLogs { - fmt.Printf("*- %d %v %016x %016x\n", mclock.Now()/1000000, e.topic, t.self.sha[:8], e.node.sha[:8]) - } - ne := t.nodes[e.node].entries - delete(ne, e.topic) - if len(ne) == 0 { - t.checkDeleteNode(e.node) - } - te := t.topics[e.topic] - delete(te.entries, e.fifoIdx) - if len(te.entries) == 0 { - t.checkDeleteTopic(e.topic) - } - t.globalEntries-- -} - -// It is assumed that topics and waitPeriods have the same length. -func (t *topicTable) useTicket(node *Node, serialNo uint32, topics []Topic, idx int, issueTime uint64, waitPeriods []uint32) (registered bool) { - log.Trace("Using discovery ticket", "serial", serialNo, "topics", topics, "waits", waitPeriods) - //fmt.Println("useTicket", serialNo, topics, waitPeriods) - t.collectGarbage() - - n := t.getOrNewNode(node) - if serialNo < n.lastUsedTicket { - return false - } - - tm := mclock.Now() - if serialNo > n.lastUsedTicket && tm < n.noRegUntil { - return false - } - if serialNo != n.lastUsedTicket { - n.lastUsedTicket = serialNo - n.noRegUntil = tm + mclock.AbsTime(noRegTimeout()) - t.storeTicketCounters(node) - } - - currTime := uint64(tm / mclock.AbsTime(time.Second)) - regTime := issueTime + uint64(waitPeriods[idx]) - relTime := int64(currTime - regTime) - if relTime >= -1 && relTime <= regTimeWindow+1 { // give clients a little security margin on both ends - if e := n.entries[topics[idx]]; e == nil { - t.addEntry(node, topics[idx]) - } else { - // if there is an active entry, don't move to the front of the FIFO but prolong expire time - e.expire = tm + mclock.AbsTime(fallbackRegistrationExpiry) - } - return true - } - - return false -} - -func (t *topicTable) getTicket(node *Node, topics []Topic) *ticket { - t.collectGarbage() - - now := mclock.Now() - n := t.getOrNewNode(node) - n.lastIssuedTicket++ - t.storeTicketCounters(node) - - tic := &ticket{ - issueTime: now, - topics: topics, - serial: n.lastIssuedTicket, - regTime: make([]mclock.AbsTime, len(topics)), - } - for i, topic := range topics { - var waitPeriod time.Duration - if topic := t.topics[topic]; topic != nil { - waitPeriod = topic.wcl.waitPeriod - } else { - waitPeriod = minWaitPeriod - } - - tic.regTime[i] = now + mclock.AbsTime(waitPeriod) - } - return tic -} - -const gcInterval = time.Minute - -func (t *topicTable) collectGarbage() { - tm := mclock.Now() - if time.Duration(tm-t.lastGarbageCollection) < gcInterval { - return - } - t.lastGarbageCollection = tm - - for node, n := range t.nodes { - for _, e := range n.entries { - if e.expire <= tm { - t.deleteEntry(e) - } - } - - t.checkDeleteNode(node) - } - - for topic := range t.topics { - t.checkDeleteTopic(topic) - } -} - -const ( - minWaitPeriod = time.Minute - regTimeWindow = 10 // seconds - avgnoRegTimeout = time.Minute * 10 - // target average interval between two incoming ad requests - wcTargetRegInterval = time.Minute * 10 / maxEntriesPerTopic - // - wcTimeConst = time.Minute * 10 -) - -// initialization is not required, will set to minWaitPeriod at first registration -type waitControlLoop struct { - lastIncoming mclock.AbsTime - waitPeriod time.Duration -} - -func (w *waitControlLoop) registered(tm mclock.AbsTime) { - w.waitPeriod = w.nextWaitPeriod(tm) - w.lastIncoming = tm -} - -func (w *waitControlLoop) nextWaitPeriod(tm mclock.AbsTime) time.Duration { - period := tm - w.lastIncoming - wp := time.Duration(float64(w.waitPeriod) * math.Exp((float64(wcTargetRegInterval)-float64(period))/float64(wcTimeConst))) - if wp < minWaitPeriod { - wp = minWaitPeriod - } - return wp -} - -func (w *waitControlLoop) hasMinimumWaitPeriod() bool { - return w.nextWaitPeriod(mclock.Now()) == minWaitPeriod -} - -func noRegTimeout() time.Duration { - e := rand.ExpFloat64() - if e > 100 { - e = 100 - } - return time.Duration(float64(avgnoRegTimeout) * e) -} - -type topicRequestQueueItem struct { - topic Topic - priority uint64 - index int -} - -// A topicRequestQueue implements heap.Interface and holds topicRequestQueueItems. -type topicRequestQueue []*topicRequestQueueItem - -func (tq topicRequestQueue) Len() int { return len(tq) } - -func (tq topicRequestQueue) Less(i, j int) bool { - return tq[i].priority < tq[j].priority -} - -func (tq topicRequestQueue) Swap(i, j int) { - tq[i], tq[j] = tq[j], tq[i] - tq[i].index = i - tq[j].index = j -} - -func (tq *topicRequestQueue) Push(x interface{}) { - n := len(*tq) - item := x.(*topicRequestQueueItem) - item.index = n - *tq = append(*tq, item) -} - -func (tq *topicRequestQueue) Pop() interface{} { - old := *tq - n := len(old) - item := old[n-1] - item.index = -1 - *tq = old[0 : n-1] - return item -} - -func (tq *topicRequestQueue) update(item *topicRequestQueueItem, priority uint64) { - item.priority = priority - heap.Fix(tq, item.index) -} diff --git a/p2p/discv5/topic_test.go b/p2p/discv5/topic_test.go deleted file mode 100644 index ba79993f29..0000000000 --- a/p2p/discv5/topic_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "encoding/binary" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/mclock" -) - -func TestTopicRadius(t *testing.T) { - now := mclock.Now() - topic := Topic("qwerty") - rad := newTopicRadius(topic) - targetRad := (^uint64(0)) / 100 - - waitFn := func(addr common.Hash) time.Duration { - prefix := binary.BigEndian.Uint64(addr[0:8]) - dist := prefix ^ rad.topicHashPrefix - relDist := float64(dist) / float64(targetRad) - relTime := (1 - relDist/2) * 2 - if relTime < 0 { - relTime = 0 - } - return time.Duration(float64(targetWaitTime) * relTime) - } - - bcnt := 0 - cnt := 0 - var sum float64 - for cnt < 100 { - addr := rad.nextTarget(false).target - wait := waitFn(addr) - ticket := &ticket{ - topics: []Topic{topic}, - regTime: []mclock.AbsTime{mclock.AbsTime(wait)}, - node: &Node{nodeNetGuts: nodeNetGuts{sha: addr}}, - } - rad.adjustWithTicket(now, addr, ticketRef{ticket, 0}) - if rad.radius != maxRadius { - cnt++ - sum += float64(rad.radius) - } else { - bcnt++ - if bcnt > 500 { - t.Errorf("Radius did not converge in 500 iterations") - } - } - } - avgRel := sum / float64(cnt) / float64(targetRad) - if avgRel > 1.05 || avgRel < 0.95 { - t.Errorf("Average/target ratio is too far from 1 (%v)", avgRel) - } -} diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go deleted file mode 100644 index 088f95cac6..0000000000 --- a/p2p/discv5/udp.go +++ /dev/null @@ -1,429 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package discv5 - -import ( - "bytes" - "crypto/ecdsa" - "errors" - "fmt" - "net" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/netutil" - "github.com/ethereum/go-ethereum/rlp" -) - -const Version = 4 - -// Errors -var ( - errPacketTooSmall = errors.New("too small") - errBadPrefix = errors.New("bad prefix") -) - -// Timeouts -const ( - respTimeout = 500 * time.Millisecond - expiration = 20 * time.Second -) - -// RPC request structures -type ( - ping struct { - Version uint - From, To rpcEndpoint - Expiration uint64 - - // v5 - Topics []Topic - - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` - } - - // pong is the reply to ping. - pong struct { - // This field should mirror the UDP envelope address - // of the ping packet, which provides a way to discover the - // the external address (after NAT). - To rpcEndpoint - - ReplyTok []byte // This contains the hash of the ping packet. - Expiration uint64 // Absolute timestamp at which the packet becomes invalid. - - // v5 - TopicHash common.Hash - TicketSerial uint32 - WaitPeriods []uint32 - - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` - } - - // findnode is a query for nodes close to the given target. - findnode struct { - Target NodeID // doesn't need to be an actual public key - Expiration uint64 - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` - } - - // findnode is a query for nodes close to the given target. - findnodeHash struct { - Target common.Hash - Expiration uint64 - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` - } - - // reply to findnode - neighbors struct { - Nodes []rpcNode - Expiration uint64 - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` - } - - topicRegister struct { - Topics []Topic - Idx uint - Pong []byte - } - - topicQuery struct { - Topic Topic - Expiration uint64 - } - - // reply to topicQuery - topicNodes struct { - Echo common.Hash - Nodes []rpcNode - } - - rpcNode struct { - IP net.IP // len 4 for IPv4 or 16 for IPv6 - UDP uint16 // for discovery protocol - TCP uint16 // for RLPx protocol - ID NodeID - } - - rpcEndpoint struct { - IP net.IP // len 4 for IPv4 or 16 for IPv6 - UDP uint16 // for discovery protocol - TCP uint16 // for RLPx protocol - } -) - -var ( - versionPrefix = []byte("temporary discovery v5") - versionPrefixSize = len(versionPrefix) - sigSize = 520 / 8 - headSize = versionPrefixSize + sigSize // space of packet frame data -) - -// Neighbors replies are sent across multiple packets to -// stay below the 1280 byte limit. We compute the maximum number -// of entries by stuffing a packet until it grows too large. -var maxNeighbors = func() int { - p := neighbors{Expiration: ^uint64(0)} - maxSizeNode := rpcNode{IP: make(net.IP, 16), UDP: ^uint16(0), TCP: ^uint16(0)} - for n := 0; ; n++ { - p.Nodes = append(p.Nodes, maxSizeNode) - size, _, err := rlp.EncodeToReader(p) - if err != nil { - // If this ever happens, it will be caught by the unit tests. - panic("cannot encode: " + err.Error()) - } - if headSize+size+1 >= 1280 { - return n - } - } -}() - -var maxTopicNodes = func() int { - p := topicNodes{} - maxSizeNode := rpcNode{IP: make(net.IP, 16), UDP: ^uint16(0), TCP: ^uint16(0)} - for n := 0; ; n++ { - p.Nodes = append(p.Nodes, maxSizeNode) - size, _, err := rlp.EncodeToReader(p) - if err != nil { - // If this ever happens, it will be caught by the unit tests. - panic("cannot encode: " + err.Error()) - } - if headSize+size+1 >= 1280 { - return n - } - } -}() - -func makeEndpoint(addr *net.UDPAddr, tcpPort uint16) rpcEndpoint { - ip := addr.IP.To4() - if ip == nil { - ip = addr.IP.To16() - } - return rpcEndpoint{IP: ip, UDP: uint16(addr.Port), TCP: tcpPort} -} - -func nodeFromRPC(sender *net.UDPAddr, rn rpcNode) (*Node, error) { - if err := netutil.CheckRelayIP(sender.IP, rn.IP); err != nil { - return nil, err - } - n := NewNode(rn.ID, rn.IP, rn.UDP, rn.TCP) - err := n.validateComplete() - return n, err -} - -func nodeToRPC(n *Node) rpcNode { - return rpcNode{ID: n.ID, IP: n.IP, UDP: n.UDP, TCP: n.TCP} -} - -type ingressPacket struct { - remoteID NodeID - remoteAddr *net.UDPAddr - ev nodeEvent - hash []byte - data interface{} // one of the RPC structs - rawData []byte -} - -type conn interface { - ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) - WriteToUDP(b []byte, addr *net.UDPAddr) (n int, err error) - Close() error - LocalAddr() net.Addr -} - -// udp implements the RPC protocol. -type udp struct { - conn conn - priv *ecdsa.PrivateKey - ourEndpoint rpcEndpoint - net *Network -} - -// ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, conn conn, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { - realaddr := conn.LocalAddr().(*net.UDPAddr) - transport, err := listenUDP(priv, conn, realaddr) - if err != nil { - return nil, err - } - net, err := newNetwork(transport, priv.PublicKey, nodeDBPath, netrestrict) - if err != nil { - return nil, err - } - log.Info("UDP listener up", "net", net.tab.self) - transport.net = net - go transport.readLoop() - return net, nil -} - -func listenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr) (*udp, error) { - return &udp{conn: conn, priv: priv, ourEndpoint: makeEndpoint(realaddr, uint16(realaddr.Port))}, nil -} - -func (t *udp) localAddr() *net.UDPAddr { - return t.conn.LocalAddr().(*net.UDPAddr) -} - -func (t *udp) Close() { - t.conn.Close() -} - -func (t *udp) send(remote *Node, ptype nodeEvent, data interface{}) (hash []byte) { - hash, _ = t.sendPacket(remote.ID, remote.addr(), byte(ptype), data) - return hash -} - -func (t *udp) sendPing(remote *Node, toaddr *net.UDPAddr, topics []Topic) (hash []byte) { - hash, _ = t.sendPacket(remote.ID, toaddr, byte(pingPacket), ping{ - Version: Version, - From: t.ourEndpoint, - To: makeEndpoint(toaddr, uint16(toaddr.Port)), // TODO: maybe use known TCP port from DB - Expiration: uint64(time.Now().Add(expiration).Unix()), - Topics: topics, - }) - return hash -} - -func (t *udp) sendNeighbours(remote *Node, results []*Node) { - // Send neighbors in chunks with at most maxNeighbors per packet - // to stay below the 1280 byte limit. - p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())} - for i, result := range results { - p.Nodes = append(p.Nodes, nodeToRPC(result)) - if len(p.Nodes) == maxNeighbors || i == len(results)-1 { - t.sendPacket(remote.ID, remote.addr(), byte(neighborsPacket), p) - p.Nodes = p.Nodes[:0] - } - } -} - -func (t *udp) sendFindnodeHash(remote *Node, target common.Hash) { - t.sendPacket(remote.ID, remote.addr(), byte(findnodeHashPacket), findnodeHash{ - Target: target, - Expiration: uint64(time.Now().Add(expiration).Unix()), - }) -} - -func (t *udp) sendTopicRegister(remote *Node, topics []Topic, idx int, pong []byte) { - t.sendPacket(remote.ID, remote.addr(), byte(topicRegisterPacket), topicRegister{ - Topics: topics, - Idx: uint(idx), - Pong: pong, - }) -} - -func (t *udp) sendTopicNodes(remote *Node, queryHash common.Hash, nodes []*Node) { - p := topicNodes{Echo: queryHash} - var sent bool - for _, result := range nodes { - if result.IP.Equal(t.net.tab.self.IP) || netutil.CheckRelayIP(remote.IP, result.IP) == nil { - p.Nodes = append(p.Nodes, nodeToRPC(result)) - } - if len(p.Nodes) == maxTopicNodes { - t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p) - p.Nodes = p.Nodes[:0] - sent = true - } - } - if !sent || len(p.Nodes) > 0 { - t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p) - } -} - -func (t *udp) sendPacket(toid NodeID, toaddr *net.UDPAddr, ptype byte, req interface{}) (hash []byte, err error) { - //fmt.Println("sendPacket", nodeEvent(ptype), toaddr.String(), toid.String()) - packet, hash, err := encodePacket(t.priv, ptype, req) - if err != nil { - //fmt.Println(err) - return hash, err - } - log.Trace(fmt.Sprintf(">>> %v to %x@%v", nodeEvent(ptype), toid[:8], toaddr)) - if nbytes, err := t.conn.WriteToUDP(packet, toaddr); err != nil { - log.Trace(fmt.Sprint("UDP send failed:", err)) - } else { - egressTrafficMeter.Mark(int64(nbytes)) - } - //fmt.Println(err) - return hash, err -} - -// zeroed padding space for encodePacket. -var headSpace = make([]byte, headSize) - -func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (p, hash []byte, err error) { - b := new(bytes.Buffer) - b.Write(headSpace) - b.WriteByte(ptype) - if err := rlp.Encode(b, req); err != nil { - log.Error(fmt.Sprint("error encoding packet:", err)) - return nil, nil, err - } - packet := b.Bytes() - sig, err := crypto.Sign(crypto.Keccak256(packet[headSize:]), priv) - if err != nil { - log.Error(fmt.Sprint("could not sign packet:", err)) - return nil, nil, err - } - copy(packet, versionPrefix) - copy(packet[versionPrefixSize:], sig) - hash = crypto.Keccak256(packet[versionPrefixSize:]) - return packet, hash, nil -} - -// readLoop runs in its own goroutine. it injects ingress UDP packets -// into the network loop. -func (t *udp) readLoop() { - defer t.conn.Close() - // Discovery packets are defined to be no larger than 1280 bytes. - // Packets larger than this size will be cut at the end and treated - // as invalid because their hash won't match. - buf := make([]byte, 1280) - for { - nbytes, from, err := t.conn.ReadFromUDP(buf) - ingressTrafficMeter.Mark(int64(nbytes)) - if netutil.IsTemporaryError(err) { - // Ignore temporary read errors. - log.Debug(fmt.Sprintf("Temporary read error: %v", err)) - continue - } else if err != nil { - // Shut down the loop for permament errors. - log.Debug(fmt.Sprintf("Read error: %v", err)) - return - } - t.handlePacket(from, buf[:nbytes]) - } -} - -func (t *udp) handlePacket(from *net.UDPAddr, buf []byte) error { - pkt := ingressPacket{remoteAddr: from} - if err := decodePacket(buf, &pkt); err != nil { - log.Debug(fmt.Sprintf("Bad packet from %v: %v", from, err)) - //fmt.Println("bad packet", err) - return err - } - t.net.reqReadPacket(pkt) - return nil -} - -func decodePacket(buffer []byte, pkt *ingressPacket) error { - if len(buffer) < headSize+1 { - return errPacketTooSmall - } - buf := make([]byte, len(buffer)) - copy(buf, buffer) - prefix, sig, sigdata := buf[:versionPrefixSize], buf[versionPrefixSize:headSize], buf[headSize:] - if !bytes.Equal(prefix, versionPrefix) { - return errBadPrefix - } - fromID, err := recoverNodeID(crypto.Keccak256(buf[headSize:]), sig) - if err != nil { - return err - } - pkt.rawData = buf - pkt.hash = crypto.Keccak256(buf[versionPrefixSize:]) - pkt.remoteID = fromID - switch pkt.ev = nodeEvent(sigdata[0]); pkt.ev { - case pingPacket: - pkt.data = new(ping) - case pongPacket: - pkt.data = new(pong) - case findnodePacket: - pkt.data = new(findnode) - case neighborsPacket: - pkt.data = new(neighbors) - case findnodeHashPacket: - pkt.data = new(findnodeHash) - case topicRegisterPacket: - pkt.data = new(topicRegister) - case topicQueryPacket: - pkt.data = new(topicQuery) - case topicNodesPacket: - pkt.data = new(topicNodes) - default: - return fmt.Errorf("unknown packet type: %d", sigdata[0]) - } - s := rlp.NewStream(bytes.NewReader(sigdata[1:]), 0) - err = s.Decode(pkt.data) - return err -} diff --git a/p2p/server.go b/p2p/server.go index 275cb5ea5c..fc71548554 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -35,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nat" @@ -105,7 +104,7 @@ type Config struct { // BootstrapNodesV5 are used to establish connectivity // with the rest of the network using the V5 discovery // protocol. - BootstrapNodesV5 []*discv5.Node `toml:",omitempty"` + BootstrapNodesV5 []*enode.Node `toml:",omitempty"` // Static nodes are used as pre-configured connections which are always // maintained and re-connected on disconnects. @@ -182,7 +181,7 @@ type Server struct { nodedb *enode.DB localnode *enode.LocalNode ntab *discover.UDPv4 - DiscV5 *discv5.Network + DiscV5 *discover.UDPv5 discmix *enode.FairMix dialsched *dialScheduler @@ -413,7 +412,7 @@ type sharedUDPConn struct { unhandled chan discover.ReadPacket } -// ReadFromUDP implements discv5.conn +// ReadFromUDP implements discover.UDPConn func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { packet, ok := <-s.unhandled if !ok { @@ -427,7 +426,7 @@ func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err err return l, packet.Addr, nil } -// Close implements discv5.conn +// Close implements discover.UDPConn func (s *sharedUDPConn) Close() error { return nil } @@ -586,7 +585,7 @@ func (srv *Server) setupDiscovery() error { Unhandled: unhandled, Log: srv.log, } - ntab, err := discover.ListenUDP(conn, srv.localnode, cfg) + ntab, err := discover.ListenV4(conn, srv.localnode, cfg) if err != nil { return err } @@ -596,20 +595,21 @@ func (srv *Server) setupDiscovery() error { // Discovery V5 if srv.DiscoveryV5 { - var ntab *discv5.Network + cfg := discover.Config{ + PrivateKey: srv.PrivateKey, + NetRestrict: srv.NetRestrict, + Bootnodes: srv.BootstrapNodesV5, + Log: srv.log, + } var err error if sconn != nil { - ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, "", srv.NetRestrict) + srv.DiscV5, err = discover.ListenV5(sconn, srv.localnode, cfg) } else { - ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, "", srv.NetRestrict) + srv.DiscV5, err = discover.ListenV5(conn, srv.localnode, cfg) } if err != nil { return err } - if err := ntab.SetFallbackNodes(srv.BootstrapNodesV5); err != nil { - return err - } - srv.DiscV5 = ntab } return nil } diff --git a/params/bootnodes.go b/params/bootnodes.go index d4512bf789..66f917015e 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -73,6 +73,24 @@ var YoloV2Bootnodes = []string{ "enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303", } +var V5Bootnodes = []string{ + // Teku team's bootnode + "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", + "enr:-KG4QDyytgmE4f7AnvW-ZaUOIi9i79qX4JwjRAiXBZCU65wOfBu-3Nb5I7b_Rmg3KCOcZM_C3y5pg7EBU5XGrcLTduQEhGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQ2_DUbiXNlY3AyNTZrMaEDKnz_-ps3UUOfHWVYaskI5kWYO_vtYMGYCQRAR3gHDouDdGNwgiMog3VkcIIjKA", + // Prylab team's bootnodes + "enr:-Ku4QImhMc1z8yCiNJ1TyUxdcfNucje3BGwEHzodEZUan8PherEo4sF7pPHPSIB1NNuSg5fZy7qFsjmUKs2ea1Whi0EBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQOVphkDqal4QzPMksc5wnpuC3gvSC8AfbFOnZY_On34wIN1ZHCCIyg", + "enr:-Ku4QP2xDnEtUXIjzJ_DhlCRN9SN99RYQPJL92TMlSv7U5C1YnYLjwOQHgZIUXw6c-BvRg2Yc2QsZxxoS_pPRVe0yK8Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMeFF5GrS7UZpAH2Ly84aLK-TyvH-dRo0JM1i8yygH50YN1ZHCCJxA", + "enr:-Ku4QPp9z1W4tAO8Ber_NQierYaOStqhDqQdOPY3bB3jDgkjcbk6YrEnVYIiCBbTxuar3CzS528d2iE7TdJsrL-dEKoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMw5fqqkw2hHC4F5HZZDPsNmPdB1Gi8JPQK7pRc9XHh-oN1ZHCCKvg", + // Lighthouse team's bootnodes + "enr:-IS4QLkKqDMy_ExrpOEWa59NiClemOnor-krjp4qoeZwIw2QduPC-q7Kz4u1IOWf3DDbdxqQIgC4fejavBOuUPy-HE4BgmlkgnY0gmlwhCLzAHqJc2VjcDI1NmsxoQLQSJfEAHZApkm5edTCZ_4qps_1k_ub2CxHFxi-gr2JMIN1ZHCCIyg", + "enr:-IS4QDAyibHCzYZmIYZCjXwU9BqpotWmv2BsFlIq1V31BwDDMJPFEbox1ijT5c2Ou3kvieOKejxuaCqIcjxBjJ_3j_cBgmlkgnY0gmlwhAMaHiCJc2VjcDI1NmsxoQJIdpj_foZ02MXz4It8xKD7yUHTBx7lVFn3oeRP21KRV4N1ZHCCIyg", + // EF bootnodes + "enr:-Ku4QHqVeJ8PPICcWk1vSn_XcSkjOkNiTg6Fmii5j6vUQgvzMc9L1goFnLKgXqBJspJjIsB91LTOleFmyWWrFVATGngBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAMRHkWJc2VjcDI1NmsxoQKLVXFOhp2uX6jeT0DvvDpPcU8FWMjQdR4wMuORMhpX24N1ZHCCIyg", + "enr:-Ku4QG-2_Md3sZIAUebGYT6g0SMskIml77l6yR-M_JXc-UdNHCmHQeOiMLbylPejyJsdAPsTHJyjJB2sYGDLe0dn8uYBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhBLY-NyJc2VjcDI1NmsxoQORcM6e19T1T9gi7jxEZjk_sjVLGFscUNqAY9obgZaxbIN1ZHCCIyg", + "enr:-Ku4QPn5eVhcoF1opaFEvg1b6JNFD2rqVkHQ8HApOKK61OIcIXD127bKWgAtbwI7pnxx6cDyk_nI88TrZKQaGMZj0q0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhDayLMaJc2VjcDI1NmsxoQK2sBOLGcUb4AwuYzFuAVCaNHA-dy24UuEKkeFNgCVCsIN1ZHCCIyg", + "enr:-Ku4QEWzdnVtXc2Q0ZVigfCGggOVB2Vc1ZCPEc6j21NIFLODSJbvNaef1g4PxhPwl_3kax86YPheFUSLXPRs98vvYsoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhDZBrP2Jc2VjcDI1NmsxoQM6jr8Rb1ktLEsVcKAPa08wCsKUmvoQ8khiOl_SLozf9IN1ZHCCIyg", +} + const dnsPrefix = "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@" // KnownDNSNetwork returns the address of a public DNS-based node list for the given From 2e5d14170846ae72adc47467a1129e41d6800349 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Wed, 27 Jan 2021 09:20:34 +0000 Subject: [PATCH 104/709] rpc: deprecate Client.ShhSubscribe (#22239) It never worked, whisper uses polling. Co-authored-by: Felix Lange --- rpc/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/client.go b/rpc/client.go index 9393adb779..198ce63573 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -414,6 +414,7 @@ func (c *Client) EthSubscribe(ctx context.Context, channel interface{}, args ... } // ShhSubscribe registers a subscripion under the "shh" namespace. +// Deprecated: use Subscribe(ctx, "shh", ...). func (c *Client) ShhSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (*ClientSubscription, error) { return c.Subscribe(ctx, "shh", channel, args...) } From eb21c652c0a9d8f651efc0251cc5797a3328d863 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Thu, 28 Jan 2021 21:19:07 +0100 Subject: [PATCH 105/709] cmd,core,eth,params,tests: define yolov3 + enable EIP-2565 (#22213) Removes the yolov2 definition, adds yolov3, including EIP-2565. This PR also disables some of the erroneously generated blockchain and statetests, and adds the new genesis hash + alloc for yolov3. This PR disables the CLI switches for yolo, since it's not complete until we merge support for 2930. --- cmd/evm/internal/t8ntool/execution.go | 2 +- cmd/geth/chaincmd.go | 4 +-- cmd/geth/consolecmd.go | 4 +-- cmd/geth/main.go | 4 ++- cmd/geth/usage.go | 3 ++- cmd/puppeth/wizard_genesis.go | 4 +-- cmd/utils/flags.go | 28 ++++++++++----------- core/genesis.go | 14 +++++------ core/genesis_alloc.go | 3 ++- core/state_processor.go | 2 +- core/vm/contracts.go | 35 +++++++++++++++------------ core/vm/evm.go | 10 ++++---- core/vm/interpreter.go | 4 +-- core/vm/jump_table.go | 6 ++--- core/vm/runtime/runtime.go | 8 +++--- eth/tracers/api.go | 4 +-- params/bootnodes.go | 7 +++--- params/config.go | 31 ++++++++++++------------ tests/block_test.go | 7 ++++++ tests/fuzzers/bls12381/bls_fuzzer.go | 2 +- tests/init.go | 8 +++--- tests/state_test.go | 8 ++++++ tests/state_test_util.go | 2 +- 23 files changed, 112 insertions(+), 88 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 171443e156..95e6de37cb 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -145,7 +145,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txContext := core.NewEVMTxContext(msg) evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) - if chainConfig.IsYoloV2(vmContext.BlockNumber) { + if chainConfig.IsYoloV3(vmContext.BlockNumber) { statedb.AddAddressToAccessList(msg.From()) if dst := msg.To(); dst != nil { statedb.AddAddressToAccessList(*dst) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index f539322654..9eec30f813 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -164,7 +164,7 @@ The export-preimages command export hash preimages to an RLP encoded stream`, utils.RinkebyFlag, utils.TxLookupLimitFlag, utils.GoerliFlag, - utils.YoloV2Flag, + utils.YoloV3Flag, utils.LegacyTestnetFlag, }, Category: "BLOCKCHAIN COMMANDS", @@ -215,7 +215,7 @@ Use "ethereum dump 0" to dump the genesis block.`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV2Flag, + utils.YoloV3Flag, utils.LegacyTestnetFlag, utils.SyncModeFlag, }, diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 7822c73b31..5369612e87 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -136,8 +136,8 @@ func remoteConsole(ctx *cli.Context) error { path = filepath.Join(path, "rinkeby") } else if ctx.GlobalBool(utils.GoerliFlag.Name) { path = filepath.Join(path, "goerli") - } else if ctx.GlobalBool(utils.YoloV2Flag.Name) { - path = filepath.Join(path, "yolo-v2") + } else if ctx.GlobalBool(utils.YoloV3Flag.Name) { + path = filepath.Join(path, "yolo-v3") } } endpoint = fmt.Sprintf("%s/geth.ipc", path) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e577ab370d..86dc6f40fe 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -148,7 +148,9 @@ var ( utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV2Flag, + // YOLOv3 is not yet complete! + // TODO: enable this once 2718/2930 is added + //utils.YoloV3Flag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 25accc9b79..ba311bf7f6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -44,7 +44,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MainnetFlag, utils.GoerliFlag, utils.RinkebyFlag, - utils.YoloV2Flag, + // TODO: Re-enable this when 2718/2930 is added + //utils.YoloV3Flag, utils.RopstenFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 2d014e83bc..52093975cb 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -236,8 +236,8 @@ func (w *wizard) manageGenesis() { w.conf.Genesis.Config.IstanbulBlock = w.readDefaultBigInt(w.conf.Genesis.Config.IstanbulBlock) fmt.Println() - fmt.Printf("Which block should YOLOv2 come into effect? (default = %v)\n", w.conf.Genesis.Config.YoloV2Block) - w.conf.Genesis.Config.YoloV2Block = w.readDefaultBigInt(w.conf.Genesis.Config.YoloV2Block) + fmt.Printf("Which block should YOLOv3 come into effect? (default = %v)\n", w.conf.Genesis.Config.YoloV3Block) + w.conf.Genesis.Config.YoloV3Block = w.readDefaultBigInt(w.conf.Genesis.Config.YoloV3Block) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1d6d2d86b6..8f9f68ba68 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -147,9 +147,9 @@ var ( Name: "goerli", Usage: "Görli network: pre-configured proof-of-authority test network", } - YoloV2Flag = cli.BoolFlag{ - Name: "yolov2", - Usage: "YOLOv2 network: pre-configured proof-of-authority shortlived test network.", + YoloV3Flag = cli.BoolFlag{ + Name: "yolov3", + Usage: "YOLOv3 network: pre-configured proof-of-authority shortlived test network.", } RinkebyFlag = cli.BoolFlag{ Name: "rinkeby", @@ -760,8 +760,8 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(GoerliFlag.Name) { return filepath.Join(path, "goerli") } - if ctx.GlobalBool(YoloV2Flag.Name) { - return filepath.Join(path, "yolo-v2") + if ctx.GlobalBool(YoloV3Flag.Name) { + return filepath.Join(path, "yolo-v3") } return path } @@ -819,8 +819,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.RinkebyBootnodes case ctx.GlobalBool(GoerliFlag.Name): urls = params.GoerliBootnodes - case ctx.GlobalBool(YoloV2Flag.Name): - urls = params.YoloV2Bootnodes + case ctx.GlobalBool(YoloV3Flag.Name): + urls = params.YoloV3Bootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -1280,7 +1280,7 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") - case ctx.GlobalBool(YoloV2Flag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.GlobalBool(YoloV3Flag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "yolo-v2") } } @@ -1494,7 +1494,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, MainnetFlag, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV2Flag) + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV3Flag) CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag) @@ -1631,11 +1631,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) - case ctx.GlobalBool(YoloV2Flag.Name): + case ctx.GlobalBool(YoloV3Flag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = 133519467574834 // "yolov2" + cfg.NetworkId = new(big.Int).SetBytes([]byte("yolov3")).Uint64() // "yolov3" } - cfg.Genesis = core.DefaultYoloV2GenesisBlock() + cfg.Genesis = core.DefaultYoloV3GenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1824,8 +1824,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() - case ctx.GlobalBool(YoloV2Flag.Name): - genesis = core.DefaultYoloV2GenesisBlock() + case ctx.GlobalBool(YoloV3Flag.Name): + genesis = core.DefaultYoloV3GenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } diff --git a/core/genesis.go b/core/genesis.go index 908a969afd..f678a3bbca 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -243,8 +243,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return params.RinkebyChainConfig case ghash == params.GoerliGenesisHash: return params.GoerliChainConfig - case ghash == params.YoloV2GenesisHash: - return params.YoloV2ChainConfig + case ghash == params.YoloV3GenesisHash: + return params.YoloV3ChainConfig default: return params.AllEthashProtocolChanges } @@ -380,15 +380,15 @@ func DefaultGoerliGenesisBlock() *Genesis { } } -func DefaultYoloV2GenesisBlock() *Genesis { - // TODO: Update with yolov2 values + regenerate alloc data +func DefaultYoloV3GenesisBlock() *Genesis { + // Full genesis: https://gist.github.com/holiman/b2c32a05ff2e2712e11c0787d987d46f return &Genesis{ - Config: params.YoloV2ChainConfig, - Timestamp: 0x5f91b932, + Config: params.YoloV3ChainConfig, + Timestamp: 0x60117f8b, ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000008a37866fd3627c9205a37c8685666f32ec07bb1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), GasLimit: 0x47b760, Difficulty: big.NewInt(1), - Alloc: decodePrealloc(yoloV1AllocData), + Alloc: decodePrealloc(yoloV3AllocData), } } diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 3e03d16407..6eecbbf0e8 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,4 +25,5 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const yoloV1AllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x8a7\x86o\xd3b|\x92\x05\xa3|\x86\x85fo2\xec\a\xbb\x1b\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" +const yoloV3AllocData = "\xf9\x05\x01\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + diff --git a/core/state_processor.go b/core/state_processor.go index cdc86a694e..c5b0bb1609 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -93,7 +93,7 @@ func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainCon // Create a new context to be used in the EVM environment txContext := NewEVMTxContext(msg) // Add addresses to access list if applicable - if config.IsYoloV2(header.Number) { + if config.IsYoloV3(header.Number) { statedb.AddAddressToAccessList(msg.From()) if dst := msg.To(); dst != nil { statedb.AddAddressToAccessList(*dst) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 0c828b1cd9..9ea19d38a7 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -78,18 +78,23 @@ var PrecompiledContractsIstanbul = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{9}): &blake2F{}, } -// PrecompiledContractsYoloV2 contains the default set of pre-compiled Ethereum -// contracts used in the Yolo v2 test release. -var PrecompiledContractsYoloV2 = map[common.Address]PrecompiledContract{ - common.BytesToAddress([]byte{1}): &ecrecover{}, - common.BytesToAddress([]byte{2}): &sha256hash{}, - common.BytesToAddress([]byte{3}): &ripemd160hash{}, - common.BytesToAddress([]byte{4}): &dataCopy{}, - common.BytesToAddress([]byte{5}): &bigModExp{eip2565: false}, - common.BytesToAddress([]byte{6}): &bn256AddIstanbul{}, - common.BytesToAddress([]byte{7}): &bn256ScalarMulIstanbul{}, - common.BytesToAddress([]byte{8}): &bn256PairingIstanbul{}, - common.BytesToAddress([]byte{9}): &blake2F{}, +// PrecompiledContractsYoloV3 contains the default set of pre-compiled Ethereum +// contracts used in the Yolo v3 test release. +var PrecompiledContractsYoloV3 = map[common.Address]PrecompiledContract{ + common.BytesToAddress([]byte{1}): &ecrecover{}, + common.BytesToAddress([]byte{2}): &sha256hash{}, + common.BytesToAddress([]byte{3}): &ripemd160hash{}, + common.BytesToAddress([]byte{4}): &dataCopy{}, + common.BytesToAddress([]byte{5}): &bigModExp{eip2565: true}, + common.BytesToAddress([]byte{6}): &bn256AddIstanbul{}, + common.BytesToAddress([]byte{7}): &bn256ScalarMulIstanbul{}, + common.BytesToAddress([]byte{8}): &bn256PairingIstanbul{}, + common.BytesToAddress([]byte{9}): &blake2F{}, +} + +// PrecompiledContractsBLS contains the set of pre-compiled Ethereum +// contracts specified in EIP-2537. These are exported for testing purposes. +var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{10}): &bls12381G1Add{}, common.BytesToAddress([]byte{11}): &bls12381G1Mul{}, common.BytesToAddress([]byte{12}): &bls12381G1MultiExp{}, @@ -102,7 +107,7 @@ var PrecompiledContractsYoloV2 = map[common.Address]PrecompiledContract{ } var ( - PrecompiledAddressesYoloV2 []common.Address + PrecompiledAddressesYoloV3 []common.Address PrecompiledAddressesIstanbul []common.Address PrecompiledAddressesByzantium []common.Address PrecompiledAddressesHomestead []common.Address @@ -118,8 +123,8 @@ func init() { for k := range PrecompiledContractsIstanbul { PrecompiledAddressesIstanbul = append(PrecompiledAddressesIstanbul, k) } - for k := range PrecompiledContractsYoloV2 { - PrecompiledAddressesYoloV2 = append(PrecompiledAddressesYoloV2, k) + for k := range PrecompiledContractsYoloV3 { + PrecompiledAddressesYoloV3 = append(PrecompiledAddressesYoloV3, k) } } diff --git a/core/vm/evm.go b/core/vm/evm.go index 9afef76643..3617d77b12 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -46,8 +46,8 @@ type ( // configuration func (evm *EVM) ActivePrecompiles() []common.Address { switch { - case evm.chainRules.IsYoloV2: - return PrecompiledAddressesYoloV2 + case evm.chainRules.IsYoloV3: + return PrecompiledAddressesYoloV3 case evm.chainRules.IsIstanbul: return PrecompiledAddressesIstanbul case evm.chainRules.IsByzantium: @@ -60,8 +60,8 @@ func (evm *EVM) ActivePrecompiles() []common.Address { func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) { var precompiles map[common.Address]PrecompiledContract switch { - case evm.chainRules.IsYoloV2: - precompiles = PrecompiledContractsYoloV2 + case evm.chainRules.IsYoloV3: + precompiles = PrecompiledContractsYoloV3 case evm.chainRules.IsIstanbul: precompiles = PrecompiledContractsIstanbul case evm.chainRules.IsByzantium: @@ -446,7 +446,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, evm.StateDB.SetNonce(caller.Address(), nonce+1) // We add this to the access list _before_ taking a snapshot. Even if the creation fails, // the access-list change should not be rolled back - if evm.chainRules.IsYoloV2 { + if evm.chainRules.IsYoloV3 { evm.StateDB.AddAddressToAccessList(address) } // Ensure there's no existing contract already at the designated address diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index bffc5013a6..967db32780 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -99,8 +99,8 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { if cfg.JumpTable[STOP] == nil { var jt JumpTable switch { - case evm.chainRules.IsYoloV2: - jt = yoloV2InstructionSet + case evm.chainRules.IsYoloV3: + jt = yoloV3InstructionSet case evm.chainRules.IsIstanbul: jt = istanbulInstructionSet case evm.chainRules.IsConstantinople: diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 83fb2c1ed6..954f657a94 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -56,16 +56,16 @@ var ( byzantiumInstructionSet = newByzantiumInstructionSet() constantinopleInstructionSet = newConstantinopleInstructionSet() istanbulInstructionSet = newIstanbulInstructionSet() - yoloV2InstructionSet = newYoloV2InstructionSet() + yoloV3InstructionSet = newYoloV3InstructionSet() ) // JumpTable contains the EVM opcodes supported at a given fork. type JumpTable [256]*operation -// newYoloV2InstructionSet creates an instructionset containing +// newYoloV3InstructionSet creates an instructionset containing // - "EIP-2315: Simple Subroutines" // - "EIP-2929: Gas cost increases for state access opcodes" -func newYoloV2InstructionSet() JumpTable { +func newYoloV3InstructionSet() JumpTable { instructionSet := newIstanbulInstructionSet() enable2315(&instructionSet) // Subroutines - https://eips.ethereum.org/EIPS/eip-2315 enable2929(&instructionSet) // Access lists for trie accesses https://eips.ethereum.org/EIPS/eip-2929 diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 2586535f9d..7cdb4ebd22 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -65,7 +65,7 @@ func setDefaults(cfg *Config) { PetersburgBlock: new(big.Int), IstanbulBlock: new(big.Int), MuirGlacierBlock: new(big.Int), - YoloV2Block: nil, + YoloV3Block: nil, } } @@ -113,7 +113,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { vmenv = NewEnv(cfg) sender = vm.AccountRef(cfg.Origin) ) - if cfg.ChainConfig.IsYoloV2(vmenv.Context.BlockNumber) { + if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { cfg.State.AddAddressToAccessList(cfg.Origin) cfg.State.AddAddressToAccessList(address) for _, addr := range vmenv.ActivePrecompiles() { @@ -149,7 +149,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { vmenv = NewEnv(cfg) sender = vm.AccountRef(cfg.Origin) ) - if cfg.ChainConfig.IsYoloV2(vmenv.Context.BlockNumber) { + if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { cfg.State.AddAddressToAccessList(cfg.Origin) for _, addr := range vmenv.ActivePrecompiles() { cfg.State.AddAddressToAccessList(addr) @@ -177,7 +177,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er vmenv := NewEnv(cfg) sender := cfg.State.GetOrNewStateObject(cfg.Origin) - if cfg.ChainConfig.IsYoloV2(vmenv.Context.BlockNumber) { + if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { cfg.State.AddAddressToAccessList(cfg.Origin) cfg.State.AddAddressToAccessList(address) for _, addr := range vmenv.ActivePrecompiles() { diff --git a/eth/tracers/api.go b/eth/tracers/api.go index dca990ab95..1bf59552c9 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -581,8 +581,8 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block chainConfigCopy := new(params.ChainConfig) *chainConfigCopy = *chainConfig chainConfig = chainConfigCopy - if yolov2 := config.LogConfig.Overrides.YoloV2Block; yolov2 != nil { - chainConfig.YoloV2Block = yolov2 + if yolov3 := config.LogConfig.Overrides.YoloV3Block; yolov3 != nil { + chainConfig.YoloV3Block = yolov3 canon = false } } diff --git a/params/bootnodes.go b/params/bootnodes.go index 66f917015e..f36ad61729 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -67,9 +67,10 @@ var GoerliBootnodes = []string{ "enode://a59e33ccd2b3e52d578f1fbd70c6f9babda2650f0760d6ff3b37742fdcdfdb3defba5d56d315b40c46b70198c7621e63ffa3f987389c7118634b0fefbbdfa7fd@51.15.119.157:40303", } -// YoloV2Bootnodes are the enode URLs of the P2P bootstrap nodes running on the -// YOLOv2 ephemeral test network. -var YoloV2Bootnodes = []string{ +// YoloV3Bootnodes are the enode URLs of the P2P bootstrap nodes running on the +// YOLOv3 ephemeral test network. +// TODO: Set Yolov3 bootnodes +var YoloV3Bootnodes = []string{ "enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303", } diff --git a/params/config.go b/params/config.go index 588d848f1c..0dbf592d19 100644 --- a/params/config.go +++ b/params/config.go @@ -31,8 +31,7 @@ var ( RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - // TODO: update with yolov2 values - YoloV2GenesisHash = common.HexToHash("0x498a7239036dd2cd09e2bb8a80922b78632017958c332b42044c250d603a8a3e") + YoloV3GenesisHash = common.HexToHash("0x374f07cc7fa7c251fc5f36849f574b43db43600526410349efdca2bcea14101a") ) // TrustedCheckpoints associates each known checkpoint with the genesis hash of @@ -214,8 +213,8 @@ var ( Threshold: 2, } - // YoloV2ChainConfig contains the chain parameters to run a node on the YOLOv2 test network. - YoloV2ChainConfig = &ChainConfig{ + // YoloV3ChainConfig contains the chain parameters to run a node on the YOLOv3 test network. + YoloV3ChainConfig = &ChainConfig{ ChainID: big.NewInt(133519467574834), HomesteadBlock: big.NewInt(0), DAOForkBlock: nil, @@ -228,7 +227,7 @@ var ( PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), MuirGlacierBlock: nil, - YoloV2Block: big.NewInt(0), + YoloV3Block: big.NewInt(0), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -321,7 +320,7 @@ type ChainConfig struct { IstanbulBlock *big.Int `json:"istanbulBlock,omitempty"` // Istanbul switch block (nil = no fork, 0 = already on istanbul) MuirGlacierBlock *big.Int `json:"muirGlacierBlock,omitempty"` // Eip-2384 (bomb delay) switch block (nil = no fork, 0 = already activated) - YoloV2Block *big.Int `json:"yoloV2Block,omitempty"` // YOLO v2: Gas repricings TODO @holiman add EIP references + YoloV3Block *big.Int `json:"yoloV3Block,omitempty"` // YOLO v3: Gas repricings TODO @holiman add EIP references EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) // Various consensus engines @@ -359,7 +358,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, YOLO v2: %v, Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, YOLO v3: %v, Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -372,7 +371,7 @@ func (c *ChainConfig) String() string { c.PetersburgBlock, c.IstanbulBlock, c.MuirGlacierBlock, - c.YoloV2Block, + c.YoloV3Block, engine, ) } @@ -429,9 +428,9 @@ func (c *ChainConfig) IsIstanbul(num *big.Int) bool { return isForked(c.IstanbulBlock, num) } -// IsYoloV2 returns whether num is either equal to the YoloV1 fork block or greater. -func (c *ChainConfig) IsYoloV2(num *big.Int) bool { - return isForked(c.YoloV2Block, num) +// IsYoloV3 returns whether num is either equal to the YoloV3 fork block or greater. +func (c *ChainConfig) IsYoloV3(num *big.Int) bool { + return isForked(c.YoloV3Block, num) } // IsEWASM returns whether num represents a block number after the EWASM fork @@ -477,7 +476,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "petersburgBlock", block: c.PetersburgBlock}, {name: "istanbulBlock", block: c.IstanbulBlock}, {name: "muirGlacierBlock", block: c.MuirGlacierBlock, optional: true}, - {name: "yoloV2Block", block: c.YoloV2Block}, + {name: "yoloV3Block", block: c.YoloV3Block}, } { if lastFork.name != "" { // Next one must be higher number @@ -541,8 +540,8 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.MuirGlacierBlock, newcfg.MuirGlacierBlock, head) { return newCompatError("Muir Glacier fork block", c.MuirGlacierBlock, newcfg.MuirGlacierBlock) } - if isForkIncompatible(c.YoloV2Block, newcfg.YoloV2Block, head) { - return newCompatError("YOLOv2 fork block", c.YoloV2Block, newcfg.YoloV2Block) + if isForkIncompatible(c.YoloV3Block, newcfg.YoloV3Block, head) { + return newCompatError("YOLOv3 fork block", c.YoloV3Block, newcfg.YoloV3Block) } if isForkIncompatible(c.EWASMBlock, newcfg.EWASMBlock, head) { return newCompatError("ewasm fork block", c.EWASMBlock, newcfg.EWASMBlock) @@ -614,7 +613,7 @@ type Rules struct { ChainID *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool - IsYoloV2 bool + IsYoloV3 bool } // Rules ensures c's ChainID is not nil. @@ -633,6 +632,6 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { IsConstantinople: c.IsConstantinople(num), IsPetersburg: c.IsPetersburg(num), IsIstanbul: c.IsIstanbul(num), - IsYoloV2: c.IsYoloV2(num), + IsYoloV3: c.IsYoloV3(num), } } diff --git a/tests/block_test.go b/tests/block_test.go index 8fa90e3e39..84618fd0be 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -45,6 +45,13 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { + if test.json.Network == "Berlin" { + // Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them + // fail when berlin is defined as YOLOv3. We skip those, until they've been + // regenerated and re-imported + // TODO (@holiman) + return + } if err := bt.checkFailure(t, name+"/trie", test.Run(false)); err != nil { t.Errorf("test without snapshotter failed: %v", err) } diff --git a/tests/fuzzers/bls12381/bls_fuzzer.go b/tests/fuzzers/bls12381/bls_fuzzer.go index 7e3f94c2aa..bc3c456526 100644 --- a/tests/fuzzers/bls12381/bls_fuzzer.go +++ b/tests/fuzzers/bls12381/bls_fuzzer.go @@ -79,7 +79,7 @@ func checkInput(id byte, inputLen int) bool { // other values are reserved for future use. func fuzz(id byte, data []byte) int { // Even on bad input, it should not crash, so we still test the gas calc - precompile := vm.PrecompiledContractsYoloV2[common.BytesToAddress([]byte{id})] + precompile := vm.PrecompiledContractsBLS[common.BytesToAddress([]byte{id})] gas := precompile.RequiredGas(data) if !checkInput(id, len(data)) { return 0 diff --git a/tests/init.go b/tests/init.go index 607c69ddb3..a2ef040786 100644 --- a/tests/init.go +++ b/tests/init.go @@ -141,7 +141,7 @@ var Forks = map[string]*params.ChainConfig{ PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(5), }, - "YOLOv2": { + "YOLOv3": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), @@ -151,9 +151,9 @@ var Forks = map[string]*params.ChainConfig{ ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), - YoloV2Block: big.NewInt(0), + YoloV3Block: big.NewInt(0), }, - // This specification is subject to change, but is for now identical to YOLOv2 + // This specification is subject to change, but is for now identical to YOLOv3 // for cross-client testing purposes "Berlin": { ChainID: big.NewInt(1), @@ -165,7 +165,7 @@ var Forks = map[string]*params.ChainConfig{ ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), - YoloV2Block: big.NewInt(0), + YoloV3Block: big.NewInt(0), }, } diff --git a/tests/state_test.go b/tests/state_test.go index b77a898c21..0f2bd38532 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -62,6 +62,14 @@ func TestState(t *testing.T) { } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { + if subtest.Fork == "Berlin" { + // Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them + // fail when berlin is defined as YOLOv3. We skip those, until they've been + // regenerated and re-imported + // TODO (@holiman) + continue + } + subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 96c7316969..7bd2e801e4 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -187,7 +187,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context.GetHash = vmTestBlockHash evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) - if config.IsYoloV2(context.BlockNumber) { + if config.IsYoloV3(context.BlockNumber) { statedb.AddAddressToAccessList(msg.From()) if dst := msg.To(); dst != nil { statedb.AddAddressToAccessList(*dst) From 7a800f98f66103d934f24231ffec1a656d25b496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Thu, 28 Jan 2021 22:47:15 +0100 Subject: [PATCH 106/709] les/utils: UDP rate limiter (#21930) * les/utils: Limiter * les/utils: dropped prior weight vs variable cost logic, using fixed weights * les/utils: always create node selector in addressGroup * les/utils: renamed request weight to request cost * les/utils: simplified and improved the DoS penalty mechanism * les/utils: minor fixes * les/utils: made selection weight calculation nicer * les/utils: fixed linter warning * les/utils: more precise and reliable probabilistic test * les/utils: fixed linter warning --- les/utils/limiter.go | 405 +++++++++++++++++++++++++++++++++++ les/utils/limiter_test.go | 206 ++++++++++++++++++ les/utils/weighted_select.go | 28 +-- 3 files changed, 625 insertions(+), 14 deletions(-) create mode 100644 les/utils/limiter.go create mode 100644 les/utils/limiter_test.go diff --git a/les/utils/limiter.go b/les/utils/limiter.go new file mode 100644 index 0000000000..0cc2d7b262 --- /dev/null +++ b/les/utils/limiter.go @@ -0,0 +1,405 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package utils + +import ( + "sort" + "sync" + + "github.com/ethereum/go-ethereum/p2p/enode" +) + +const maxSelectionWeight = 1000000000 // maximum selection weight of each individual node/address group + +// Limiter protects a network request serving mechanism from denial-of-service attacks. +// It limits the total amount of resources used for serving requests while ensuring that +// the most valuable connections always have a reasonable chance of being served. +type Limiter struct { + lock sync.Mutex + cond *sync.Cond + quit bool + + nodes map[enode.ID]*nodeQueue + addresses map[string]*addressGroup + addressSelect, valueSelect *WeightedRandomSelect + maxValue float64 + maxCost, sumCost, sumCostLimit uint + selectAddressNext bool +} + +// nodeQueue represents queued requests coming from a single node ID +type nodeQueue struct { + queue []request // always nil if penaltyCost != 0 + id enode.ID + address string + value float64 + flatWeight, valueWeight uint64 // current selection weights in the address/value selectors + sumCost uint // summed cost of requests queued by the node + penaltyCost uint // cumulative cost of dropped requests since last processed request + groupIndex int +} + +// addressGroup is a group of node IDs that have sent their last requests from the same +// network address +type addressGroup struct { + nodes []*nodeQueue + nodeSelect *WeightedRandomSelect + sumFlatWeight, groupWeight uint64 +} + +// request represents an incoming request scheduled for processing +type request struct { + process chan chan struct{} + cost uint +} + +// flatWeight distributes weights equally between each active network address +func flatWeight(item interface{}) uint64 { return item.(*nodeQueue).flatWeight } + +// add adds the node queue to the address group. It is the caller's responsibility to +// add the address group to the address map and the address selector if it wasn't +// there before. +func (ag *addressGroup) add(nq *nodeQueue) { + if nq.groupIndex != -1 { + panic("added node queue is already in an address group") + } + l := len(ag.nodes) + nq.groupIndex = l + ag.nodes = append(ag.nodes, nq) + ag.sumFlatWeight += nq.flatWeight + ag.groupWeight = ag.sumFlatWeight / uint64(l+1) + ag.nodeSelect.Update(ag.nodes[l]) +} + +// update updates the selection weight of the node queue inside the address group. +// It is the caller's responsibility to update the group's selection weight in the +// address selector. +func (ag *addressGroup) update(nq *nodeQueue, weight uint64) { + if nq.groupIndex == -1 || nq.groupIndex >= len(ag.nodes) || ag.nodes[nq.groupIndex] != nq { + panic("updated node queue is not in this address group") + } + ag.sumFlatWeight += weight - nq.flatWeight + nq.flatWeight = weight + ag.groupWeight = ag.sumFlatWeight / uint64(len(ag.nodes)) + ag.nodeSelect.Update(nq) +} + +// remove removes the node queue from the address group. It is the caller's responsibility +// to remove the address group from the address map if it is empty. +func (ag *addressGroup) remove(nq *nodeQueue) { + if nq.groupIndex == -1 || nq.groupIndex >= len(ag.nodes) || ag.nodes[nq.groupIndex] != nq { + panic("removed node queue is not in this address group") + } + + l := len(ag.nodes) - 1 + if nq.groupIndex != l { + ag.nodes[nq.groupIndex] = ag.nodes[l] + ag.nodes[nq.groupIndex].groupIndex = nq.groupIndex + } + nq.groupIndex = -1 + ag.nodes = ag.nodes[:l] + ag.sumFlatWeight -= nq.flatWeight + if l >= 1 { + ag.groupWeight = ag.sumFlatWeight / uint64(l) + } else { + ag.groupWeight = 0 + } + ag.nodeSelect.Remove(nq) +} + +// choose selects one of the node queues belonging to the address group +func (ag *addressGroup) choose() *nodeQueue { + return ag.nodeSelect.Choose().(*nodeQueue) +} + +// NewLimiter creates a new Limiter +func NewLimiter(sumCostLimit uint) *Limiter { + l := &Limiter{ + addressSelect: NewWeightedRandomSelect(func(item interface{}) uint64 { return item.(*addressGroup).groupWeight }), + valueSelect: NewWeightedRandomSelect(func(item interface{}) uint64 { return item.(*nodeQueue).valueWeight }), + nodes: make(map[enode.ID]*nodeQueue), + addresses: make(map[string]*addressGroup), + sumCostLimit: sumCostLimit, + } + l.cond = sync.NewCond(&l.lock) + go l.processLoop() + return l +} + +// selectionWeights calculates the selection weights of a node for both the address and +// the value selector. The selection weight depends on the next request cost or the +// summed cost of recently dropped requests. +func (l *Limiter) selectionWeights(reqCost uint, value float64) (flatWeight, valueWeight uint64) { + if value > l.maxValue { + l.maxValue = value + } + if value > 0 { + // normalize value to <= 1 + value /= l.maxValue + } + if reqCost > l.maxCost { + l.maxCost = reqCost + } + relCost := float64(reqCost) / float64(l.maxCost) + var f float64 + if relCost <= 0.001 { + f = 1 + } else { + f = 0.001 / relCost + } + f *= maxSelectionWeight + flatWeight, valueWeight = uint64(f), uint64(f*value) + if flatWeight == 0 { + flatWeight = 1 + } + return +} + +// Add adds a new request to the node queue belonging to the given id. Value belongs +// to the requesting node. A higher value gives the request a higher chance of being +// served quickly in case of heavy load or a DDoS attack. Cost is a rough estimate +// of the serving cost of the request. A lower cost also gives the request a +// better chance. +func (l *Limiter) Add(id enode.ID, address string, value float64, reqCost uint) chan chan struct{} { + l.lock.Lock() + defer l.lock.Unlock() + + process := make(chan chan struct{}, 1) + if l.quit { + close(process) + return process + } + if reqCost == 0 { + reqCost = 1 + } + if nq, ok := l.nodes[id]; ok { + if nq.queue != nil { + nq.queue = append(nq.queue, request{process, reqCost}) + nq.sumCost += reqCost + nq.value = value + if address != nq.address { + // known id sending request from a new address, move to different address group + l.removeFromGroup(nq) + l.addToGroup(nq, address) + } + } else { + // already waiting on a penalty, just add to the penalty cost and drop the request + nq.penaltyCost += reqCost + l.update(nq) + close(process) + return process + } + } else { + nq := &nodeQueue{ + queue: []request{{process, reqCost}}, + id: id, + value: value, + sumCost: reqCost, + groupIndex: -1, + } + nq.flatWeight, nq.valueWeight = l.selectionWeights(reqCost, value) + if len(l.nodes) == 0 { + l.cond.Signal() + } + l.nodes[id] = nq + if nq.valueWeight != 0 { + l.valueSelect.Update(nq) + } + l.addToGroup(nq, address) + } + l.sumCost += reqCost + if l.sumCost > l.sumCostLimit { + l.dropRequests() + } + return process +} + +// update updates the selection weights of the node queue +func (l *Limiter) update(nq *nodeQueue) { + var cost uint + if nq.queue != nil { + cost = nq.queue[0].cost + } else { + cost = nq.penaltyCost + } + flatWeight, valueWeight := l.selectionWeights(cost, nq.value) + ag := l.addresses[nq.address] + ag.update(nq, flatWeight) + l.addressSelect.Update(ag) + nq.valueWeight = valueWeight + l.valueSelect.Update(nq) +} + +// addToGroup adds the node queue to the given address group. The group is created if +// it does not exist yet. +func (l *Limiter) addToGroup(nq *nodeQueue, address string) { + nq.address = address + ag := l.addresses[address] + if ag == nil { + ag = &addressGroup{nodeSelect: NewWeightedRandomSelect(flatWeight)} + l.addresses[address] = ag + } + ag.add(nq) + l.addressSelect.Update(ag) +} + +// removeFromGroup removes the node queue from its address group +func (l *Limiter) removeFromGroup(nq *nodeQueue) { + ag := l.addresses[nq.address] + ag.remove(nq) + if len(ag.nodes) == 0 { + delete(l.addresses, nq.address) + } + l.addressSelect.Update(ag) +} + +// remove removes the node queue from its address group, the nodes map and the value +// selector +func (l *Limiter) remove(nq *nodeQueue) { + l.removeFromGroup(nq) + if nq.valueWeight != 0 { + l.valueSelect.Remove(nq) + } + delete(l.nodes, nq.id) +} + +// choose selects the next node queue to process. +func (l *Limiter) choose() *nodeQueue { + if l.valueSelect.IsEmpty() || l.selectAddressNext { + if ag, ok := l.addressSelect.Choose().(*addressGroup); ok { + l.selectAddressNext = false + return ag.choose() + } + } + nq, _ := l.valueSelect.Choose().(*nodeQueue) + l.selectAddressNext = true + return nq +} + +// processLoop processes requests sequentially +func (l *Limiter) processLoop() { + l.lock.Lock() + defer l.lock.Unlock() + + for { + if l.quit { + for _, nq := range l.nodes { + for _, request := range nq.queue { + close(request.process) + } + } + return + } + nq := l.choose() + if nq == nil { + l.cond.Wait() + continue + } + if nq.queue != nil { + request := nq.queue[0] + nq.queue = nq.queue[1:] + nq.sumCost -= request.cost + l.sumCost -= request.cost + l.lock.Unlock() + ch := make(chan struct{}) + request.process <- ch + <-ch + l.lock.Lock() + if len(nq.queue) > 0 { + l.update(nq) + } else { + l.remove(nq) + } + } else { + // penalized queue removed, next request will be added to a clean queue + l.remove(nq) + } + } +} + +// Stop stops the processing loop. All queued and future requests are rejected. +func (l *Limiter) Stop() { + l.lock.Lock() + defer l.lock.Unlock() + + l.quit = true + l.cond.Signal() +} + +type ( + dropList []dropListItem + dropListItem struct { + nq *nodeQueue + priority float64 + } +) + +func (l dropList) Len() int { + return len(l) +} + +func (l dropList) Less(i, j int) bool { + return l[i].priority < l[j].priority +} + +func (l dropList) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +// dropRequests selects the nodes with the highest queued request cost to selection +// weight ratio and drops their queued request. The empty node queues stay in the +// selectors with a low selection weight in order to penalize these nodes. +func (l *Limiter) dropRequests() { + var ( + sumValue float64 + list dropList + ) + for _, nq := range l.nodes { + sumValue += nq.value + } + for _, nq := range l.nodes { + if nq.sumCost == 0 { + continue + } + w := 1 / float64(len(l.addresses)*len(l.addresses[nq.address].nodes)) + if sumValue > 0 { + w += nq.value / sumValue + } + list = append(list, dropListItem{ + nq: nq, + priority: w / float64(nq.sumCost), + }) + } + sort.Sort(list) + for _, item := range list { + for _, request := range item.nq.queue { + close(request.process) + } + // make the queue penalized; no more requests are accepted until the node is + // selected based on the penalty cost which is the cumulative cost of all dropped + // requests. This ensures that sending excess requests is always penalized + // and incentivizes the sender to stop for a while if no replies are received. + item.nq.queue = nil + item.nq.penaltyCost = item.nq.sumCost + l.sumCost -= item.nq.sumCost // penalty costs are not counted in sumCost + item.nq.sumCost = 0 + l.update(item.nq) + if l.sumCost <= l.sumCostLimit/2 { + return + } + } +} diff --git a/les/utils/limiter_test.go b/les/utils/limiter_test.go new file mode 100644 index 0000000000..43af3309ab --- /dev/null +++ b/les/utils/limiter_test.go @@ -0,0 +1,206 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package utils + +import ( + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/p2p/enode" +) + +const ( + ltTolerance = 0.03 + ltRounds = 7 +) + +type ( + ltNode struct { + addr, id int + value, exp float64 + cost uint + reqRate float64 + reqMax, runCount int + lastTotalCost uint + + served, dropped int + } + + ltResult struct { + node *ltNode + ch chan struct{} + } + + limTest struct { + limiter *Limiter + results chan ltResult + runCount int + expCost, totalCost uint + } +) + +func (lt *limTest) request(n *ltNode) { + var ( + address string + id enode.ID + ) + if n.addr >= 0 { + address = string([]byte{byte(n.addr)}) + } else { + var b [32]byte + rand.Read(b[:]) + address = string(b[:]) + } + if n.id >= 0 { + id = enode.ID{byte(n.id)} + } else { + rand.Read(id[:]) + } + lt.runCount++ + n.runCount++ + cch := lt.limiter.Add(id, address, n.value, n.cost) + go func() { + lt.results <- ltResult{n, <-cch} + }() +} + +func (lt *limTest) moreRequests(n *ltNode) { + maxStart := int(float64(lt.totalCost-n.lastTotalCost) * n.reqRate) + if maxStart != 0 { + n.lastTotalCost = lt.totalCost + } + for n.reqMax > n.runCount && maxStart > 0 { + lt.request(n) + maxStart-- + } +} + +func (lt *limTest) process() { + res := <-lt.results + lt.runCount-- + res.node.runCount-- + if res.ch != nil { + res.node.served++ + if res.node.exp != 0 { + lt.expCost += res.node.cost + } + lt.totalCost += res.node.cost + close(res.ch) + } else { + res.node.dropped++ + } +} + +func TestLimiter(t *testing.T) { + limTests := [][]*ltNode{ + { // one id from an individual address and two ids from a shared address + {addr: 0, id: 0, value: 0, cost: 1, reqRate: 1, reqMax: 1, exp: 0.5}, + {addr: 1, id: 1, value: 0, cost: 1, reqRate: 1, reqMax: 1, exp: 0.25}, + {addr: 1, id: 2, value: 0, cost: 1, reqRate: 1, reqMax: 1, exp: 0.25}, + }, + { // varying request costs + {addr: 0, id: 0, value: 0, cost: 10, reqRate: 0.2, reqMax: 1, exp: 0.5}, + {addr: 1, id: 1, value: 0, cost: 3, reqRate: 0.5, reqMax: 1, exp: 0.25}, + {addr: 1, id: 2, value: 0, cost: 1, reqRate: 1, reqMax: 1, exp: 0.25}, + }, + { // different request rate + {addr: 0, id: 0, value: 0, cost: 1, reqRate: 2, reqMax: 2, exp: 0.5}, + {addr: 1, id: 1, value: 0, cost: 1, reqRate: 10, reqMax: 10, exp: 0.25}, + {addr: 1, id: 2, value: 0, cost: 1, reqRate: 1, reqMax: 1, exp: 0.25}, + }, + { // adding value + {addr: 0, id: 0, value: 3, cost: 1, reqRate: 1, reqMax: 1, exp: (0.5 + 0.3) / 2}, + {addr: 1, id: 1, value: 0, cost: 1, reqRate: 1, reqMax: 1, exp: 0.25 / 2}, + {addr: 1, id: 2, value: 7, cost: 1, reqRate: 1, reqMax: 1, exp: (0.25 + 0.7) / 2}, + }, + { // DoS attack from a single address with a single id + {addr: 0, id: 0, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 1, id: 1, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 2, id: 2, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 3, id: 3, value: 0, cost: 1, reqRate: 10, reqMax: 1000000000, exp: 0}, + }, + { // DoS attack from a single address with different ids + {addr: 0, id: 0, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 1, id: 1, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 2, id: 2, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 3, id: -1, value: 0, cost: 1, reqRate: 1, reqMax: 1000000000, exp: 0}, + }, + { // DDoS attack from different addresses with a single id + {addr: 0, id: 0, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 1, id: 1, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 2, id: 2, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: -1, id: 3, value: 0, cost: 1, reqRate: 1, reqMax: 1000000000, exp: 0}, + }, + { // DDoS attack from different addresses with different ids + {addr: 0, id: 0, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 1, id: 1, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: 2, id: 2, value: 1, cost: 1, reqRate: 1, reqMax: 1, exp: 0.3333}, + {addr: -1, id: -1, value: 0, cost: 1, reqRate: 1, reqMax: 1000000000, exp: 0}, + }, + } + + lt := &limTest{ + limiter: NewLimiter(100), + results: make(chan ltResult), + } + for _, test := range limTests { + lt.expCost, lt.totalCost = 0, 0 + iterCount := 10000 + for j := 0; j < ltRounds; j++ { + // try to reach expected target range in multiple rounds with increasing iteration counts + last := j == ltRounds-1 + for _, n := range test { + lt.request(n) + } + for i := 0; i < iterCount; i++ { + lt.process() + for _, n := range test { + lt.moreRequests(n) + } + } + for lt.runCount > 0 { + lt.process() + } + if spamRatio := 1 - float64(lt.expCost)/float64(lt.totalCost); spamRatio > 0.5*(1+ltTolerance) { + t.Errorf("Spam ratio too high (%f)", spamRatio) + } + fail, success := false, true + for _, n := range test { + if n.exp != 0 { + if n.dropped > 0 { + t.Errorf("Dropped %d requests of non-spam node", n.dropped) + fail = true + } + r := float64(n.served) * float64(n.cost) / float64(lt.expCost) + if r < n.exp*(1-ltTolerance) || r > n.exp*(1+ltTolerance) { + if last { + // print error only if the target is still not reached in the last round + t.Errorf("Request ratio (%f) does not match expected value (%f)", r, n.exp) + } + success = false + } + } + } + if fail || success { + break + } + // neither failed nor succeeded; try more iterations to reach probability targets + iterCount *= 2 + } + } + lt.limiter.Stop() +} diff --git a/les/utils/weighted_select.go b/les/utils/weighted_select.go index 9b4413afb5..486b00820a 100644 --- a/les/utils/weighted_select.go +++ b/les/utils/weighted_select.go @@ -52,17 +52,17 @@ func (w *WeightedRandomSelect) Remove(item WrsItem) { // IsEmpty returns true if the set is empty func (w *WeightedRandomSelect) IsEmpty() bool { - return w.root.sumWeight == 0 + return w.root.sumCost == 0 } // setWeight sets an item's weight to a specific value (removes it if zero) func (w *WeightedRandomSelect) setWeight(item WrsItem, weight uint64) { - if weight > math.MaxInt64-w.root.sumWeight { - // old weight is still included in sumWeight, remove and check again + if weight > math.MaxInt64-w.root.sumCost { + // old weight is still included in sumCost, remove and check again w.setWeight(item, 0) - if weight > math.MaxInt64-w.root.sumWeight { - log.Error("WeightedRandomSelect overflow", "sumWeight", w.root.sumWeight, "new weight", weight) - weight = math.MaxInt64 - w.root.sumWeight + if weight > math.MaxInt64-w.root.sumCost { + log.Error("WeightedRandomSelect overflow", "sumCost", w.root.sumCost, "new weight", weight) + weight = math.MaxInt64 - w.root.sumCost } } idx, ok := w.idx[item] @@ -75,9 +75,9 @@ func (w *WeightedRandomSelect) setWeight(item WrsItem, weight uint64) { if weight != 0 { if w.root.itemCnt == w.root.maxItems { // add a new level - newRoot := &wrsNode{sumWeight: w.root.sumWeight, itemCnt: w.root.itemCnt, level: w.root.level + 1, maxItems: w.root.maxItems * wrsBranches} + newRoot := &wrsNode{sumCost: w.root.sumCost, itemCnt: w.root.itemCnt, level: w.root.level + 1, maxItems: w.root.maxItems * wrsBranches} newRoot.items[0] = w.root - newRoot.weights[0] = w.root.sumWeight + newRoot.weights[0] = w.root.sumCost w.root = newRoot } w.idx[item] = w.root.insert(item, weight) @@ -91,10 +91,10 @@ func (w *WeightedRandomSelect) setWeight(item WrsItem, weight uint64) { // updates its weight and selects another one func (w *WeightedRandomSelect) Choose() WrsItem { for { - if w.root.sumWeight == 0 { + if w.root.sumCost == 0 { return nil } - val := uint64(rand.Int63n(int64(w.root.sumWeight))) + val := uint64(rand.Int63n(int64(w.root.sumCost))) choice, lastWeight := w.root.choose(val) weight := w.wfn(choice) if weight != lastWeight { @@ -112,7 +112,7 @@ const wrsBranches = 8 // max number of branches in the wrsNode tree type wrsNode struct { items [wrsBranches]interface{} weights [wrsBranches]uint64 - sumWeight uint64 + sumCost uint64 level, itemCnt, maxItems int } @@ -126,7 +126,7 @@ func (n *wrsNode) insert(item WrsItem, weight uint64) int { } } n.itemCnt++ - n.sumWeight += weight + n.sumCost += weight n.weights[branch] += weight if n.level == 0 { n.items[branch] = item @@ -150,7 +150,7 @@ func (n *wrsNode) setWeight(idx int, weight uint64) uint64 { oldWeight := n.weights[idx] n.weights[idx] = weight diff := weight - oldWeight - n.sumWeight += diff + n.sumCost += diff if weight == 0 { n.items[idx] = nil n.itemCnt-- @@ -161,7 +161,7 @@ func (n *wrsNode) setWeight(idx int, weight uint64) uint64 { branch := idx / branchItems diff := n.items[branch].(*wrsNode).setWeight(idx-branch*branchItems, weight) n.weights[branch] += diff - n.sumWeight += diff + n.sumCost += diff if weight == 0 { n.itemCnt-- } From f25b437b70dd8be8ab7677fae607e01cd86c1ef4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 29 Jan 2021 16:53:44 +0100 Subject: [PATCH 107/709] cmd/clef: don't check file permissions on windows (#22251) Fixes #20123 --- cmd/clef/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 273919cb41..090edd4507 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -805,14 +805,16 @@ func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) { // checkFile is a convenience function to check if a file // * exists -// * is mode 0400 +// * is mode 0400 (unix only) func checkFile(filename string) error { info, err := os.Stat(filename) if err != nil { return fmt.Errorf("failed stat on %s: %v", filename, err) } // Check the unix permission bits - if info.Mode().Perm()&0377 != 0 { + // However, on windows, we cannot use the unix perm-bits, see + // https://github.com/ethereum/go-ethereum/issues/20123 + if runtime.GOOS != "windows" && info.Mode().Perm()&0377 != 0 { return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String()) } return nil From 3c728fb129210c044ae71aad55e4c059d70a318d Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Mon, 1 Feb 2021 14:41:43 +0100 Subject: [PATCH 108/709] eth/tracers: fix unigram tracer (#22248) --- eth/tracers/internal/tracers/assets.go | 37 ++++++++----------- .../internal/tracers/unigram_tracer.js | 4 +- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 8b89ad4182..7f45ab286e 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -8,7 +8,7 @@ // opcount_tracer.js (1.372kB) // prestate_tracer.js (4.287kB) // trigram_tracer.js (1.788kB) -// unigram_tracer.js (1.51kB) +// unigram_tracer.js (1.469kB) package tracers @@ -28,7 +28,7 @@ import ( func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { - return nil, fmt.Errorf("read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %w", name, err) } var buf bytes.Buffer @@ -36,7 +36,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { clErr := gz.Close() if err != nil { - return nil, fmt.Errorf("read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %w", name, err) } if clErr != nil { return nil, err @@ -237,7 +237,7 @@ func trigram_tracerJs() (*asset, error) { return a, nil } -var _unigram_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x54\x4d\x6f\xdb\x46\x10\xbd\xeb\x57\xbc\xa3\x8c\xa8\xa4\xd3\x5e\x0a\xa5\x09\xc0\x1a\x76\x22\xc0\x91\x0d\x89\x6e\x60\x14\x3d\x2c\xc9\x21\xb9\xe8\x6a\x87\xd8\x9d\x95\x42\x04\xfa\xef\xc5\x92\xa2\xe5\x1a\x6e\x13\x9e\x04\xcd\xbc\x8f\x79\x33\x64\x9a\xe2\x8a\xbb\xde\xe9\xa6\x15\xfc\x7c\xf9\xf6\x57\xe4\x2d\xa1\xe1\x9f\x48\x5a\x72\x14\x76\xc8\x82\xb4\xec\xfc\x2c\x4d\x91\xb7\xda\xa3\xd6\x86\xa0\x3d\x3a\xe5\x04\x5c\x43\x5e\xf4\x1b\x5d\x38\xe5\xfa\x64\x96\xa6\x23\xe6\xd5\x72\x64\xa8\x1d\x11\x3c\xd7\x72\x50\x8e\x96\xe8\x39\xa0\x54\x16\x8e\x2a\xed\xc5\xe9\x22\x08\x41\x0b\x94\xad\x52\x76\xd8\x71\xa5\xeb\x3e\x52\x6a\x41\xb0\x15\xb9\x41\x5a\xc8\xed\xfc\xe4\xe3\xe3\xfa\x01\xb7\xe4\x3d\x39\x7c\x24\x4b\x4e\x19\xdc\x87\xc2\xe8\x12\xb7\xba\x24\xeb\x09\xca\xa3\x8b\xff\xf8\x96\x2a\x14\x03\x5d\x04\xde\x44\x2b\xdb\x93\x15\xdc\x70\xb0\x95\x12\xcd\x76\x01\xd2\xd1\x39\xf6\xe4\xbc\x66\x8b\x5f\x26\xa9\x13\xe1\x02\xec\x22\xc9\x5c\x49\x1c\xc0\x81\xbb\x88\xbb\x80\xb2\x3d\x8c\x92\x33\xf4\x07\x02\x39\xcf\x5d\x41\xdb\x41\xa6\xe5\x8e\x20\xad\x92\x38\xf5\x41\x1b\x83\x82\x10\x3c\xd5\xc1\x2c\x22\x5b\x11\x04\x5f\x56\xf9\xa7\xbb\x87\x1c\xd9\xfa\x11\x5f\xb2\xcd\x26\x5b\xe7\x8f\xef\x70\xd0\xd2\x72\x10\xd0\x9e\x46\x2a\xbd\xeb\x8c\xa6\x0a\x07\xe5\x9c\xb2\xd2\x83\xeb\xc8\xf0\xf9\x7a\x73\xf5\x29\x5b\xe7\xd9\xef\xab\xdb\x55\xfe\x08\x76\xb8\x59\xe5\xeb\xeb\xed\x16\x37\x77\x1b\x64\xb8\xcf\x36\xf9\xea\xea\xe1\x36\xdb\xe0\xfe\x61\x73\x7f\xb7\xbd\x4e\xb0\xa5\xe8\x8a\x22\xfe\xfb\x99\xd7\xc3\xf6\x1c\xa1\x22\x51\xda\xf8\x29\x89\x47\x0e\xf0\x2d\x07\x53\xa1\x55\x7b\x82\xa3\x92\xf4\x9e\x2a\x28\x94\xdc\xf5\x3f\xbc\xd4\xc8\xa5\x0c\xdb\x66\x98\xf9\x3f\x0f\x12\xab\x1a\x96\x65\x01\x4f\x84\xdf\x5a\x91\x6e\x99\xa6\x87\xc3\x21\x69\x6c\x48\xd8\x35\xa9\x19\xe9\x7c\xfa\x21\x99\xcd\xbe\xcd\x00\x20\x4d\xd1\x6a\x2f\x71\x39\x91\x76\xa7\xba\xe8\x8a\xbb\x92\x2b\xf2\x10\x46\xc9\xc1\x0a\x39\x3f\x74\xc7\xd6\x25\xbe\x1d\x17\x13\xd6\x72\xe7\xc7\x16\x0f\x1b\x76\x05\xb9\x11\x3e\xb6\xc7\xea\x12\x97\x4f\xdd\x5e\xa8\x8b\x4a\xda\xee\xf9\x6f\xaa\x86\xdc\x68\x4f\xae\x3f\x09\x8e\x77\x10\x7d\xfc\xf1\x19\xf4\x95\xca\x20\xe4\x93\x01\x1d\xa1\x4b\xd4\xc1\x96\xf1\xfa\xe6\x86\x9b\x05\xaa\xe2\x02\xe3\x14\xf1\xd9\xab\x78\x9b\x78\x0f\xc3\x4d\xc2\x5d\x22\xbc\x15\xa7\x6d\x33\xbf\x78\xf7\xd4\xa3\x6b\xcc\xa5\xd5\x3e\x89\x83\xfc\xc9\xdd\x5f\x17\x67\x7c\x7c\xfe\x55\x7b\xf3\xe6\x0c\x3c\x3e\xfd\x22\xe3\x09\xff\x83\xc2\x7b\xbc\x7d\x0d\x37\x34\xc5\x40\x26\xda\x73\x88\xb5\x0a\x46\x9e\xe7\x72\x68\x4f\x17\xad\x4a\x09\xca\x9c\xa2\x88\x6f\x27\xd7\x50\x76\x4a\xab\x1e\x6f\x2d\xb2\x0c\x14\xaf\xe6\x73\x5c\xcc\x26\x1d\x47\xfe\x35\x21\x65\xcc\x20\x36\x2d\x7d\x38\xd5\x82\xc8\x42\x0b\x39\x15\xdf\x55\xde\x93\x8b\x9f\x29\x38\x92\xe0\xac\x9f\x18\x23\xac\xd6\x56\x99\x89\xfb\x74\xd1\xe2\x54\xa9\x6d\x33\x7a\x1b\x4b\xcf\xcc\x95\xf2\xf5\xf9\xe2\x74\x3d\x7f\x0a\x07\x1f\x70\xf9\x62\x27\xa3\xe4\x39\xe4\x97\xe1\x1e\x17\xb3\xe3\xec\x9f\x00\x00\x00\xff\xff\x8d\xba\x8d\xa8\xe6\x05\x00\x00") +var _unigram_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\x41\x6f\xdb\xc6\x13\xc5\xef\xfa\x14\xef\x68\x23\xfa\x8b\xc9\xbf\x97\x42\x69\x0a\xb0\x86\x9d\x08\x70\x64\x43\xa2\x1b\x18\x45\x0f\x4b\x72\x48\x2e\xba\xda\x21\x76\x67\xa5\x08\x81\xbf\x7b\x31\xa4\x68\xb9\x85\xdb\x86\x27\x41\x3b\xef\x37\x6f\xde\x0e\x99\x65\xb8\xe2\xfe\x18\x6c\xdb\x09\xfe\xff\xf6\xdd\x8f\x28\x3a\x42\xcb\xff\x23\xe9\x28\x50\xda\x21\x4f\xd2\x71\x88\xb3\x2c\x43\xd1\xd9\x88\xc6\x3a\x82\x8d\xe8\x4d\x10\x70\x03\xf9\x5b\xbd\xb3\x65\x30\xe1\xb8\x98\x65\xd9\xa8\x79\xf5\x58\x09\x4d\x20\x42\xe4\x46\x0e\x26\xd0\x12\x47\x4e\xa8\x8c\x47\xa0\xda\x46\x09\xb6\x4c\x42\xb0\x02\xe3\xeb\x8c\x03\x76\x5c\xdb\xe6\xa8\x48\x2b\x48\xbe\xa6\x30\xb4\x16\x0a\xbb\x38\xf9\xf8\xb8\x7e\xc0\x2d\xc5\x48\x01\x1f\xc9\x53\x30\x0e\xf7\xa9\x74\xb6\xc2\xad\xad\xc8\x47\x82\x89\xe8\xf5\x9f\xd8\x51\x8d\x72\xc0\xa9\xf0\x46\xad\x6c\x4f\x56\x70\xc3\xc9\xd7\x46\x2c\xfb\x39\xc8\xaa\x73\xec\x29\x44\xcb\x1e\x3f\x4c\xad\x4e\xc0\x39\x38\x28\xe4\xc2\x88\x0e\x10\xc0\xbd\xea\x2e\x61\xfc\x11\xce\xc8\x59\xfa\x1d\x81\x9c\xe7\xae\x61\xfd\xd0\xa6\xe3\x9e\x20\x9d\x11\x9d\xfa\x60\x9d\x43\x49\x48\x91\x9a\xe4\xe6\x4a\x2b\x93\xe0\xcb\xaa\xf8\x74\xf7\x50\x20\x5f\x3f\xe2\x4b\xbe\xd9\xe4\xeb\xe2\xf1\x3d\x0e\x56\x3a\x4e\x02\xda\xd3\x88\xb2\xbb\xde\x59\xaa\x71\x30\x21\x18\x2f\x47\x70\xa3\x84\xcf\xd7\x9b\xab\x4f\xf9\xba\xc8\x7f\x59\xdd\xae\x8a\x47\x70\xc0\xcd\xaa\x58\x5f\x6f\xb7\xb8\xb9\xdb\x20\xc7\x7d\xbe\x29\x56\x57\x0f\xb7\xf9\x06\xf7\x0f\x9b\xfb\xbb\xed\xf5\x02\x5b\x52\x57\xa4\xfa\xff\xce\xbc\x19\x6e\x2f\x10\x6a\x12\x63\x5d\x9c\x92\x78\xe4\x84\xd8\x71\x72\x35\x3a\xb3\x27\x04\xaa\xc8\xee\xa9\x86\x41\xc5\xfd\xf1\xbb\x2f\x55\x59\xc6\xb1\x6f\x87\x99\xff\x71\x21\xb1\x6a\xe0\x59\xe6\x88\x44\xf8\xa9\x13\xe9\x97\x59\x76\x38\x1c\x16\xad\x4f\x0b\x0e\x6d\xe6\x46\x5c\xcc\x7e\x5e\xcc\x66\xdf\x66\x00\x90\x65\xe8\x6c\x14\xbd\x1c\xc5\xee\x4c\xaf\xae\xb8\xaf\xb8\xa6\x08\x61\x54\x9c\xbc\x50\x88\x43\xb5\x96\x2e\xf1\xed\x69\x3e\x69\x3d\xf7\x71\x2c\x89\xf0\x69\x57\x52\x18\xe5\x63\xb9\x9e\x2e\xf1\xf6\xb9\x3a\x0a\xf5\xda\xc9\xfa\x3d\xff\x41\xf5\x90\x1b\xed\x29\x1c\x4f\x0d\xc7\x3d\x50\x1f\xbf\x7e\x06\x7d\xa5\x2a\x09\xc5\xc5\xa0\x56\xe9\x12\x4d\xf2\x95\x6e\xdf\x85\xe3\x76\x8e\xba\xbc\xc4\x38\x85\x3e\x7b\xa3\xbb\x89\x0f\x70\xdc\x2e\xb8\x5f\x08\x6f\x25\x58\xdf\x5e\x5c\xbe\x7f\xae\xb1\x0d\x2e\xa4\xb3\x71\xa1\x83\xfc\xc6\xfd\xef\x97\x67\xbd\x3e\x7f\x39\x7b\xf3\xe6\x2c\x7c\x7a\xfe\x45\x2e\x12\xfe\x45\x85\x0f\x78\xf7\x9a\x6e\x28\xd2\x40\x26\xec\x39\xc4\xc6\x24\x27\x2f\x73\x39\x74\xa7\x8d\x36\x95\x24\xe3\x4e\x51\xe8\xdb\xc9\x0d\x8c\x9f\xd2\x6a\xc6\x5d\x53\xca\x80\x78\x35\x9f\xa7\xf9\x6c\xea\x13\x28\xbe\xd6\xc8\x38\x37\x34\x9b\x2e\x7d\x58\xd5\x92\xc8\xc3\x0a\x05\xa3\xef\x2a\xef\x29\xe8\x67\x0a\x81\x24\x05\x1f\x27\xa2\xca\x1a\xeb\x8d\x9b\xd8\xa7\x8d\x96\x60\x2a\xeb\xdb\xd1\xdb\x78\xf4\xc2\x5c\x25\x5f\x5f\x5e\xdc\xc8\x3c\xa7\xf8\x1c\xcf\xd3\xec\xcf\x00\x00\x00\xff\xff\xf1\x91\x30\xae\xbd\x05\x00\x00") func unigram_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -253,7 +253,7 @@ func unigram_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "unigram_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2f, 0x36, 0x14, 0xc2, 0xf6, 0xc3, 0x80, 0x2b, 0x4a, 0x11, 0x7d, 0xd5, 0x3e, 0xef, 0x23, 0xb5, 0xd6, 0xe6, 0xe6, 0x5, 0x41, 0xf6, 0x14, 0x7a, 0x39, 0xf7, 0xf8, 0xac, 0x89, 0x8e, 0x43, 0xe6}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xc, 0xe6, 0x5c, 0x88, 0x18, 0xa7, 0x85, 0x61, 0x18, 0xc6, 0xec, 0x17, 0xfc, 0xdf, 0x9d, 0xc0, 0x1b, 0x49, 0xf8, 0x8d, 0xf1, 0xeb, 0x35, 0xf3, 0xd, 0x3e, 0xf6, 0xa3, 0xac, 0x8c, 0xba, 0x74}} return a, nil } @@ -348,25 +348,20 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "4byte_tracer.js": _4byte_tracerJs, - - "bigram_tracer.js": bigram_tracerJs, - - "call_tracer.js": call_tracerJs, - - "evmdis_tracer.js": evmdis_tracerJs, - - "noop_tracer.js": noop_tracerJs, - - "opcount_tracer.js": opcount_tracerJs, - + "4byte_tracer.js": _4byte_tracerJs, + "bigram_tracer.js": bigram_tracerJs, + "call_tracer.js": call_tracerJs, + "evmdis_tracer.js": evmdis_tracerJs, + "noop_tracer.js": noop_tracerJs, + "opcount_tracer.js": opcount_tracerJs, "prestate_tracer.js": prestate_tracerJs, - - "trigram_tracer.js": trigram_tracerJs, - - "unigram_tracer.js": unigram_tracerJs, + "trigram_tracer.js": trigram_tracerJs, + "unigram_tracer.js": unigram_tracerJs, } +// AssetDebug is true if the assets were built with the debug flag enabled. +const AssetDebug = false + // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the diff --git a/eth/tracers/internal/tracers/unigram_tracer.js b/eth/tracers/internal/tracers/unigram_tracer.js index 000fb13b1e..51107d8f3d 100644 --- a/eth/tracers/internal/tracers/unigram_tracer.js +++ b/eth/tracers/internal/tracers/unigram_tracer.js @@ -36,8 +36,6 @@ // result is invoked when all the opcodes have been iterated over and returns // the final result of the tracing. result: function(ctx) { - if(this.nops > 0){ - return this.hist; - } + return this.hist; }, } From e3430ac7df8a7c232f6374120eb1297c7fdfe5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 2 Feb 2021 10:44:36 +0200 Subject: [PATCH 109/709] eth: check snap satelliteness, delegate drop to eth (#22235) * eth: check snap satelliteness, delegate drop to eth * eth: better handle eth/snap satellite relation, merge reg/unreg paths --- eth/handler.go | 106 +++++++------ eth/handler_eth.go | 4 +- eth/handler_eth_test.go | 4 +- eth/handler_snap.go | 8 +- eth/peer.go | 9 +- eth/peerset.go | 261 +++++++++++++-------------------- eth/protocols/eth/handler.go | 6 +- eth/protocols/eth/protocol.go | 8 +- eth/protocols/snap/handler.go | 6 +- eth/protocols/snap/protocol.go | 8 +- eth/sync.go | 4 +- eth/sync_test.go | 2 +- p2p/peer.go | 11 +- 13 files changed, 201 insertions(+), 236 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 5ae0925bb5..a5a62b894d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -218,7 +218,7 @@ func newHandler(config *handlerConfig) (*handler, error) { h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) fetchTx := func(peer string, hashes []common.Hash) error { - p := h.peers.ethPeer(peer) + p := h.peers.peer(peer) if p == nil { return errors.New("unknown peer") } @@ -229,8 +229,17 @@ func newHandler(config *handlerConfig) (*handler, error) { return h, nil } -// runEthPeer +// runEthPeer registers an eth peer into the joint eth/snap peerset, adds it to +// various subsistems and starts handling messages. func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { + // If the peer has a `snap` extension, wait for it to connect so we can have + // a uniform initialization/teardown mechanism + snap, err := h.peers.waitSnapExtension(peer) + if err != nil { + peer.Log().Error("Snapshot extension barrier failed", "err", err) + return err + } + // TODO(karalabe): Not sure why this is needed if !h.chainSync.handlePeerEvent(peer) { return p2p.DiscQuitting } @@ -251,37 +260,46 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } reject := false // reserved peer slots - if atomic.LoadUint32(&h.snapSync) == 1 && !peer.SupportsCap("snap", 1) { - // If we are running snap-sync, we want to reserve roughly half the peer - // slots for peers supporting the snap protocol. - // The logic here is; we only allow up to 5 more non-snap peers than snap-peers. - if all, snp := h.peers.Len(), h.peers.SnapLen(); all-snp > snp+5 { - reject = true + if atomic.LoadUint32(&h.snapSync) == 1 { + if snap == nil { + // If we are running snap-sync, we want to reserve roughly half the peer + // slots for peers supporting the snap protocol. + // The logic here is; we only allow up to 5 more non-snap peers than snap-peers. + if all, snp := h.peers.len(), h.peers.snapLen(); all-snp > snp+5 { + reject = true + } } } // Ignore maxPeers if this is a trusted peer if !peer.Peer.Info().Network.Trusted { - if reject || h.peers.Len() >= h.maxPeers { + if reject || h.peers.len() >= h.maxPeers { return p2p.DiscTooManyPeers } } peer.Log().Debug("Ethereum peer connected", "name", peer.Name()) // Register the peer locally - if err := h.peers.registerEthPeer(peer); err != nil { + if err := h.peers.registerPeer(peer, snap); err != nil { peer.Log().Error("Ethereum peer registration failed", "err", err) return err } defer h.removePeer(peer.ID()) - p := h.peers.ethPeer(peer.ID()) + p := h.peers.peer(peer.ID()) if p == nil { return errors.New("peer dropped during handling") } // Register the peer in the downloader. If the downloader considers it banned, we disconnect if err := h.downloader.RegisterPeer(peer.ID(), peer.Version(), peer); err != nil { + peer.Log().Error("Failed to register peer in eth syncer", "err", err) return err } + if snap != nil { + if err := h.downloader.SnapSyncer.Register(snap); err != nil { + peer.Log().Error("Failed to register peer in snap syncer", "err", err) + return err + } + } h.chainSync.handlePeerEvent(peer) // Propagate existing transactions. new transactions appearing @@ -317,25 +335,23 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return handler(peer) } -// runSnapPeer -func (h *handler) runSnapPeer(peer *snap.Peer, handler snap.Handler) error { +// runSnapExtension registers a `snap` peer into the joint eth/snap peerset and +// starts handling inbound messages. As `snap` is only a satellite protocol to +// `eth`, all subsystem registrations and lifecycle management will be done by +// the main `eth` handler to prevent strange races. +func (h *handler) runSnapExtension(peer *snap.Peer, handler snap.Handler) error { h.peerWG.Add(1) defer h.peerWG.Done() - // Register the peer locally - if err := h.peers.registerSnapPeer(peer); err != nil { - peer.Log().Error("Snapshot peer registration failed", "err", err) - return err - } - defer h.removePeer(peer.ID()) - - if err := h.downloader.SnapSyncer.Register(peer); err != nil { + if err := h.peers.registerSnapExtension(peer); err != nil { + peer.Log().Error("Snapshot extension registration failed", "err", err) return err } - // Handle incoming messages until the connection is torn down return handler(peer) } +// removePeer unregisters a peer from the downloader and fetchers, removes it from +// the set of tracked peers and closes the network connection to it. func (h *handler) removePeer(id string) { // Create a custom logger to avoid printing the entire id var logger log.Logger @@ -345,33 +361,27 @@ func (h *handler) removePeer(id string) { } else { logger = log.New("peer", id[:8]) } - // Remove the eth peer if it exists - eth := h.peers.ethPeer(id) - if eth != nil { - logger.Debug("Removing Ethereum peer") - h.downloader.UnregisterPeer(id) - h.txFetcher.Drop(id) - - if err := h.peers.unregisterEthPeer(id); err != nil { - logger.Error("Ethereum peer removal failed", "err", err) - } + // Abort if the peer does not exist + peer := h.peers.peer(id) + if peer == nil { + logger.Error("Ethereum peer removal failed", "err", errPeerNotRegistered) + return } - // Remove the snap peer if it exists - snap := h.peers.snapPeer(id) - if snap != nil { - logger.Debug("Removing Snapshot peer") + // Remove the `eth` peer if it exists + logger.Debug("Removing Ethereum peer", "snap", peer.snapExt != nil) + + // Remove the `snap` extension if it exists + if peer.snapExt != nil { h.downloader.SnapSyncer.Unregister(id) - if err := h.peers.unregisterSnapPeer(id); err != nil { - logger.Error("Snapshot peer removel failed", "err", err) - } - } - // Hard disconnect at the networking layer - if eth != nil { - eth.Peer.Disconnect(p2p.DiscUselessPeer) } - if snap != nil { - snap.Peer.Disconnect(p2p.DiscUselessPeer) + h.downloader.UnregisterPeer(id) + h.txFetcher.Drop(id) + + if err := h.peers.unregisterPeer(id); err != nil { + logger.Error("Ethereum peer removal failed", "err", err) } + // Hard disconnect at the networking layer + peer.Peer.Disconnect(p2p.DiscUselessPeer) } func (h *handler) Start(maxPeers int) { @@ -417,7 +427,7 @@ func (h *handler) Stop() { // will only announce its availability (depending what's requested). func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { hash := block.Hash() - peers := h.peers.ethPeersWithoutBlock(hash) + peers := h.peers.peersWithoutBlock(hash) // If propagation is requested, send to a subset of the peer if propagate { @@ -456,7 +466,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions, propagate bool) // Broadcast transactions to a batch of peers not knowing about it if propagate { for _, tx := range txs { - peers := h.peers.ethPeersWithoutTransaction(tx.Hash()) + peers := h.peers.peersWithoutTransaction(tx.Hash()) // Send the block to a subset of our peers transfer := peers[:int(math.Sqrt(float64(len(peers))))] @@ -472,7 +482,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions, propagate bool) } // Otherwise only broadcast the announcement to peers for _, tx := range txs { - peers := h.peers.ethPeersWithoutTransaction(tx.Hash()) + peers := h.peers.peersWithoutTransaction(tx.Hash()) for _, peer := range peers { annos[peer] = append(annos[peer], tx.Hash()) } diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 84bdac659a..3ff9f2245b 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -47,7 +47,7 @@ func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error { // PeerInfo retrieves all known `eth` information about a peer. func (h *ethHandler) PeerInfo(id enode.ID) interface{} { - if p := h.peers.ethPeer(id.String()); p != nil { + if p := h.peers.peer(id.String()); p != nil { return p.info() } return nil @@ -107,7 +107,7 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // handleHeaders is invoked from a peer's message handler when it transmits a batch // of headers for the local node to process. func (h *ethHandler) handleHeaders(peer *eth.Peer, headers []*types.Header) error { - p := h.peers.ethPeer(peer.ID()) + p := h.peers.peer(peer.ID()) if p == nil { return errors.New("unregistered during callback") } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 0e5c0c90ee..5f5d4e9e82 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -574,11 +574,11 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo // Verify that the remote peer is maintained or dropped if drop { - if peers := handler.handler.peers.Len(); peers != 0 { + if peers := handler.handler.peers.len(); peers != 0 { t.Fatalf("peer count mismatch: have %d, want %d", peers, 0) } } else { - if peers := handler.handler.peers.Len(); peers != 1 { + if peers := handler.handler.peers.len(); peers != 1 { t.Fatalf("peer count mismatch: have %d, want %d", peers, 1) } } diff --git a/eth/handler_snap.go b/eth/handler_snap.go index 25975bf60b..767416ffd6 100644 --- a/eth/handler_snap.go +++ b/eth/handler_snap.go @@ -30,13 +30,15 @@ func (h *snapHandler) Chain() *core.BlockChain { return h.chain } // RunPeer is invoked when a peer joins on the `snap` protocol. func (h *snapHandler) RunPeer(peer *snap.Peer, hand snap.Handler) error { - return (*handler)(h).runSnapPeer(peer, hand) + return (*handler)(h).runSnapExtension(peer, hand) } // PeerInfo retrieves all known `snap` information about a peer. func (h *snapHandler) PeerInfo(id enode.ID) interface{} { - if p := h.peers.snapPeer(id.String()); p != nil { - return p.info() + if p := h.peers.peer(id.String()); p != nil { + if p.snapExt != nil { + return p.snapExt.info() + } } return nil } diff --git a/eth/peer.go b/eth/peer.go index 6970c8afd3..1cea9c640e 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -36,9 +36,11 @@ type ethPeerInfo struct { // ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. type ethPeer struct { *eth.Peer + snapExt *snapPeer // Satellite `snap` connection - syncDrop *time.Timer // Connection dropper if `eth` sync progress isn't validated in time - lock sync.RWMutex // Mutex protecting the internal fields + syncDrop *time.Timer // Connection dropper if `eth` sync progress isn't validated in time + snapWait chan struct{} // Notification channel for snap connections + lock sync.RWMutex // Mutex protecting the internal fields } // info gathers and returns some `eth` protocol metadata known about a peer. @@ -61,9 +63,6 @@ type snapPeerInfo struct { // snapPeer is a wrapper around snap.Peer to maintain a few extra metadata. type snapPeer struct { *snap.Peer - - ethDrop *time.Timer // Connection dropper if `eth` doesn't connect in time - lock sync.RWMutex // Mutex protecting the internal fields } // info gathers and returns some `snap` protocol metadata known about a peer. diff --git a/eth/peerset.go b/eth/peerset.go index 663c5ce36b..f0657e140b 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -20,12 +20,10 @@ import ( "errors" "math/big" "sync" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/p2p" ) @@ -42,22 +40,19 @@ var ( // a peer set, but no peer with the given id exists. errPeerNotRegistered = errors.New("peer not registered") - // ethConnectTimeout is the `snap` timeout for `eth` to connect too. - ethConnectTimeout = 3 * time.Second + // errSnapWithoutEth is returned if a peer attempts to connect only on the + // snap protocol without advertizing the eth main protocol. + errSnapWithoutEth = errors.New("peer connected on snap without compatible eth support") ) // peerSet represents the collection of active peers currently participating in -// the `eth` or `snap` protocols. +// the `eth` protocol, with or without the `snap` extension. type peerSet struct { - ethPeers map[string]*ethPeer // Peers connected on the `eth` protocol - snapPeers map[string]*snapPeer // Peers connected on the `snap` protocol + peers map[string]*ethPeer // Peers connected on the `eth` protocol + snapPeers int // Number of `snap` compatible peers for connection prioritization - ethJoinFeed event.Feed // Events when an `eth` peer successfully joins - ethDropFeed event.Feed // Events when an `eth` peer gets dropped - snapJoinFeed event.Feed // Events when a `snap` peer joins on both `eth` and `snap` - snapDropFeed event.Feed // Events when a `snap` peer gets dropped (only if fully joined) - - scope event.SubscriptionScope // Subscription group to unsubscribe everyone at once + snapWait map[string]chan *snap.Peer // Peers connected on `eth` waiting for their snap extension + snapPend map[string]*snap.Peer // Peers connected on the `snap` protocol, but not yet on `eth` lock sync.RWMutex closed bool @@ -66,176 +61,134 @@ type peerSet struct { // newPeerSet creates a new peer set to track the active participants. func newPeerSet() *peerSet { return &peerSet{ - ethPeers: make(map[string]*ethPeer), - snapPeers: make(map[string]*snapPeer), + peers: make(map[string]*ethPeer), + snapWait: make(map[string]chan *snap.Peer), + snapPend: make(map[string]*snap.Peer), } } -// subscribeEthJoin registers a subscription for peers joining (and completing -// the handshake) on the `eth` protocol. -func (ps *peerSet) subscribeEthJoin(ch chan<- *eth.Peer) event.Subscription { - return ps.scope.Track(ps.ethJoinFeed.Subscribe(ch)) -} - -// subscribeEthDrop registers a subscription for peers being dropped from the -// `eth` protocol. -func (ps *peerSet) subscribeEthDrop(ch chan<- *eth.Peer) event.Subscription { - return ps.scope.Track(ps.ethDropFeed.Subscribe(ch)) -} - -// subscribeSnapJoin registers a subscription for peers joining (and completing -// the `eth` join) on the `snap` protocol. -func (ps *peerSet) subscribeSnapJoin(ch chan<- *snap.Peer) event.Subscription { - return ps.scope.Track(ps.snapJoinFeed.Subscribe(ch)) -} - -// subscribeSnapDrop registers a subscription for peers being dropped from the -// `snap` protocol. -func (ps *peerSet) subscribeSnapDrop(ch chan<- *snap.Peer) event.Subscription { - return ps.scope.Track(ps.snapDropFeed.Subscribe(ch)) -} - -// registerEthPeer injects a new `eth` peer into the working set, or returns an -// error if the peer is already known. The peer is announced on the `eth` join -// feed and if it completes a pending `snap` peer, also on that feed. -func (ps *peerSet) registerEthPeer(peer *eth.Peer) error { - ps.lock.Lock() - if ps.closed { - ps.lock.Unlock() - return errPeerSetClosed +// registerSnapExtension unblocks an already connected `eth` peer waiting for its +// `snap` extension, or if no such peer exists, tracks the extension for the time +// being until the `eth` main protocol starts looking for it. +func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { + // Reject the peer if it advertises `snap` without `eth` as `snap` is only a + // satellite protocol meaningful with the chain selection of `eth` + if !peer.SupportsCap(eth.ProtocolName, eth.ProtocolVersions) { + return errSnapWithoutEth } + // Ensure nobody can double connect + ps.lock.Lock() + defer ps.lock.Unlock() + id := peer.ID() - if _, ok := ps.ethPeers[id]; ok { - ps.lock.Unlock() - return errPeerAlreadyRegistered + if _, ok := ps.peers[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as existing ones } - ps.ethPeers[id] = ðPeer{Peer: peer} - - snap, ok := ps.snapPeers[id] - ps.lock.Unlock() - - if ok { - // Previously dangling `snap` peer, stop it's timer since `eth` connected - snap.lock.Lock() - if snap.ethDrop != nil { - snap.ethDrop.Stop() - snap.ethDrop = nil - } - snap.lock.Unlock() + if _, ok := ps.snapPend[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as pending ones } - ps.ethJoinFeed.Send(peer) - if ok { - ps.snapJoinFeed.Send(snap.Peer) + // Inject the peer into an `eth` counterpart is available, otherwise save for later + if wait, ok := ps.snapWait[id]; ok { + delete(ps.snapWait, id) + wait <- peer + return nil } + ps.snapPend[id] = peer return nil } -// unregisterEthPeer removes a remote peer from the active set, disabling any further -// actions to/from that particular entity. The drop is announced on the `eth` drop -// feed and also on the `snap` feed if the eth/snap duality was broken just now. -func (ps *peerSet) unregisterEthPeer(id string) error { +// waitExtensions blocks until all satellite protocols are connected and tracked +// by the peerset. +func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { + // If the peer does not support a compatible `snap`, don't wait + if !peer.SupportsCap(snap.ProtocolName, snap.ProtocolVersions) { + return nil, nil + } + // Ensure nobody can double connect ps.lock.Lock() - eth, ok := ps.ethPeers[id] - if !ok { + + id := peer.ID() + if _, ok := ps.peers[id]; ok { ps.lock.Unlock() - return errPeerNotRegistered + return nil, errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.snapWait[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as pending ones } - delete(ps.ethPeers, id) + // If `snap` already connected, retrieve the peer from the pending set + if snap, ok := ps.snapPend[id]; ok { + delete(ps.snapPend, id) - snap, ok := ps.snapPeers[id] + ps.lock.Unlock() + return snap, nil + } + // Otherwise wait for `snap` to connect concurrently + wait := make(chan *snap.Peer) + ps.snapWait[id] = wait ps.lock.Unlock() - ps.ethDropFeed.Send(eth) - if ok { - ps.snapDropFeed.Send(snap) - } - return nil + return <-wait, nil } -// registerSnapPeer injects a new `snap` peer into the working set, or returns -// an error if the peer is already known. The peer is announced on the `snap` -// join feed if it completes an existing `eth` peer. -// -// If the peer isn't yet connected on `eth` and fails to do so within a given -// amount of time, it is dropped. This enforces that `snap` is an extension to -// `eth`, not a standalone leeching protocol. -func (ps *peerSet) registerSnapPeer(peer *snap.Peer) error { +// registerPeer injects a new `eth` peer into the working set, or returns an error +// if the peer is already known. +func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer) error { + // Start tracking the new peer ps.lock.Lock() + defer ps.lock.Unlock() + if ps.closed { - ps.lock.Unlock() return errPeerSetClosed } id := peer.ID() - if _, ok := ps.snapPeers[id]; ok { - ps.lock.Unlock() + if _, ok := ps.peers[id]; ok { return errPeerAlreadyRegistered } - ps.snapPeers[id] = &snapPeer{Peer: peer} - - _, ok := ps.ethPeers[id] - if !ok { - // Dangling `snap` peer, start a timer to drop if `eth` doesn't connect - ps.snapPeers[id].ethDrop = time.AfterFunc(ethConnectTimeout, func() { - peer.Log().Warn("Snapshot peer missing eth, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) - peer.Disconnect(p2p.DiscUselessPeer) - }) + eth := ðPeer{ + Peer: peer, } - ps.lock.Unlock() - - if ok { - ps.snapJoinFeed.Send(peer) + if ext != nil { + eth.snapExt = &snapPeer{ext} + ps.snapPeers++ } + ps.peers[id] = eth return nil } -// unregisterSnapPeer removes a remote peer from the active set, disabling any -// further actions to/from that particular entity. The drop is announced on the -// `snap` drop feed. -func (ps *peerSet) unregisterSnapPeer(id string) error { +// unregisterPeer removes a remote peer from the active set, disabling any further +// actions to/from that particular entity. +func (ps *peerSet) unregisterPeer(id string) error { ps.lock.Lock() - peer, ok := ps.snapPeers[id] + defer ps.lock.Unlock() + + peer, ok := ps.peers[id] if !ok { - ps.lock.Unlock() return errPeerNotRegistered } - delete(ps.snapPeers, id) - ps.lock.Unlock() - - peer.lock.Lock() - if peer.ethDrop != nil { - peer.ethDrop.Stop() - peer.ethDrop = nil + delete(ps.peers, id) + if peer.snapExt != nil { + ps.snapPeers-- } - peer.lock.Unlock() - - ps.snapDropFeed.Send(peer) return nil } -// ethPeer retrieves the registered `eth` peer with the given id. -func (ps *peerSet) ethPeer(id string) *ethPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return ps.ethPeers[id] -} - -// snapPeer retrieves the registered `snap` peer with the given id. -func (ps *peerSet) snapPeer(id string) *snapPeer { +// peer retrieves the registered peer with the given id. +func (ps *peerSet) peer(id string) *ethPeer { ps.lock.RLock() defer ps.lock.RUnlock() - return ps.snapPeers[id] + return ps.peers[id] } -// ethPeersWithoutBlock retrieves a list of `eth` peers that do not have a given -// block in their set of known hashes so it might be propagated to them. -func (ps *peerSet) ethPeersWithoutBlock(hash common.Hash) []*ethPeer { +// peersWithoutBlock retrieves a list of peers that do not have a given block in +// their set of known hashes so it might be propagated to them. +func (ps *peerSet) peersWithoutBlock(hash common.Hash) []*ethPeer { ps.lock.RLock() defer ps.lock.RUnlock() - list := make([]*ethPeer, 0, len(ps.ethPeers)) - for _, p := range ps.ethPeers { + list := make([]*ethPeer, 0, len(ps.peers)) + for _, p := range ps.peers { if !p.KnownBlock(hash) { list = append(list, p) } @@ -243,14 +196,14 @@ func (ps *peerSet) ethPeersWithoutBlock(hash common.Hash) []*ethPeer { return list } -// ethPeersWithoutTransaction retrieves a list of `eth` peers that do not have a -// given transaction in their set of known hashes. -func (ps *peerSet) ethPeersWithoutTransaction(hash common.Hash) []*ethPeer { +// peersWithoutTransaction retrieves a list of peers that do not have a given +// transaction in their set of known hashes. +func (ps *peerSet) peersWithoutTransaction(hash common.Hash) []*ethPeer { ps.lock.RLock() defer ps.lock.RUnlock() - list := make([]*ethPeer, 0, len(ps.ethPeers)) - for _, p := range ps.ethPeers { + list := make([]*ethPeer, 0, len(ps.peers)) + for _, p := range ps.peers { if !p.KnownTransaction(hash) { list = append(list, p) } @@ -258,28 +211,27 @@ func (ps *peerSet) ethPeersWithoutTransaction(hash common.Hash) []*ethPeer { return list } -// Len returns if the current number of `eth` peers in the set. Since the `snap` +// len returns if the current number of `eth` peers in the set. Since the `snap` // peers are tied to the existence of an `eth` connection, that will always be a // subset of `eth`. -func (ps *peerSet) Len() int { +func (ps *peerSet) len() int { ps.lock.RLock() defer ps.lock.RUnlock() - return len(ps.ethPeers) + return len(ps.peers) } -// SnapLen returns if the current number of `snap` peers in the set. Since the `snap` -// peers are tied to the existence of an `eth` connection, that will always be a -// subset of `eth`. -func (ps *peerSet) SnapLen() int { +// snapLen returns if the current number of `snap` peers in the set. +func (ps *peerSet) snapLen() int { ps.lock.RLock() defer ps.lock.RUnlock() - return len(ps.snapPeers) + + return ps.snapPeers } -// ethPeerWithHighestTD retrieves the known peer with the currently highest total +// peerWithHighestTD retrieves the known peer with the currently highest total // difficulty. -func (ps *peerSet) ethPeerWithHighestTD() *eth.Peer { +func (ps *peerSet) peerWithHighestTD() *eth.Peer { ps.lock.RLock() defer ps.lock.RUnlock() @@ -287,7 +239,7 @@ func (ps *peerSet) ethPeerWithHighestTD() *eth.Peer { bestPeer *eth.Peer bestTd *big.Int ) - for _, p := range ps.ethPeers { + for _, p := range ps.peers { if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { bestPeer, bestTd = p.Peer, td } @@ -300,10 +252,7 @@ func (ps *peerSet) close() { ps.lock.Lock() defer ps.lock.Unlock() - for _, p := range ps.ethPeers { - p.Disconnect(p2p.DiscQuitting) - } - for _, p := range ps.snapPeers { + for _, p := range ps.peers { p.Disconnect(p2p.DiscQuitting) } ps.closed = true diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 25ddcd93ec..e32008fb41 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -103,12 +103,12 @@ type TxPool interface { // MakeProtocols constructs the P2P protocol definitions for `eth`. func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2p.Protocol { - protocols := make([]p2p.Protocol, len(protocolVersions)) - for i, version := range protocolVersions { + protocols := make([]p2p.Protocol, len(ProtocolVersions)) + for i, version := range ProtocolVersions { version := version // Closure protocols[i] = p2p.Protocol{ - Name: protocolName, + Name: ProtocolName, Version: version, Length: protocolLengths[version], Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 63d3494ec4..9fff64b72a 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -34,13 +34,13 @@ const ( ETH65 = 65 ) -// protocolName is the official short name of the `eth` protocol used during +// ProtocolName is the official short name of the `eth` protocol used during // devp2p capability negotiation. -const protocolName = "eth" +const ProtocolName = "eth" -// protocolVersions are the supported versions of the `eth` protocol (first +// ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var protocolVersions = []uint{ETH65, ETH64} +var ProtocolVersions = []uint{ETH65, ETH64} // protocolLengths are the number of implemented message corresponding to // different protocol versions. diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 36322e648b..24c8599552 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -77,12 +77,12 @@ type Backend interface { // MakeProtocols constructs the P2P protocol definitions for `snap`. func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { - protocols := make([]p2p.Protocol, len(protocolVersions)) - for i, version := range protocolVersions { + protocols := make([]p2p.Protocol, len(ProtocolVersions)) + for i, version := range ProtocolVersions { version := version // Closure protocols[i] = p2p.Protocol{ - Name: protocolName, + Name: ProtocolName, Version: version, Length: protocolLengths[version], Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { diff --git a/eth/protocols/snap/protocol.go b/eth/protocols/snap/protocol.go index f1a25a2066..a74142cafb 100644 --- a/eth/protocols/snap/protocol.go +++ b/eth/protocols/snap/protocol.go @@ -30,13 +30,13 @@ const ( snap1 = 1 ) -// protocolName is the official short name of the `snap` protocol used during +// ProtocolName is the official short name of the `snap` protocol used during // devp2p capability negotiation. -const protocolName = "snap" +const ProtocolName = "snap" -// protocolVersions are the supported versions of the `snap` protocol (first +// ProtocolVersions are the supported versions of the `snap` protocol (first // is primary). -var protocolVersions = []uint{snap1} +var ProtocolVersions = []uint{snap1} // protocolLengths are the number of implemented message corresponding to // different protocol versions. diff --git a/eth/sync.go b/eth/sync.go index eedb8b7476..dc72e88388 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -247,11 +247,11 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { } else if minPeers > cs.handler.maxPeers { minPeers = cs.handler.maxPeers } - if cs.handler.peers.Len() < minPeers { + if cs.handler.peers.len() < minPeers { return nil } // We have enough peers, check TD - peer := cs.handler.peers.ethPeerWithHighestTD() + peer := cs.handler.peers.peerWithHighestTD() if peer == nil { return nil } diff --git a/eth/sync_test.go b/eth/sync_test.go index 473e19518f..9cc806b18a 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -70,7 +70,7 @@ func testFastSyncDisabling(t *testing.T, protocol uint) { time.Sleep(250 * time.Millisecond) // Check that fast sync was disabled - op := peerToSyncOp(downloader.FastSync, empty.handler.peers.ethPeerWithHighestTD()) + op := peerToSyncOp(downloader.FastSync, empty.handler.peers.peerWithHighestTD()) if err := empty.handler.doSync(op); err != nil { t.Fatal("sync failed:", err) } diff --git a/p2p/peer.go b/p2p/peer.go index 43ccef5c43..08881e2583 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -158,11 +158,16 @@ func (p *Peer) Caps() []Cap { return p.rw.caps } -// SupportsCap returns true if the peer supports the given protocol/version -func (p *Peer) SupportsCap(protocol string, version uint) bool { +// SupportsCap returns true if the peer supports any of the enumerated versions +// of a specific protocol. +func (p *Peer) SupportsCap(protocol string, versions []uint) bool { for _, cap := range p.rw.caps { if cap.Name == protocol { - return version <= cap.Version + for _, ver := range versions { + if cap.Version == ver { + return true + } + } } } return false From 4eae0c6b6f2bcaa450df50077bb991e3bbbb106f Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 2 Feb 2021 10:05:46 +0100 Subject: [PATCH 110/709] cmd/geth, node: allow configuring JSON-RPC on custom path prefix (#22184) This change allows users to set a custom path prefix on which to mount the http-rpc or ws-rpc handlers via the new flags --http.rpcprefix and --ws.rpcprefix. Fixes #21826 Co-authored-by: Felix Lange --- cmd/geth/main.go | 2 + cmd/geth/usage.go | 2 + cmd/utils/flags.go | 18 ++++++ go.sum | 5 -- graphql/graphql_test.go | 14 +--- node/api_test.go | 3 + node/config.go | 6 ++ node/node.go | 20 ++++-- node/node_test.go | 107 ++++++++++++++++++++++++++++++- node/rpcstack.go | 71 +++++++++++++++++---- node/rpcstack_test.go | 137 +++++++++++++++++++++++++++++++--------- 11 files changed, 320 insertions(+), 65 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 86dc6f40fe..11829cbad9 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -181,6 +181,7 @@ var ( utils.GraphQLCORSDomainFlag, utils.GraphQLVirtualHostsFlag, utils.HTTPApiFlag, + utils.HTTPPathPrefixFlag, utils.LegacyRPCApiFlag, utils.WSEnabledFlag, utils.WSListenAddrFlag, @@ -190,6 +191,7 @@ var ( utils.WSApiFlag, utils.LegacyWSApiFlag, utils.WSAllowedOriginsFlag, + utils.WSPathPrefixFlag, utils.LegacyWSAllowedOriginsFlag, utils.IPCDisabledFlag, utils.IPCPathFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index ba311bf7f6..0ed31d7da6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -138,12 +138,14 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.HTTPListenAddrFlag, utils.HTTPPortFlag, utils.HTTPApiFlag, + utils.HTTPPathPrefixFlag, utils.HTTPCORSDomainFlag, utils.HTTPVirtualHostsFlag, utils.WSEnabledFlag, utils.WSListenAddrFlag, utils.WSPortFlag, utils.WSApiFlag, + utils.WSPathPrefixFlag, utils.WSAllowedOriginsFlag, utils.GraphQLEnabledFlag, utils.GraphQLCORSDomainFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8f9f68ba68..aa5180dd9f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -531,6 +531,11 @@ var ( Usage: "API's offered over the HTTP-RPC interface", Value: "", } + HTTPPathPrefixFlag = cli.StringFlag{ + Name: "http.rpcprefix", + Usage: "HTTP path path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", + Value: "", + } GraphQLEnabledFlag = cli.BoolFlag{ Name: "graphql", Usage: "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.", @@ -569,6 +574,11 @@ var ( Usage: "Origins from which to accept websockets requests", Value: "", } + WSPathPrefixFlag = cli.StringFlag{ + Name: "ws.rpcprefix", + Usage: "HTTP path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", + Value: "", + } ExecFlag = cli.StringFlag{ Name: "exec", Usage: "Execute JavaScript statement", @@ -946,6 +956,10 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(HTTPVirtualHostsFlag.Name) { cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(HTTPVirtualHostsFlag.Name)) } + + if ctx.GlobalIsSet(HTTPPathPrefixFlag.Name) { + cfg.HTTPPathPrefix = ctx.GlobalString(HTTPPathPrefixFlag.Name) + } } // setGraphQL creates the GraphQL listener interface string from the set @@ -995,6 +1009,10 @@ func setWS(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(WSApiFlag.Name) { cfg.WSModules = SplitAndTrim(ctx.GlobalString(WSApiFlag.Name)) } + + if ctx.GlobalIsSet(WSPathPrefixFlag.Name) { + cfg.WSPathPrefix = ctx.GlobalString(WSPathPrefixFlag.Name) + } } // setIPC creates an IPC path configuration from the set command line flags, diff --git a/go.sum b/go.sum index 4b799d77fa..2c16d81f04 100644 --- a/go.sum +++ b/go.sum @@ -433,7 +433,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -501,7 +500,6 @@ golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapK golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= @@ -549,15 +547,12 @@ google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyz google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 71320012d5..a88c9b30b1 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" + + "github.com/stretchr/testify/assert" ) func TestBuildSchema(t *testing.T) { @@ -166,18 +168,8 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { if err != nil { t.Fatalf("could not post: %v", err) } - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatalf("could not read from response body: %v", err) - } - resp.Body.Close() // make sure the request is not handled successfully - if want, have := "404 page not found\n", string(bodyBytes); have != want { - t.Errorf("have:\n%v\nwant:\n%v", have, want) - } - if want, have := 404, resp.StatusCode; want != have { - t.Errorf("wrong statuscode, have:\n%v\nwant:%v", have, want) - } + assert.Equal(t, http.StatusNotFound, resp.StatusCode) } func createNode(t *testing.T, gqlEnabled bool) *node.Node { diff --git a/node/api_test.go b/node/api_test.go index a07ce833c4..9c3fa3a31d 100644 --- a/node/api_test.go +++ b/node/api_test.go @@ -244,7 +244,10 @@ func TestStartRPC(t *testing.T) { } for _, test := range tests { + test := test t.Run(test.name, func(t *testing.T) { + t.Parallel() + // Apply some sane defaults. config := test.cfg // config.Logger = testlog.Logger(t, log.LvlDebug) diff --git a/node/config.go b/node/config.go index 61e41cd7d0..447a69505c 100644 --- a/node/config.go +++ b/node/config.go @@ -139,6 +139,9 @@ type Config struct { // interface. HTTPTimeouts rpc.HTTPTimeouts + // HTTPPathPrefix specifies a path prefix on which http-rpc is to be served. + HTTPPathPrefix string `toml:",omitempty"` + // WSHost is the host interface on which to start the websocket RPC server. If // this field is empty, no websocket API endpoint will be started. WSHost string @@ -148,6 +151,9 @@ type Config struct { // ephemeral nodes). WSPort int `toml:",omitempty"` + // WSPathPrefix specifies a path prefix on which ws-rpc is to be served. + WSPathPrefix string `toml:",omitempty"` + // WSOrigins is the list of domain to accept websocket requests from. Please be // aware that the server can only act upon the HTTP request the client sends and // cannot verify the validity of the request header. diff --git a/node/node.go b/node/node.go index b58594ef1d..2ed4c31f60 100644 --- a/node/node.go +++ b/node/node.go @@ -135,6 +135,14 @@ func New(conf *Config) (*Node, error) { node.server.Config.NodeDatabase = node.config.NodeDB() } + // Check HTTP/WS prefixes are valid. + if err := validatePrefix("HTTP", conf.HTTPPathPrefix); err != nil { + return nil, err + } + if err := validatePrefix("WebSocket", conf.WSPathPrefix); err != nil { + return nil, err + } + // Configure RPC servers. node.http = newHTTPServer(node.log, conf.HTTPTimeouts) node.ws = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) @@ -346,6 +354,7 @@ func (n *Node) startRPC() error { CorsAllowedOrigins: n.config.HTTPCors, Vhosts: n.config.HTTPVirtualHosts, Modules: n.config.HTTPModules, + prefix: n.config.HTTPPathPrefix, } if err := n.http.setListenAddr(n.config.HTTPHost, n.config.HTTPPort); err != nil { return err @@ -361,6 +370,7 @@ func (n *Node) startRPC() error { config := wsConfig{ Modules: n.config.WSModules, Origins: n.config.WSOrigins, + prefix: n.config.WSPathPrefix, } if err := server.setListenAddr(n.config.WSHost, n.config.WSPort); err != nil { return err @@ -457,6 +467,7 @@ func (n *Node) RegisterHandler(name, path string, handler http.Handler) { if n.state != initializingState { panic("can't register HTTP handler on running/stopped node") } + n.http.mux.Handle(path, handler) n.http.handlerNames[path] = name } @@ -513,17 +524,18 @@ func (n *Node) IPCEndpoint() string { return n.ipc.endpoint } -// HTTPEndpoint returns the URL of the HTTP server. +// HTTPEndpoint returns the URL of the HTTP server. Note that this URL does not +// contain the JSON-RPC path prefix set by HTTPPathPrefix. func (n *Node) HTTPEndpoint() string { return "http://" + n.http.listenAddr() } -// WSEndpoint retrieves the current WS endpoint used by the protocol stack. +// WSEndpoint returns the current JSON-RPC over WebSocket endpoint. func (n *Node) WSEndpoint() string { if n.http.wsAllowed() { - return "ws://" + n.http.listenAddr() + return "ws://" + n.http.listenAddr() + n.http.wsConfig.prefix } - return "ws://" + n.ws.listenAddr() + return "ws://" + n.ws.listenAddr() + n.ws.wsConfig.prefix } // EventMux retrieves the event multiplexer used by all the network services in diff --git a/node/node_test.go b/node/node_test.go index 8f306ef021..6731dbac1f 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -390,7 +390,7 @@ func TestLifecycleTerminationGuarantee(t *testing.T) { } // Tests whether a handler can be successfully mounted on the canonical HTTP server -// on the given path +// on the given prefix func TestRegisterHandler_Successful(t *testing.T) { node := createNode(t, 7878, 7979) @@ -483,7 +483,112 @@ func TestWebsocketHTTPOnSeparatePort_WSRequest(t *testing.T) { if !checkRPC(node.HTTPEndpoint()) { t.Fatalf("http request failed") } +} + +type rpcPrefixTest struct { + httpPrefix, wsPrefix string + // These lists paths on which JSON-RPC should be served / not served. + wantHTTP []string + wantNoHTTP []string + wantWS []string + wantNoWS []string +} + +func TestNodeRPCPrefix(t *testing.T) { + t.Parallel() + + tests := []rpcPrefixTest{ + // both off + { + httpPrefix: "", wsPrefix: "", + wantHTTP: []string{"/", "/?p=1"}, + wantNoHTTP: []string{"/test", "/test?p=1"}, + wantWS: []string{"/", "/?p=1"}, + wantNoWS: []string{"/test", "/test?p=1"}, + }, + // only http prefix + { + httpPrefix: "/testprefix", wsPrefix: "", + wantHTTP: []string{"/testprefix", "/testprefix?p=1", "/testprefix/x", "/testprefix/x?p=1"}, + wantNoHTTP: []string{"/", "/?p=1", "/test", "/test?p=1"}, + wantWS: []string{"/", "/?p=1"}, + wantNoWS: []string{"/testprefix", "/testprefix?p=1", "/test", "/test?p=1"}, + }, + // only ws prefix + { + httpPrefix: "", wsPrefix: "/testprefix", + wantHTTP: []string{"/", "/?p=1"}, + wantNoHTTP: []string{"/testprefix", "/testprefix?p=1", "/test", "/test?p=1"}, + wantWS: []string{"/testprefix", "/testprefix?p=1", "/testprefix/x", "/testprefix/x?p=1"}, + wantNoWS: []string{"/", "/?p=1", "/test", "/test?p=1"}, + }, + // both set + { + httpPrefix: "/testprefix", wsPrefix: "/testprefix", + wantHTTP: []string{"/testprefix", "/testprefix?p=1", "/testprefix/x", "/testprefix/x?p=1"}, + wantNoHTTP: []string{"/", "/?p=1", "/test", "/test?p=1"}, + wantWS: []string{"/testprefix", "/testprefix?p=1", "/testprefix/x", "/testprefix/x?p=1"}, + wantNoWS: []string{"/", "/?p=1", "/test", "/test?p=1"}, + }, + } + + for _, test := range tests { + test := test + name := fmt.Sprintf("http=%s ws=%s", test.httpPrefix, test.wsPrefix) + t.Run(name, func(t *testing.T) { + cfg := &Config{ + HTTPHost: "127.0.0.1", + HTTPPathPrefix: test.httpPrefix, + WSHost: "127.0.0.1", + WSPathPrefix: test.wsPrefix, + } + node, err := New(cfg) + if err != nil { + t.Fatal("can't create node:", err) + } + defer node.Close() + if err := node.Start(); err != nil { + t.Fatal("can't start node:", err) + } + test.check(t, node) + }) + } +} + +func (test rpcPrefixTest) check(t *testing.T, node *Node) { + t.Helper() + httpBase := "http://" + node.http.listenAddr() + wsBase := "ws://" + node.http.listenAddr() + + if node.WSEndpoint() != wsBase+test.wsPrefix { + t.Errorf("Error: node has wrong WSEndpoint %q", node.WSEndpoint()) + } + + for _, path := range test.wantHTTP { + resp := rpcRequest(t, httpBase+path) + if resp.StatusCode != 200 { + t.Errorf("Error: %s: bad status code %d, want 200", path, resp.StatusCode) + } + } + for _, path := range test.wantNoHTTP { + resp := rpcRequest(t, httpBase+path) + if resp.StatusCode != 404 { + t.Errorf("Error: %s: bad status code %d, want 404", path, resp.StatusCode) + } + } + for _, path := range test.wantWS { + err := wsRequest(t, wsBase+path, "") + if err != nil { + t.Errorf("Error: %s: WebSocket connection failed: %v", path, err) + } + } + for _, path := range test.wantNoWS { + err := wsRequest(t, wsBase+path, "") + if err == nil { + t.Errorf("Error: %s: WebSocket connection succeeded for path in wantNoWS", path) + } + } } func createNode(t *testing.T, httpPort, wsPort int) *Node { diff --git a/node/rpcstack.go b/node/rpcstack.go index 81e054ec99..d693bb0bbd 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -39,12 +39,14 @@ type httpConfig struct { Modules []string CorsAllowedOrigins []string Vhosts []string + prefix string // path prefix on which to mount http handler } // wsConfig is the JSON-RPC/Websocket configuration type wsConfig struct { Origins []string Modules []string + prefix string // path prefix on which to mount ws handler } type rpcHandler struct { @@ -62,6 +64,7 @@ type httpServer struct { listener net.Listener // non-nil when server is running // HTTP RPC handler things. + httpConfig httpConfig httpHandler atomic.Value // *rpcHandler @@ -79,6 +82,7 @@ type httpServer struct { func newHTTPServer(log log.Logger, timeouts rpc.HTTPTimeouts) *httpServer { h := &httpServer{log: log, timeouts: timeouts, handlerNames: make(map[string]string)} + h.httpHandler.Store((*rpcHandler)(nil)) h.wsHandler.Store((*rpcHandler)(nil)) return h @@ -142,12 +146,17 @@ func (h *httpServer) start() error { // if server is websocket only, return after logging if h.wsAllowed() && !h.rpcAllowed() { - h.log.Info("WebSocket enabled", "url", fmt.Sprintf("ws://%v", listener.Addr())) + url := fmt.Sprintf("ws://%v", listener.Addr()) + if h.wsConfig.prefix != "" { + url += h.wsConfig.prefix + } + h.log.Info("WebSocket enabled", "url", url) return nil } // Log http endpoint. h.log.Info("HTTP server started", "endpoint", listener.Addr(), + "prefix", h.httpConfig.prefix, "cors", strings.Join(h.httpConfig.CorsAllowedOrigins, ","), "vhosts", strings.Join(h.httpConfig.Vhosts, ","), ) @@ -170,26 +179,60 @@ func (h *httpServer) start() error { } func (h *httpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - rpc := h.httpHandler.Load().(*rpcHandler) - if r.RequestURI == "/" { - // Serve JSON-RPC on the root path. - ws := h.wsHandler.Load().(*rpcHandler) - if ws != nil && isWebsocket(r) { + // check if ws request and serve if ws enabled + ws := h.wsHandler.Load().(*rpcHandler) + if ws != nil && isWebsocket(r) { + if checkPath(r, h.wsConfig.prefix) { ws.ServeHTTP(w, r) + } + return + } + // if http-rpc is enabled, try to serve request + rpc := h.httpHandler.Load().(*rpcHandler) + if rpc != nil { + // First try to route in the mux. + // Requests to a path below root are handled by the mux, + // which has all the handlers registered via Node.RegisterHandler. + // These are made available when RPC is enabled. + muxHandler, pattern := h.mux.Handler(r) + if pattern != "" { + muxHandler.ServeHTTP(w, r) return } - if rpc != nil { + + if checkPath(r, h.httpConfig.prefix) { rpc.ServeHTTP(w, r) return } - } else if rpc != nil { - // Requests to a path below root are handled by the mux, - // which has all the handlers registered via Node.RegisterHandler. - // These are made available when RPC is enabled. - h.mux.ServeHTTP(w, r) - return } - w.WriteHeader(404) + w.WriteHeader(http.StatusNotFound) +} + +// checkPath checks whether a given request URL matches a given path prefix. +func checkPath(r *http.Request, path string) bool { + // if no prefix has been specified, request URL must be on root + if path == "" { + return r.URL.Path == "/" + } + // otherwise, check to make sure prefix matches + return len(r.URL.Path) >= len(path) && r.URL.Path[:len(path)] == path +} + +// validatePrefix checks if 'path' is a valid configuration value for the RPC prefix option. +func validatePrefix(what, path string) error { + if path == "" { + return nil + } + if path[0] != '/' { + return fmt.Errorf(`%s RPC path prefix %q does not contain leading "/"`, what, path) + } + if strings.ContainsAny(path, "?#") { + // This is just to avoid confusion. While these would match correctly (i.e. they'd + // match if URL-escaped into path), it's not easy to understand for users when + // setting that on the command line. + return fmt.Errorf("%s RPC path prefix %q contains URL meta-characters", what, path) + } + return nil } // stop shuts down the HTTP server. diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 8267fb2f1d..f92f0ba396 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -18,7 +18,10 @@ package node import ( "bytes" + "fmt" "net/http" + "net/url" + "strconv" "strings" "testing" @@ -31,25 +34,27 @@ import ( // TestCorsHandler makes sure CORS are properly handled on the http server. func TestCorsHandler(t *testing.T) { - srv := createAndStartServer(t, httpConfig{CorsAllowedOrigins: []string{"test", "test.com"}}, false, wsConfig{}) + srv := createAndStartServer(t, &httpConfig{CorsAllowedOrigins: []string{"test", "test.com"}}, false, &wsConfig{}) defer srv.stop() + url := "http://" + srv.listenAddr() - resp := testRequest(t, "origin", "test.com", "", srv) + resp := rpcRequest(t, url, "origin", "test.com") assert.Equal(t, "test.com", resp.Header.Get("Access-Control-Allow-Origin")) - resp2 := testRequest(t, "origin", "bad", "", srv) + resp2 := rpcRequest(t, url, "origin", "bad") assert.Equal(t, "", resp2.Header.Get("Access-Control-Allow-Origin")) } // TestVhosts makes sure vhosts are properly handled on the http server. func TestVhosts(t *testing.T) { - srv := createAndStartServer(t, httpConfig{Vhosts: []string{"test"}}, false, wsConfig{}) + srv := createAndStartServer(t, &httpConfig{Vhosts: []string{"test"}}, false, &wsConfig{}) defer srv.stop() + url := "http://" + srv.listenAddr() - resp := testRequest(t, "", "", "test", srv) + resp := rpcRequest(t, url, "host", "test") assert.Equal(t, resp.StatusCode, http.StatusOK) - resp2 := testRequest(t, "", "", "bad", srv) + resp2 := rpcRequest(t, url, "host", "bad") assert.Equal(t, resp2.StatusCode, http.StatusForbidden) } @@ -138,14 +143,15 @@ func TestWebsocketOrigins(t *testing.T) { }, } for _, tc := range tests { - srv := createAndStartServer(t, httpConfig{}, true, wsConfig{Origins: splitAndTrim(tc.spec)}) + srv := createAndStartServer(t, &httpConfig{}, true, &wsConfig{Origins: splitAndTrim(tc.spec)}) + url := fmt.Sprintf("ws://%v", srv.listenAddr()) for _, origin := range tc.expOk { - if err := attemptWebsocketConnectionFromOrigin(t, srv, origin); err != nil { + if err := wsRequest(t, url, origin); err != nil { t.Errorf("spec '%v', origin '%v': expected ok, got %v", tc.spec, origin, err) } } for _, origin := range tc.expFail { - if err := attemptWebsocketConnectionFromOrigin(t, srv, origin); err == nil { + if err := wsRequest(t, url, origin); err == nil { t.Errorf("spec '%v', origin '%v': expected not to allow, got ok", tc.spec, origin) } } @@ -168,47 +174,118 @@ func TestIsWebsocket(t *testing.T) { assert.True(t, isWebsocket(r)) } -func createAndStartServer(t *testing.T, conf httpConfig, ws bool, wsConf wsConfig) *httpServer { +func Test_checkPath(t *testing.T) { + tests := []struct { + req *http.Request + prefix string + expected bool + }{ + { + req: &http.Request{URL: &url.URL{Path: "/test"}}, + prefix: "/test", + expected: true, + }, + { + req: &http.Request{URL: &url.URL{Path: "/testing"}}, + prefix: "/test", + expected: true, + }, + { + req: &http.Request{URL: &url.URL{Path: "/"}}, + prefix: "/test", + expected: false, + }, + { + req: &http.Request{URL: &url.URL{Path: "/fail"}}, + prefix: "/test", + expected: false, + }, + { + req: &http.Request{URL: &url.URL{Path: "/"}}, + prefix: "", + expected: true, + }, + { + req: &http.Request{URL: &url.URL{Path: "/fail"}}, + prefix: "", + expected: false, + }, + { + req: &http.Request{URL: &url.URL{Path: "/"}}, + prefix: "/", + expected: true, + }, + { + req: &http.Request{URL: &url.URL{Path: "/testing"}}, + prefix: "/", + expected: true, + }, + } + + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + assert.Equal(t, tt.expected, checkPath(tt.req, tt.prefix)) + }) + } +} + +func createAndStartServer(t *testing.T, conf *httpConfig, ws bool, wsConf *wsConfig) *httpServer { t.Helper() srv := newHTTPServer(testlog.Logger(t, log.LvlDebug), rpc.DefaultHTTPTimeouts) - - assert.NoError(t, srv.enableRPC(nil, conf)) + assert.NoError(t, srv.enableRPC(nil, *conf)) if ws { - assert.NoError(t, srv.enableWS(nil, wsConf)) + assert.NoError(t, srv.enableWS(nil, *wsConf)) } assert.NoError(t, srv.setListenAddr("localhost", 0)) assert.NoError(t, srv.start()) - return srv } -func attemptWebsocketConnectionFromOrigin(t *testing.T, srv *httpServer, browserOrigin string) error { +// wsRequest attempts to open a WebSocket connection to the given URL. +func wsRequest(t *testing.T, url, browserOrigin string) error { t.Helper() - dialer := websocket.DefaultDialer - _, _, err := dialer.Dial("ws://"+srv.listenAddr(), http.Header{ - "Content-type": []string{"application/json"}, - "Sec-WebSocket-Version": []string{"13"}, - "Origin": []string{browserOrigin}, - }) + t.Logf("checking WebSocket on %s (origin %q)", url, browserOrigin) + + headers := make(http.Header) + if browserOrigin != "" { + headers.Set("Origin", browserOrigin) + } + conn, _, err := websocket.DefaultDialer.Dial(url, headers) + if conn != nil { + conn.Close() + } return err } -func testRequest(t *testing.T, key, value, host string, srv *httpServer) *http.Response { +// rpcRequest performs a JSON-RPC request to the given URL. +func rpcRequest(t *testing.T, url string, extraHeaders ...string) *http.Response { t.Helper() - body := bytes.NewReader([]byte(`{"jsonrpc":"2.0","id":1,method":"rpc_modules"}`)) - req, _ := http.NewRequest("POST", "http://"+srv.listenAddr(), body) + // Create the request. + body := bytes.NewReader([]byte(`{"jsonrpc":"2.0","id":1,"method":"rpc_modules","params":[]}`)) + req, err := http.NewRequest("POST", url, body) + if err != nil { + t.Fatal("could not create http request:", err) + } req.Header.Set("content-type", "application/json") - if key != "" && value != "" { - req.Header.Set(key, value) + + // Apply extra headers. + if len(extraHeaders)%2 != 0 { + panic("odd extraHeaders length") } - if host != "" { - req.Host = host + for i := 0; i < len(extraHeaders); i += 2 { + key, value := extraHeaders[i], extraHeaders[i+1] + if strings.ToLower(key) == "host" { + req.Host = value + } else { + req.Header.Set(key, value) + } } - client := http.DefaultClient - resp, err := client.Do(req) + // Perform the request. + t.Logf("checking RPC/HTTP on %s %v", url, extraHeaders) + resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } From ef84da8481feedce8616c43ec48e15b7a4838290 Mon Sep 17 00:00:00 2001 From: Alex Prut <1648497+alexprut@users.noreply.github.com> Date: Tue, 2 Feb 2021 10:32:44 +0100 Subject: [PATCH 111/709] all: remove unneeded parentheses (#21921) * remove uneeded convertion type * remove redundant type in composite literal * omit explicit type where implicit * remove unused redundant parenthesis * remove redundant import alias duktape --- accounts/keystore/account_cache.go | 2 +- cmd/puppeth/genesis.go | 2 +- cmd/puppeth/wizard_genesis.go | 2 +- core/state/snapshot/conversion.go | 4 ++-- core/vm/runtime/runtime_test.go | 4 ++-- crypto/bls12381/bls12_381_test.go | 2 +- crypto/signify/signify.go | 2 +- eth/protocols/snap/sync.go | 2 +- eth/tracers/tracer.go | 2 +- metrics/cpu_syscall.go | 2 +- metrics/exp/exp.go | 2 +- metrics/gauge_float64_test.go | 12 ++++++------ node/utils_test.go | 4 ++-- signer/core/api.go | 2 +- trie/trie_test.go | 2 +- 15 files changed, 23 insertions(+), 23 deletions(-) diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index 8f660e282f..a3ec6e9c56 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -262,7 +262,7 @@ func (ac *accountCache) scanAccounts() error { switch { case err != nil: log.Debug("Failed to decode keystore key", "path", path, "err", err) - case (addr == common.Address{}): + case addr == common.Address{}: log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address") default: return &accounts.Account{ diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 46268ec11b..ef1f977bf0 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -425,7 +425,7 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Params.EIP98Transition = math.MaxInt64 spec.Genesis.Seal.Ethereum.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.Seal.Ethereum.MixHash = (genesis.Mixhash[:]) + spec.Genesis.Seal.Ethereum.MixHash = genesis.Mixhash[:] spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) spec.Genesis.Author = genesis.Coinbase spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 52093975cb..78f63e1c7a 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -259,7 +259,7 @@ func (w *wizard) manageGenesis() { // Export the native genesis spec used by puppeth and Geth gethJson := filepath.Join(folder, fmt.Sprintf("%s.json", w.network)) - if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { + if err := ioutil.WriteFile(gethJson, out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) return } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 9832225345..4ec229b7ac 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -38,7 +38,7 @@ type trieKV struct { type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(in chan trieKV, out chan common.Hash) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -266,7 +266,7 @@ func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieGenerato // stdGenerate is a very basic hexary trie builder which uses the same Trie // as the rest of geth, with no enhancements or optimizations -func stdGenerate(in chan (trieKV), out chan (common.Hash)) { +func stdGenerate(in chan trieKV, out chan common.Hash) { t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New())) for leaf := range in { t.TryUpdate(leaf.key[:], leaf.value) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index b185258dad..af69e3333f 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -500,7 +500,7 @@ func DisabledTestEipExampleCases(t *testing.T) { { code := []byte{ - byte(vm.PUSH9), 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, (4 + 8), + byte(vm.PUSH9), 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 4 + 8, byte(vm.JUMPSUB), byte(vm.STOP), byte(vm.BEGINSUB), @@ -516,7 +516,7 @@ func DisabledTestEipExampleCases(t *testing.T) { // out the trace. { code := []byte{ - byte(vm.PUSH9), 0x01, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, (4 + 8), + byte(vm.PUSH9), 0x01, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 4 + 8, byte(vm.JUMPSUB), byte(vm.STOP), byte(vm.BEGINSUB), diff --git a/crypto/bls12381/bls12_381_test.go b/crypto/bls12381/bls12_381_test.go index 51523c9ee7..6bf5834105 100644 --- a/crypto/bls12381/bls12_381_test.go +++ b/crypto/bls12381/bls12_381_test.go @@ -5,7 +5,7 @@ import ( "math/big" ) -var fuz int = 10 +var fuz = 10 func randScalar(max *big.Int) *big.Int { a, _ := rand.Int(rand.Reader, max) diff --git a/crypto/signify/signify.go b/crypto/signify/signify.go index 7ba9705491..e280f87268 100644 --- a/crypto/signify/signify.go +++ b/crypto/signify/signify.go @@ -46,7 +46,7 @@ func parsePrivateKey(key string) (k ed25519.PrivateKey, header []byte, keyNum [] if string(keydata[:2]) != "Ed" { return nil, nil, nil, errInvalidKeyHeader } - return ed25519.PrivateKey(keydata[40:]), keydata[:2], keydata[32:40], nil + return keydata[40:], keydata[:2], keydata[32:40], nil } // SignFile creates a signature of the input file. diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index e7720026bf..422cdf8f72 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -1494,7 +1494,7 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { // retrievals as not-pending, ready for resheduling req.timeout.Stop() for i, hash := range req.hashes { - req.task.trieTasks[hash] = [][]byte(req.paths[i]) + req.task.trieTasks[hash] = req.paths[i] } } diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index c9f00d7371..dba7ce87cf 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - duktape "gopkg.in/olebedev/go-duktape.v3" + "gopkg.in/olebedev/go-duktape.v3" ) // bigIntegerJS is the minified version of https://github.com/peterolson/BigInteger.js. diff --git a/metrics/cpu_syscall.go b/metrics/cpu_syscall.go index e245453e82..106637af5a 100644 --- a/metrics/cpu_syscall.go +++ b/metrics/cpu_syscall.go @@ -31,5 +31,5 @@ func getProcessCPUTime() int64 { log.Warn("Failed to retrieve CPU time", "err", err) return 0 } - return int64(usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert + return (usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert } diff --git a/metrics/exp/exp.go b/metrics/exp/exp.go index f510b8381e..3ebe8cc68a 100644 --- a/metrics/exp/exp.go +++ b/metrics/exp/exp.go @@ -128,7 +128,7 @@ func (exp *exp) publishMeter(name string, metric metrics.Meter) { exp.getInt(name + ".count").Set(m.Count()) exp.getFloat(name + ".one-minute").Set(m.Rate1()) exp.getFloat(name + ".five-minute").Set(m.Rate5()) - exp.getFloat(name + ".fifteen-minute").Set((m.Rate15())) + exp.getFloat(name + ".fifteen-minute").Set(m.Rate15()) exp.getFloat(name + ".mean").Set(m.RateMean()) } diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go index 3ee568e7ba..02b75580c4 100644 --- a/metrics/gauge_float64_test.go +++ b/metrics/gauge_float64_test.go @@ -12,27 +12,27 @@ func BenchmarkGuageFloat64(b *testing.B) { func TestGaugeFloat64(t *testing.T) { g := NewGaugeFloat64() - g.Update(float64(47.0)) - if v := g.Value(); float64(47.0) != v { + g.Update(47.0) + if v := g.Value(); 47.0 != v { t.Errorf("g.Value(): 47.0 != %v\n", v) } } func TestGaugeFloat64Snapshot(t *testing.T) { g := NewGaugeFloat64() - g.Update(float64(47.0)) + g.Update(47.0) snapshot := g.Snapshot() g.Update(float64(0)) - if v := snapshot.Value(); float64(47.0) != v { + if v := snapshot.Value(); 47.0 != v { t.Errorf("g.Value(): 47.0 != %v\n", v) } } func TestGetOrRegisterGaugeFloat64(t *testing.T) { r := NewRegistry() - NewRegisteredGaugeFloat64("foo", r).Update(float64(47.0)) + NewRegisteredGaugeFloat64("foo", r).Update(47.0) t.Logf("registry: %v", r) - if g := GetOrRegisterGaugeFloat64("foo", r); float64(47.0) != g.Value() { + if g := GetOrRegisterGaugeFloat64("foo", r); 47.0 != g.Value() { t.Fatal(g) } } diff --git a/node/utils_test.go b/node/utils_test.go index 44c83e22da..b7474bb706 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -82,11 +82,11 @@ func (f *FullService) Stop() error { return nil } func (f *FullService) Protocols() []p2p.Protocol { return []p2p.Protocol{ - p2p.Protocol{ + { Name: "test1", Version: uint(1), }, - p2p.Protocol{ + { Name: "test2", Version: uint(2), }, diff --git a/signer/core/api.go b/signer/core/api.go index 7595d2d484..07e206a74c 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -439,7 +439,7 @@ func (api *SignerAPI) newAccount() (common.Address, error) { continue } if pwErr := ValidatePasswordFormat(resp.Text); pwErr != nil { - api.UI.ShowError(fmt.Sprintf("Account creation attempt #%d failed due to password requirements: %v", (i + 1), pwErr)) + api.UI.ShowError(fmt.Sprintf("Account creation attempt #%d failed due to password requirements: %v", i+1, pwErr)) } else { // No error acc, err := be[0].(*keystore.KeyStore).NewAccount(resp.Text) diff --git a/trie/trie_test.go b/trie/trie_test.go index ddbdcbbd5b..87bce9abca 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -322,7 +322,7 @@ func TestLargeValue(t *testing.T) { // TestRandomCases tests som cases that were found via random fuzzing func TestRandomCases(t *testing.T) { - var rt []randTestStep = []randTestStep{ + var rt = []randTestStep{ {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 0 {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 1 {op: 0, key: common.Hex2Bytes("d51b182b95d677e5f1c82508c0228de96b73092d78ce78b2230cd948674f66fd1483bd"), value: common.Hex2Bytes("0000000000000002")}, // step 2 From 83e4c49e2b8d3fa83675cf5647e45a6ade7b8b4f Mon Sep 17 00:00:00 2001 From: ucwong Date: Tue, 2 Feb 2021 20:09:23 +0800 Subject: [PATCH 112/709] trie : use trie.NewStackTrie instead of new(trie.Trie) (#22246) The PR makes use of the stacktrie, which is is more lenient on resource consumption, than the regular trie, in cases where we only need it for DeriveSha --- cmd/evm/internal/t8ntool/execution.go | 4 ++-- consensus/clique/clique.go | 2 +- consensus/ethash/consensus.go | 2 +- core/blockchain_test.go | 4 ++-- core/genesis.go | 2 +- core/state_processor_test.go | 2 +- core/tx_pool_test.go | 2 +- eth/fetcher/block_fetcher.go | 2 +- eth/fetcher/block_fetcher_test.go | 2 +- les/odr_requests.go | 4 ++-- miner/miner_test.go | 2 +- miner/worker.go | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 95e6de37cb..525723e3cb 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -229,8 +229,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } execRs := &ExecutionResult{ StateRoot: root, - TxRoot: types.DeriveSha(includedTxs, new(trie.Trie)), - ReceiptRoot: types.DeriveSha(receipts, new(trie.Trie)), + TxRoot: types.DeriveSha(includedTxs, trie.NewStackTrie(nil)), + ReceiptRoot: types.DeriveSha(receipts, trie.NewStackTrie(nil)), Bloom: types.CreateBloom(receipts), LogsHash: rlpHash(statedb.Logs()), Receipts: receipts, diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 6c667804d8..61d4358b4e 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -565,7 +565,7 @@ func (c *Clique) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header * c.Finalize(chain, header, state, txs, uncles) // Assemble and return the final block for sealing - return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)), nil + return types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)), nil } // Authorize injects a private key into the consensus engine to mint new blocks diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index c58fe7a530..1b801b253f 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -586,7 +586,7 @@ func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea ethash.Finalize(chain, header, state, txs, uncles) // Header seems complete, assemble into a block and return - return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie)), nil + return types.NewBlock(header, txs, uncles, receipts, trie.NewStackTrie(nil)), nil } // SealHash returns the hash of a block prior to it being sealed. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index d60a235981..c33e7321ec 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -683,12 +683,12 @@ func TestFastVsFullChains(t *testing.T) { } if fblock, arblock, anblock := fast.GetBlockByHash(hash), archive.GetBlockByHash(hash), ancient.GetBlockByHash(hash); fblock.Hash() != arblock.Hash() || anblock.Hash() != arblock.Hash() { t.Errorf("block #%d [%x]: block mismatch: fastdb %v, ancientdb %v, archivedb %v", num, hash, fblock, anblock, arblock) - } else if types.DeriveSha(fblock.Transactions(), new(trie.Trie)) != types.DeriveSha(arblock.Transactions(), new(trie.Trie)) || types.DeriveSha(anblock.Transactions(), new(trie.Trie)) != types.DeriveSha(arblock.Transactions(), new(trie.Trie)) { + } else if types.DeriveSha(fblock.Transactions(), trie.NewStackTrie(nil)) != types.DeriveSha(arblock.Transactions(), trie.NewStackTrie(nil)) || types.DeriveSha(anblock.Transactions(), trie.NewStackTrie(nil)) != types.DeriveSha(arblock.Transactions(), trie.NewStackTrie(nil)) { t.Errorf("block #%d [%x]: transactions mismatch: fastdb %v, ancientdb %v, archivedb %v", num, hash, fblock.Transactions(), anblock.Transactions(), arblock.Transactions()) } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(arblock.Uncles()) || types.CalcUncleHash(anblock.Uncles()) != types.CalcUncleHash(arblock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: fastdb %v, ancientdb %v, archivedb %v", num, hash, fblock.Uncles(), anblock, arblock.Uncles()) } - if freceipts, anreceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash), fast.Config()), rawdb.ReadReceipts(ancientDb, hash, *rawdb.ReadHeaderNumber(ancientDb, hash), fast.Config()), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash), fast.Config()); types.DeriveSha(freceipts, new(trie.Trie)) != types.DeriveSha(areceipts, new(trie.Trie)) { + if freceipts, anreceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash), fast.Config()), rawdb.ReadReceipts(ancientDb, hash, *rawdb.ReadHeaderNumber(ancientDb, hash), fast.Config()), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash), fast.Config()); types.DeriveSha(freceipts, trie.NewStackTrie(nil)) != types.DeriveSha(areceipts, trie.NewStackTrie(nil)) { t.Errorf("block #%d [%x]: receipts mismatch: fastdb %v, ancientdb %v, archivedb %v", num, hash, freceipts, anreceipts, areceipts) } } diff --git a/core/genesis.go b/core/genesis.go index f678a3bbca..a01eee9eb2 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -288,7 +288,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { statedb.Commit(false) statedb.Database().TrieDB().Commit(root, true, nil) - return types.NewBlock(head, nil, nil, nil, new(trie.Trie)) + return types.NewBlock(head, nil, nil, nil, trie.NewStackTrie(nil)) } // Commit writes the block and state of a genesis specification to the database. diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 6e63975ac1..5976ecc3d4 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -148,5 +148,5 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr } header.Root = common.BytesToHash(hasher.Sum(nil)) // Assemble and return the final block for sealing - return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)) + return types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) } diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 47d3830b06..5d555f5a9c 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -55,7 +55,7 @@ type testBlockChain struct { func (bc *testBlockChain) CurrentBlock() *types.Block { return types.NewBlock(&types.Header{ GasLimit: bc.gasLimit, - }, nil, nil, nil, new(trie.Trie)) + }, nil, nil, nil, trie.NewStackTrie(nil)) } func (bc *testBlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 270aaf5918..5ea8a128d9 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -620,7 +620,7 @@ func (f *BlockFetcher) loop() { continue } if txnHash == (common.Hash{}) { - txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), new(trie.Trie)) + txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), trie.NewStackTrie(nil)) } if txnHash != announce.header.TxHash { continue diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index 3220002a99..a6eef71da0 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -39,7 +39,7 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) - unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit}, nil, nil, nil, new(trie.Trie)) + unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit}, nil, nil, nil, trie.NewStackTrie(nil)) ) // makeChain creates a chain of n blocks starting at and including parent. diff --git a/les/odr_requests.go b/les/odr_requests.go index 962b88a322..711c54b40f 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -116,7 +116,7 @@ func (r *BlockRequest) Validate(db ethdb.Database, msg *Msg) error { if r.Header == nil { return errHeaderUnavailable } - if r.Header.TxHash != types.DeriveSha(types.Transactions(body.Transactions), new(trie.Trie)) { + if r.Header.TxHash != types.DeriveSha(types.Transactions(body.Transactions), trie.NewStackTrie(nil)) { return errTxHashMismatch } if r.Header.UncleHash != types.CalcUncleHash(body.Uncles) { @@ -174,7 +174,7 @@ func (r *ReceiptsRequest) Validate(db ethdb.Database, msg *Msg) error { if r.Header == nil { return errHeaderUnavailable } - if r.Header.ReceiptHash != types.DeriveSha(receipt, new(trie.Trie)) { + if r.Header.ReceiptHash != types.DeriveSha(receipt, trie.NewStackTrie(nil)) { return errReceiptHashMismatch } // Validations passed, store and return diff --git a/miner/miner_test.go b/miner/miner_test.go index 127b4c7687..da1e472dbd 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -63,7 +63,7 @@ type testBlockChain struct { func (bc *testBlockChain) CurrentBlock() *types.Block { return types.NewBlock(&types.Header{ GasLimit: bc.gasLimit, - }, nil, nil, nil, new(trie.Trie)) + }, nil, nil, nil, trie.NewStackTrie(nil)) } func (bc *testBlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { diff --git a/miner/worker.go b/miner/worker.go index 82d08d4c7e..e81d50e46e 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -728,7 +728,7 @@ func (w *worker) updateSnapshot() { w.current.txs, uncles, w.current.receipts, - new(trie.Trie), + trie.NewStackTrie(nil), ) w.snapshotState = w.current.state.Copy() } From 3512b41c5cd024421a2048c70688c1b82f122dff Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 3 Feb 2021 11:02:35 +0100 Subject: [PATCH 113/709] core: reset txpool state on sethead (#22247) fixes an issue where local transactions that were included in the chain before a SetHead were rejected if resubmitted, since the txpool had not reset the state to the current (older) state. --- core/tx_pool.go | 59 +++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 4a17c31ca8..36546cde75 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -1137,44 +1137,45 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { // head from the chain. // If that is the case, we don't have the lost transactions any more, and // there's nothing to add - if newNum < oldNum { - // If the reorg ended up on a lower number, it's indicative of setHead being the cause - log.Debug("Skipping transaction reset caused by setHead", - "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) - } else { + if newNum >= oldNum { // If we reorged to a same or higher number, then it's not a case of setHead log.Warn("Transaction pool reset with missing oldhead", "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) - } - return - } - for rem.NumberU64() > add.NumberU64() { - discarded = append(discarded, rem.Transactions()...) - if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { - log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) return } - } - for add.NumberU64() > rem.NumberU64() { - included = append(included, add.Transactions()...) - if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { - log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) - return + // If the reorg ended up on a lower number, it's indicative of setHead being the cause + log.Debug("Skipping transaction reset caused by setHead", + "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) + // We still need to update the current state s.th. the lost transactions can be readded by the user + } else { + for rem.NumberU64() > add.NumberU64() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } } - } - for rem.Hash() != add.Hash() { - discarded = append(discarded, rem.Transactions()...) - if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { - log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) - return + for add.NumberU64() > rem.NumberU64() { + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } } - included = append(included, add.Transactions()...) - if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { - log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) - return + for rem.Hash() != add.Hash() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } } + reinject = types.TxDifference(discarded, included) } - reinject = types.TxDifference(discarded, included) } } // Initialize the internal state to the current head From 54735a67239d19d3f29e3f8ebd348a6c88ed31fa Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 3 Feb 2021 15:04:28 +0100 Subject: [PATCH 114/709] fuzzers: added consensys/gurvy library to bn256 differential fuzzer (#21812) This pr adds consensys' gurvy bn256 variant into the code for differential fuzzing. --- go.mod | 1 + go.sum | 124 ++++++++++++++++++++++++++++++ tests/fuzzers/bn256/bn256_fuzz.go | 73 +++++++++++++----- 3 files changed, 180 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 48c6889fbe..5fd3f772a4 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/btcsuite/btcd v0.20.1-beta github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 + github.com/consensys/gurvy v0.3.8 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea github.com/dlclark/regexp2 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 2c16d81f04..1ac0cf81f2 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbf cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= @@ -56,12 +57,19 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= @@ -92,7 +100,17 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/consensys/bavard v0.1.8-0.20210105233146-c16790d2aa8b/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/goff v0.3.10/go.mod h1:xTldOBEHmFiYS0gPXd3NsaEqZWlnmeWcRLWgD3ba3xc= +github.com/consensys/gurvy v0.3.8 h1:H2hvjvT2OFMgdMn5ZbhXqHt+F8DJ2clZW7Vmc0kFFxc= +github.com/consensys/gurvy v0.3.8/go.mod h1:sN75xnsiD593XnhbhvG2PkOy194pZBzqShWF/kwuW/g= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -110,13 +128,19 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmak github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -126,6 +150,7 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -145,10 +170,12 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -190,26 +217,52 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 h1:HarGZ5h9HD9LgEg1yRVMXyfiw4wlXiLiYM2oMjeA/SE= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= @@ -226,6 +279,7 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QU github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -233,12 +287,15 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kilic/bls12-381 v0.0.0-20201226121925-69dacb279461/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= @@ -257,13 +314,18 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -273,6 +335,15 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -300,9 +371,11 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -314,25 +387,36 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -342,12 +426,20 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= +github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -358,22 +450,30 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -387,6 +487,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -404,14 +505,19 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -440,9 +546,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -464,6 +572,10 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -472,6 +584,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -491,15 +604,19 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= @@ -547,18 +664,25 @@ google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyz google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/tests/fuzzers/bn256/bn256_fuzz.go b/tests/fuzzers/bn256/bn256_fuzz.go index 477fe0a160..c98fbc33ae 100644 --- a/tests/fuzzers/bn256/bn256_fuzz.go +++ b/tests/fuzzers/bn256/bn256_fuzz.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. +// +build gofuzz + package bn256 import ( @@ -10,44 +12,53 @@ import ( "io" "math/big" + gurvy "github.com/consensys/gurvy/bn256" cloudflare "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare" google "github.com/ethereum/go-ethereum/crypto/bn256/google" ) -func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1) { +func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1, *gurvy.G1Affine) { _, xc, err := cloudflare.RandomG1(input) if err != nil { // insufficient input - return nil, nil + return nil, nil, nil } xg := new(google.G1) if _, err := xg.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err)) + } + xs := new(gurvy.G1Affine) + if err := xs.Unmarshal(xc.Marshal()); err != nil { + panic(fmt.Sprintf("Could not marshal cloudflare -> consensys:", err)) } - return xc, xg + return xc, xg, xs } -func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2) { +func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2, *gurvy.G2Affine) { _, xc, err := cloudflare.RandomG2(input) if err != nil { // insufficient input - return nil, nil + return nil, nil, nil } xg := new(google.G2) if _, err := xg.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err)) } - return xc, xg + xs := new(gurvy.G2Affine) + if err := xs.Unmarshal(xc.Marshal()); err != nil { + panic(fmt.Sprintf("Could not marshal cloudflare -> consensys:", err)) + } + return xc, xg, xs } // FuzzAdd fuzzez bn256 addition between the Google and Cloudflare libraries. func FuzzAdd(data []byte) int { input := bytes.NewReader(data) - xc, xg := getG1Points(input) + xc, xg, xs := getG1Points(input) if xc == nil { return 0 } - yc, yg := getG1Points(input) + yc, yg, ys := getG1Points(input) if yc == nil { return 0 } @@ -59,8 +70,16 @@ func FuzzAdd(data []byte) int { rg := new(google.G1) rg.Add(xg, yg) + tmpX := new(gurvy.G1Jac).FromAffine(xs) + tmpY := new(gurvy.G1Jac).FromAffine(ys) + rs := new(gurvy.G1Affine).FromJacobian(tmpX.AddAssign(tmpY)) + if !bytes.Equal(rc.Marshal(), rg.Marshal()) { - panic("add mismatch") + panic("add mismatch: cloudflare/google") + } + + if !bytes.Equal(rc.Marshal(), rs.Marshal()) { + panic("add mismatch: cloudflare/consensys") } return 1 } @@ -69,7 +88,7 @@ func FuzzAdd(data []byte) int { // libraries. func FuzzMul(data []byte) int { input := bytes.NewReader(data) - pc, pg := getG1Points(input) + pc, pg, ps := getG1Points(input) if pc == nil { return 0 } @@ -93,25 +112,43 @@ func FuzzMul(data []byte) int { rg := new(google.G1) rg.ScalarMult(pg, new(big.Int).SetBytes(buf)) + rs := new(gurvy.G1Jac) + psJac := new(gurvy.G1Jac).FromAffine(ps) + rs.ScalarMultiplication(psJac, new(big.Int).SetBytes(buf)) + rsAffine := new(gurvy.G1Affine).FromJacobian(rs) + if !bytes.Equal(rc.Marshal(), rg.Marshal()) { - panic("scalar mul mismatch") + panic("scalar mul mismatch: cloudflare/google") + } + if !bytes.Equal(rc.Marshal(), rsAffine.Marshal()) { + panic("scalar mul mismatch: cloudflare/consensys") } return 1 } func FuzzPair(data []byte) int { input := bytes.NewReader(data) - pc, pg := getG1Points(input) + pc, pg, ps := getG1Points(input) if pc == nil { return 0 } - tc, tg := getG2Points(input) + tc, tg, ts := getG2Points(input) if tc == nil { return 0 } - // Pair the two points and ensure thet result in the same output - if cloudflare.PairingCheck([]*cloudflare.G1{pc}, []*cloudflare.G2{tc}) != google.PairingCheck([]*google.G1{pg}, []*google.G2{tg}) { - panic("pair mismatch") + // Pair the two points and ensure they result in the same output + clPair := cloudflare.PairingCheck([]*cloudflare.G1{pc}, []*cloudflare.G2{tc}) + if clPair != google.PairingCheck([]*google.G1{pg}, []*google.G2{tg}) { + panic("pairing mismatch: cloudflare/google") + } + + coPair, err := gurvy.PairingCheck([]gurvy.G1Affine{*ps}, []gurvy.G2Affine{*ts}) + if err != nil { + panic(fmt.Sprintf("gurvy encountered error: %v", err)) + } + if clPair != coPair { + panic("pairing mismatch: cloudflare/consensys") } + return 1 } From 28121324ac42ad88b911da514ae2c092f5718f5d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 5 Feb 2021 11:35:55 +0100 Subject: [PATCH 115/709] internal/ethapi: comment nitpick (#22270) --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9c3f6b9161..ed109cd201 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -368,7 +368,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArg } // SendTransaction will create a transaction from the given arguments and -// tries to sign it with the key associated with args.To. If the given passwd isn't +// tries to sign it with the key associated with args.From. If the given passwd isn't // able to decrypt the key it fails. func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) { if args.Nonce == nil { From 098a2b6e26e59ce09c72869f7919724ba7fd97ee Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 5 Feb 2021 20:51:15 +0800 Subject: [PATCH 116/709] eth: move eth.Config to a common package (#22205) This moves the eth config definition into a separate package, eth/ethconfig. Packages eth and les can now import this common package instead of importing eth from les, reducing dependencies. Co-authored-by: Felix Lange --- cmd/faucet/faucet.go | 4 +- cmd/geth/config.go | 6 +- cmd/utils/cmd.go | 4 +- cmd/utils/flags.go | 109 +++++++++++++++--------------- cmd/utils/flags_legacy.go | 14 ++-- console/console_test.go | 5 +- core/bloom_indexer.go | 92 +++++++++++++++++++++++++ eth/backend.go | 51 +++----------- eth/bloombits.go | 69 ------------------- eth/{ => ethconfig}/config.go | 64 ++++++++++++++---- eth/{ => ethconfig}/gen_config.go | 2 +- ethclient/ethclient_test.go | 3 +- graphql/graphql_test.go | 3 +- les/api_test.go | 5 +- les/client.go | 8 +-- les/commons.go | 6 +- les/costtracker.go | 4 +- les/server.go | 15 +++- les/test_helper.go | 8 +-- miner/stress_clique.go | 2 +- miner/stress_ethash.go | 2 +- mobile/geth.go | 4 +- 22 files changed, 265 insertions(+), 215 deletions(-) create mode 100644 core/bloom_indexer.go rename eth/{ => ethconfig}/config.go (72%) rename eth/{ => ethconfig}/gen_config.go (99%) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 763b8d25f8..e839f1c886 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -47,8 +47,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/les" @@ -247,7 +247,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network ui } // Assemble the Ethereum light client protocol - cfg := eth.DefaultConfig + cfg := ethconfig.Defaults cfg.SyncMode = downloader.LightSync cfg.NetworkId = network cfg.Genesis = genesis diff --git a/cmd/geth/config.go b/cmd/geth/config.go index c5b330b2d8..c71d5eb653 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -27,7 +27,7 @@ import ( "gopkg.in/urfave/cli.v1" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -85,7 +85,7 @@ type whisperDeprecatedConfig struct { } type gethConfig struct { - Eth eth.Config + Eth ethconfig.Config Shh whisperDeprecatedConfig Node node.Config Ethstats ethstatsConfig @@ -121,7 +121,7 @@ func defaultNodeConfig() node.Config { func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // Load defaults. cfg := gethConfig{ - Eth: eth.DefaultConfig, + Eth: ethconfig.Defaults, Node: defaultNodeConfig(), Metrics: metrics.DefaultConfig, } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 4306216892..d4051e59ef 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" @@ -75,7 +75,7 @@ func StartNode(ctx *cli.Context, stack *node.Node) { signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sigc) - minFreeDiskSpace := eth.DefaultConfig.TrieDirtyCache + minFreeDiskSpace := ethconfig.Defaults.TrieDirtyCache if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) { minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name) } else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index aa5180dd9f..1ea26be457 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -44,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -137,7 +138,7 @@ var ( NetworkIdFlag = cli.Uint64Flag{ Name: "networkid", Usage: "Explicitly set network id (integer)(For testnets: use --ropsten, --rinkeby, --goerli instead)", - Value: eth.DefaultConfig.NetworkId, + Value: ethconfig.Defaults.NetworkId, } MainnetFlag = cli.BoolFlag{ Name: "mainnet", @@ -196,7 +197,7 @@ var ( Name: "nocode", Usage: "Exclude contract code (save db lookups)", } - defaultSyncMode = eth.DefaultConfig.SyncMode + defaultSyncMode = ethconfig.Defaults.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", Usage: `Blockchain sync mode ("fast", "full", "snap" or "light")`, @@ -228,32 +229,32 @@ var ( LightServeFlag = cli.IntFlag{ Name: "light.serve", Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", - Value: eth.DefaultConfig.LightServ, + Value: ethconfig.Defaults.LightServ, } LightIngressFlag = cli.IntFlag{ Name: "light.ingress", Usage: "Incoming bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", - Value: eth.DefaultConfig.LightIngress, + Value: ethconfig.Defaults.LightIngress, } LightEgressFlag = cli.IntFlag{ Name: "light.egress", Usage: "Outgoing bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", - Value: eth.DefaultConfig.LightEgress, + Value: ethconfig.Defaults.LightEgress, } LightMaxPeersFlag = cli.IntFlag{ Name: "light.maxpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to", - Value: eth.DefaultConfig.LightPeers, + Value: ethconfig.Defaults.LightPeers, } UltraLightServersFlag = cli.StringFlag{ Name: "ulc.servers", Usage: "List of trusted ultra-light servers", - Value: strings.Join(eth.DefaultConfig.UltraLightServers, ","), + Value: strings.Join(ethconfig.Defaults.UltraLightServers, ","), } UltraLightFractionFlag = cli.IntFlag{ Name: "ulc.fraction", Usage: "Minimum % of trusted ultra-light servers required to announce a new head", - Value: eth.DefaultConfig.UltraLightFraction, + Value: ethconfig.Defaults.UltraLightFraction, } UltraLightOnlyAnnounceFlag = cli.BoolFlag{ Name: "ulc.onlyannounce", @@ -271,12 +272,12 @@ var ( EthashCachesInMemoryFlag = cli.IntFlag{ Name: "ethash.cachesinmem", Usage: "Number of recent ethash caches to keep in memory (16MB each)", - Value: eth.DefaultConfig.Ethash.CachesInMem, + Value: ethconfig.Defaults.Ethash.CachesInMem, } EthashCachesOnDiskFlag = cli.IntFlag{ Name: "ethash.cachesondisk", Usage: "Number of recent ethash caches to keep on disk (16MB each)", - Value: eth.DefaultConfig.Ethash.CachesOnDisk, + Value: ethconfig.Defaults.Ethash.CachesOnDisk, } EthashCachesLockMmapFlag = cli.BoolFlag{ Name: "ethash.cacheslockmmap", @@ -285,17 +286,17 @@ var ( EthashDatasetDirFlag = DirectoryFlag{ Name: "ethash.dagdir", Usage: "Directory to store the ethash mining DAGs", - Value: DirectoryString(eth.DefaultConfig.Ethash.DatasetDir), + Value: DirectoryString(ethconfig.Defaults.Ethash.DatasetDir), } EthashDatasetsInMemoryFlag = cli.IntFlag{ Name: "ethash.dagsinmem", Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", - Value: eth.DefaultConfig.Ethash.DatasetsInMem, + Value: ethconfig.Defaults.Ethash.DatasetsInMem, } EthashDatasetsOnDiskFlag = cli.IntFlag{ Name: "ethash.dagsondisk", Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", - Value: eth.DefaultConfig.Ethash.DatasetsOnDisk, + Value: ethconfig.Defaults.Ethash.DatasetsOnDisk, } EthashDatasetsLockMmapFlag = cli.BoolFlag{ Name: "ethash.dagslockmmap", @@ -323,37 +324,37 @@ var ( TxPoolPriceLimitFlag = cli.Uint64Flag{ Name: "txpool.pricelimit", Usage: "Minimum gas price limit to enforce for acceptance into the pool", - Value: eth.DefaultConfig.TxPool.PriceLimit, + Value: ethconfig.Defaults.TxPool.PriceLimit, } TxPoolPriceBumpFlag = cli.Uint64Flag{ Name: "txpool.pricebump", Usage: "Price bump percentage to replace an already existing transaction", - Value: eth.DefaultConfig.TxPool.PriceBump, + Value: ethconfig.Defaults.TxPool.PriceBump, } TxPoolAccountSlotsFlag = cli.Uint64Flag{ Name: "txpool.accountslots", Usage: "Minimum number of executable transaction slots guaranteed per account", - Value: eth.DefaultConfig.TxPool.AccountSlots, + Value: ethconfig.Defaults.TxPool.AccountSlots, } TxPoolGlobalSlotsFlag = cli.Uint64Flag{ Name: "txpool.globalslots", Usage: "Maximum number of executable transaction slots for all accounts", - Value: eth.DefaultConfig.TxPool.GlobalSlots, + Value: ethconfig.Defaults.TxPool.GlobalSlots, } TxPoolAccountQueueFlag = cli.Uint64Flag{ Name: "txpool.accountqueue", Usage: "Maximum number of non-executable transaction slots permitted per account", - Value: eth.DefaultConfig.TxPool.AccountQueue, + Value: ethconfig.Defaults.TxPool.AccountQueue, } TxPoolGlobalQueueFlag = cli.Uint64Flag{ Name: "txpool.globalqueue", Usage: "Maximum number of non-executable transaction slots for all accounts", - Value: eth.DefaultConfig.TxPool.GlobalQueue, + Value: ethconfig.Defaults.TxPool.GlobalQueue, } TxPoolLifetimeFlag = cli.DurationFlag{ Name: "txpool.lifetime", Usage: "Maximum amount of time non-executable transaction are queued", - Value: eth.DefaultConfig.TxPool.Lifetime, + Value: ethconfig.Defaults.TxPool.Lifetime, } // Performance tuning settings CacheFlag = cli.IntFlag{ @@ -374,12 +375,12 @@ var ( CacheTrieJournalFlag = cli.StringFlag{ Name: "cache.trie.journal", Usage: "Disk journal directory for trie cache to survive node restarts", - Value: eth.DefaultConfig.TrieCleanCacheJournal, + Value: ethconfig.Defaults.TrieCleanCacheJournal, } CacheTrieRejournalFlag = cli.DurationFlag{ Name: "cache.trie.rejournal", Usage: "Time interval to regenerate the trie cache journal", - Value: eth.DefaultConfig.TrieCleanCacheRejournal, + Value: ethconfig.Defaults.TrieCleanCacheRejournal, } CacheGCFlag = cli.IntFlag{ Name: "cache.gc", @@ -416,17 +417,17 @@ var ( MinerGasTargetFlag = cli.Uint64Flag{ Name: "miner.gastarget", Usage: "Target gas floor for mined blocks", - Value: eth.DefaultConfig.Miner.GasFloor, + Value: ethconfig.Defaults.Miner.GasFloor, } MinerGasLimitFlag = cli.Uint64Flag{ Name: "miner.gaslimit", Usage: "Target gas ceiling for mined blocks", - Value: eth.DefaultConfig.Miner.GasCeil, + Value: ethconfig.Defaults.Miner.GasCeil, } MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", Usage: "Minimum gas price for mining a transaction", - Value: eth.DefaultConfig.Miner.GasPrice, + Value: ethconfig.Defaults.Miner.GasPrice, } MinerEtherbaseFlag = cli.StringFlag{ Name: "miner.etherbase", @@ -440,7 +441,7 @@ var ( MinerRecommitIntervalFlag = cli.DurationFlag{ Name: "miner.recommit", Usage: "Time interval to recreate the block being mined", - Value: eth.DefaultConfig.Miner.Recommit, + Value: ethconfig.Defaults.Miner.Recommit, } MinerNoVerfiyFlag = cli.BoolFlag{ Name: "miner.noverify", @@ -473,12 +474,12 @@ var ( RPCGlobalGasCapFlag = cli.Uint64Flag{ Name: "rpc.gascap", Usage: "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)", - Value: eth.DefaultConfig.RPCGasCap, + Value: ethconfig.Defaults.RPCGasCap, } RPCGlobalTxFeeCapFlag = cli.Float64Flag{ Name: "rpc.txfeecap", Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", - Value: eth.DefaultConfig.RPCTxFeeCap, + Value: ethconfig.Defaults.RPCTxFeeCap, } // Logging and debug settings EthStatsURLFlag = cli.StringFlag{ @@ -650,17 +651,17 @@ var ( GpoBlocksFlag = cli.IntFlag{ Name: "gpo.blocks", Usage: "Number of recent blocks to check for gas prices", - Value: eth.DefaultConfig.GPO.Blocks, + Value: ethconfig.Defaults.GPO.Blocks, } GpoPercentileFlag = cli.IntFlag{ Name: "gpo.percentile", Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", - Value: eth.DefaultConfig.GPO.Percentile, + Value: ethconfig.Defaults.GPO.Percentile, } GpoMaxGasPriceFlag = cli.Int64Flag{ Name: "gpo.maxprice", Usage: "Maximum gas price will be recommended by gpo", - Value: eth.DefaultConfig.GPO.MaxPrice.Int64(), + Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), } WhisperEnabledFlag = cli.BoolFlag{ Name: "shh", @@ -1028,7 +1029,7 @@ func setIPC(ctx *cli.Context, cfg *node.Config) { } // setLes configures the les server and ultra light client settings from the command line flags. -func setLes(ctx *cli.Context, cfg *eth.Config) { +func setLes(ctx *cli.Context, cfg *ethconfig.Config) { if ctx.GlobalIsSet(LegacyLightServFlag.Name) { cfg.LightServ = ctx.GlobalInt(LegacyLightServFlag.Name) log.Warn("The flag --lightserv is deprecated and will be removed in the future, please use --light.serve") @@ -1056,8 +1057,8 @@ func setLes(ctx *cli.Context, cfg *eth.Config) { cfg.UltraLightFraction = ctx.GlobalInt(UltraLightFractionFlag.Name) } if cfg.UltraLightFraction <= 0 && cfg.UltraLightFraction > 100 { - log.Error("Ultra light fraction is invalid", "had", cfg.UltraLightFraction, "updated", eth.DefaultConfig.UltraLightFraction) - cfg.UltraLightFraction = eth.DefaultConfig.UltraLightFraction + log.Error("Ultra light fraction is invalid", "had", cfg.UltraLightFraction, "updated", ethconfig.Defaults.UltraLightFraction) + cfg.UltraLightFraction = ethconfig.Defaults.UltraLightFraction } if ctx.GlobalIsSet(UltraLightOnlyAnnounceFlag.Name) { cfg.UltraLightOnlyAnnounce = ctx.GlobalBool(UltraLightOnlyAnnounceFlag.Name) @@ -1108,7 +1109,7 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase either from the directly specified // command line flags or from the keystore if CLI indexed. -func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) { +func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config) { // Extract the current etherbase, new flag overriding legacy one var etherbase string if ctx.GlobalIsSet(LegacyMinerEtherbaseFlag.Name) { @@ -1307,8 +1308,8 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) { // If we are running the light client, apply another group // settings for gas oracle. if light { - cfg.Blocks = eth.DefaultLightGPOConfig.Blocks - cfg.Percentile = eth.DefaultLightGPOConfig.Percentile + cfg.Blocks = ethconfig.LightClientGPO.Blocks + cfg.Percentile = ethconfig.LightClientGPO.Percentile } if ctx.GlobalIsSet(LegacyGpoBlocksFlag.Name) { cfg.Blocks = ctx.GlobalInt(LegacyGpoBlocksFlag.Name) @@ -1372,7 +1373,7 @@ func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { } } -func setEthash(ctx *cli.Context, cfg *eth.Config) { +func setEthash(ctx *cli.Context, cfg *ethconfig.Config) { if ctx.GlobalIsSet(EthashCacheDirFlag.Name) { cfg.Ethash.CacheDir = ctx.GlobalString(EthashCacheDirFlag.Name) } @@ -1435,7 +1436,7 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { } } -func setWhitelist(ctx *cli.Context, cfg *eth.Config) { +func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) { whitelist := ctx.GlobalString(WhitelistFlag.Name) if whitelist == "" { return @@ -1510,7 +1511,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node) { } // SetEthConfig applies eth-related command line flags to the config. -func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { +func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags CheckExclusive(ctx, MainnetFlag, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV3Flag) CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light") @@ -1709,7 +1710,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if // no URLs are set. -func SetDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) { +func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) { if cfg.EthDiscoveryURLs != nil { return // already set through flags/config } @@ -1728,7 +1729,7 @@ func SetDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) { } // RegisterEthService adds an Ethereum client to the stack. -func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend { +func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) ethapi.Backend { if cfg.SyncMode == downloader.LightSync { backend, err := les.New(stack, cfg) if err != nil { @@ -1865,14 +1866,14 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B engine = ethash.NewFaker() if !ctx.GlobalBool(FakePoWFlag.Name) { engine = ethash.New(ethash.Config{ - CacheDir: stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir), - CachesInMem: eth.DefaultConfig.Ethash.CachesInMem, - CachesOnDisk: eth.DefaultConfig.Ethash.CachesOnDisk, - CachesLockMmap: eth.DefaultConfig.Ethash.CachesLockMmap, - DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir), - DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem, - DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk, - DatasetsLockMmap: eth.DefaultConfig.Ethash.DatasetsLockMmap, + CacheDir: stack.ResolvePath(ethconfig.Defaults.Ethash.CacheDir), + CachesInMem: ethconfig.Defaults.Ethash.CachesInMem, + CachesOnDisk: ethconfig.Defaults.Ethash.CachesOnDisk, + CachesLockMmap: ethconfig.Defaults.Ethash.CachesLockMmap, + DatasetDir: stack.ResolvePath(ethconfig.Defaults.Ethash.DatasetDir), + DatasetsInMem: ethconfig.Defaults.Ethash.DatasetsInMem, + DatasetsOnDisk: ethconfig.Defaults.Ethash.DatasetsOnDisk, + DatasetsLockMmap: ethconfig.Defaults.Ethash.DatasetsLockMmap, }, nil, false) } } @@ -1880,12 +1881,12 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cache := &core.CacheConfig{ - TrieCleanLimit: eth.DefaultConfig.TrieCleanCache, + TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, TrieCleanNoPrefetch: ctx.GlobalBool(CacheNoPrefetchFlag.Name), - TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache, + TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", - TrieTimeLimit: eth.DefaultConfig.TrieTimeout, - SnapshotLimit: eth.DefaultConfig.SnapshotCache, + TrieTimeLimit: ethconfig.Defaults.TrieTimeout, + SnapshotLimit: ethconfig.Defaults.SnapshotCache, Preimages: ctx.GlobalBool(CachePreimagesFlag.Name), } if cache.TrieDirtyDisabled && !cache.Preimages { diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 1376d47c05..ff45ab9094 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" "gopkg.in/urfave/cli.v1" ) @@ -55,12 +55,12 @@ var ( LegacyMinerGasTargetFlag = cli.Uint64Flag{ Name: "targetgaslimit", Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)", - Value: eth.DefaultConfig.Miner.GasFloor, + Value: ethconfig.Defaults.Miner.GasFloor, } LegacyMinerGasPriceFlag = BigFlag{ Name: "gasprice", Usage: "Minimum gas price for mining a transaction (deprecated, use --miner.gasprice)", - Value: eth.DefaultConfig.Miner.GasPrice, + Value: ethconfig.Defaults.Miner.GasPrice, } LegacyMinerEtherbaseFlag = cli.StringFlag{ Name: "etherbase", @@ -76,12 +76,12 @@ var ( LegacyLightServFlag = cli.IntFlag{ Name: "lightserv", Usage: "Maximum percentage of time allowed for serving LES requests (deprecated, use --light.serve)", - Value: eth.DefaultConfig.LightServ, + Value: ethconfig.Defaults.LightServ, } LegacyLightPeersFlag = cli.IntFlag{ Name: "lightpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated, use --light.maxpeers)", - Value: eth.DefaultConfig.LightPeers, + Value: ethconfig.Defaults.LightPeers, } // (Deprecated April 2020) @@ -143,12 +143,12 @@ var ( LegacyGpoBlocksFlag = cli.IntFlag{ Name: "gpoblocks", Usage: "Number of recent blocks to check for gas prices (deprecated, use --gpo.blocks)", - Value: eth.DefaultConfig.GPO.Blocks, + Value: ethconfig.Defaults.GPO.Blocks, } LegacyGpoPercentileFlag = cli.IntFlag{ Name: "gpopercentile", Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices (deprecated, use --gpo.percentile)", - Value: eth.DefaultConfig.GPO.Percentile, + Value: ethconfig.Defaults.GPO.Percentile, } LegacyBootnodesV4Flag = cli.StringFlag{ Name: "bootnodesv4", diff --git a/console/console_test.go b/console/console_test.go index 68c03d108d..f6ab781410 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/jsre" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" @@ -85,7 +86,7 @@ type tester struct { // newTester creates a test environment based on which the console can operate. // Please ensure you call Close() on the returned tester to avoid leaks. -func newTester(t *testing.T, confOverride func(*eth.Config)) *tester { +func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester { // Create a temporary storage for the node keys and initialize it workspace, err := ioutil.TempDir("", "console-tester-") if err != nil { @@ -97,7 +98,7 @@ func newTester(t *testing.T, confOverride func(*eth.Config)) *tester { if err != nil { t.Fatalf("failed to create node: %v", err) } - ethConf := ð.Config{ + ethConf := ðconfig.Config{ Genesis: core.DeveloperGenesisBlock(15, common.Address{}), Miner: miner.Config{ Etherbase: common.HexToAddress(testAddress), diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go new file mode 100644 index 0000000000..856746a1c0 --- /dev/null +++ b/core/bloom_indexer.go @@ -0,0 +1,92 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" +) + +const ( + // bloomThrottling is the time to wait between processing two consecutive index + // sections. It's useful during chain upgrades to prevent disk overload. + bloomThrottling = 100 * time.Millisecond +) + +// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index +// for the Ethereum header bloom filters, permitting blazing fast filtering. +type BloomIndexer struct { + size uint64 // section size to generate bloombits for + db ethdb.Database // database instance to write index data and metadata into + gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index + section uint64 // Section is the section number being processed currently + head common.Hash // Head is the hash of the last header processed +} + +// NewBloomIndexer returns a chain indexer that generates bloom bits data for the +// canonical chain for fast logs filtering. +func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *ChainIndexer { + backend := &BloomIndexer{ + db: db, + size: size, + } + table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) + + return NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") +} + +// Reset implements core.ChainIndexerBackend, starting a new bloombits index +// section. +func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { + gen, err := bloombits.NewGenerator(uint(b.size)) + b.gen, b.section, b.head = gen, section, common.Hash{} + return err +} + +// Process implements core.ChainIndexerBackend, adding a new header's bloom into +// the index. +func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error { + b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) + b.head = header.Hash() + return nil +} + +// Commit implements core.ChainIndexerBackend, finalizing the bloom section and +// writing it out into the database. +func (b *BloomIndexer) Commit() error { + batch := b.db.NewBatch() + for i := 0; i < types.BloomBitLength; i++ { + bits, err := b.gen.Bitset(uint(i)) + if err != nil { + return err + } + rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) + } + return batch.Write() +} + +// Prune returns an empty error since we don't support pruning here. +func (b *BloomIndexer) Prune(threshold uint64) error { + return nil +} diff --git a/eth/backend.go b/eth/backend.go index a6390facb3..51ad0ccf3f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -31,13 +31,13 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" - "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -55,9 +55,13 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) +// Config contains the configuration options of the ETH protocol. +// Deprecated: use ethconfig.Config instead. +type Config = ethconfig.Config + // Ethereum implements the Ethereum full node service. type Ethereum struct { - config *Config + config *ethconfig.Config // Handlers txPool *core.TxPool @@ -93,7 +97,7 @@ type Ethereum struct { // New creates a new Ethereum object (including the // initialisation of the common Ethereum object) -func New(stack *node.Node, config *Config) (*Ethereum, error) { +func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Ensure configuration values are compatible and sane if config.SyncMode == downloader.LightSync { return nil, errors.New("can't run eth.Ethereum in light sync mode, use les.LightEthereum") @@ -102,8 +106,8 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode) } if config.Miner.GasPrice == nil || config.Miner.GasPrice.Cmp(common.Big0) <= 0 { - log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", DefaultConfig.Miner.GasPrice) - config.Miner.GasPrice = new(big.Int).Set(DefaultConfig.Miner.GasPrice) + log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice) + config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice) } if config.NoPruning && config.TrieDirtyCache > 0 { if config.SnapshotCache > 0 { @@ -132,13 +136,13 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: CreateConsensusEngine(stack, chainConfig, &config.Ethash, config.Miner.Notify, config.Miner.Noverify, chainDb), + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, config.Miner.Notify, config.Miner.Noverify, chainDb), closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, etherbase: config.Miner.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), + bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), p2pServer: stack.Server(), } @@ -269,39 +273,6 @@ func makeExtraData(extra []byte) []byte { return extra } -// CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { - // If proof-of-authority is requested, set it up - if chainConfig.Clique != nil { - return clique.New(chainConfig.Clique, db) - } - // Otherwise assume proof-of-work - switch config.PowMode { - case ethash.ModeFake: - log.Warn("Ethash used in fake mode") - return ethash.NewFaker() - case ethash.ModeTest: - log.Warn("Ethash used in test mode") - return ethash.NewTester(nil, noverify) - case ethash.ModeShared: - log.Warn("Ethash used in shared mode") - return ethash.NewShared() - default: - engine := ethash.New(ethash.Config{ - CacheDir: stack.ResolvePath(config.CacheDir), - CachesInMem: config.CachesInMem, - CachesOnDisk: config.CachesOnDisk, - CachesLockMmap: config.CachesLockMmap, - DatasetDir: config.DatasetDir, - DatasetsInMem: config.DatasetsInMem, - DatasetsOnDisk: config.DatasetsOnDisk, - DatasetsLockMmap: config.DatasetsLockMmap, - }, notify, noverify) - engine.SetThreads(-1) // Disable CPU mining - return engine - } -} - // APIs return the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *Ethereum) APIs() []rpc.API { diff --git a/eth/bloombits.go b/eth/bloombits.go index bd34bd7b69..0cb7050d23 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -17,16 +17,10 @@ package eth import ( - "context" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" ) const ( @@ -78,66 +72,3 @@ func (eth *Ethereum) startBloomHandlers(sectionSize uint64) { }() } } - -const ( - // bloomThrottling is the time to wait between processing two consecutive index - // sections. It's useful during chain upgrades to prevent disk overload. - bloomThrottling = 100 * time.Millisecond -) - -// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index -// for the Ethereum header bloom filters, permitting blazing fast filtering. -type BloomIndexer struct { - size uint64 // section size to generate bloombits for - db ethdb.Database // database instance to write index data and metadata into - gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index - section uint64 // Section is the section number being processed currently - head common.Hash // Head is the hash of the last header processed -} - -// NewBloomIndexer returns a chain indexer that generates bloom bits data for the -// canonical chain for fast logs filtering. -func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *core.ChainIndexer { - backend := &BloomIndexer{ - db: db, - size: size, - } - table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) - - return core.NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") -} - -// Reset implements core.ChainIndexerBackend, starting a new bloombits index -// section. -func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { - gen, err := bloombits.NewGenerator(uint(b.size)) - b.gen, b.section, b.head = gen, section, common.Hash{} - return err -} - -// Process implements core.ChainIndexerBackend, adding a new header's bloom into -// the index. -func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error { - b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) - b.head = header.Hash() - return nil -} - -// Commit implements core.ChainIndexerBackend, finalizing the bloom section and -// writing it out into the database. -func (b *BloomIndexer) Commit() error { - batch := b.db.NewBatch() - for i := 0; i < types.BloomBitLength; i++ { - bits, err := b.gen.Bitset(uint(i)) - if err != nil { - return err - } - rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) - } - return batch.Write() -} - -// Prune returns an empty error since we don't support pruning here. -func (b *BloomIndexer) Prune(threshold uint64) error { - return nil -} diff --git a/eth/config.go b/eth/ethconfig/config.go similarity index 72% rename from eth/config.go rename to eth/ethconfig/config.go index 446467d364..9147a602d5 100644 --- a/eth/config.go +++ b/eth/ethconfig/config.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package eth +// Package ethconfig contains the configuration of the ETH and LES protocols. +package ethconfig import ( "math/big" @@ -25,30 +26,35 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" ) -// DefaultFullGPOConfig contains default gasprice oracle settings for full node. -var DefaultFullGPOConfig = gasprice.Config{ +// FullNodeGPO contains default gasprice oracle settings for full node. +var FullNodeGPO = gasprice.Config{ Blocks: 20, Percentile: 60, MaxPrice: gasprice.DefaultMaxPrice, } -// DefaultLightGPOConfig contains default gasprice oracle settings for light client. -var DefaultLightGPOConfig = gasprice.Config{ +// LightClientGPO contains default gasprice oracle settings for light client. +var LightClientGPO = gasprice.Config{ Blocks: 2, Percentile: 60, MaxPrice: gasprice.DefaultMaxPrice, } -// DefaultConfig contains default settings for use on the Ethereum main net. -var DefaultConfig = Config{ +// Defaults contains default settings for use on the Ethereum main net. +var Defaults = Config{ SyncMode: downloader.FastSync, Ethash: ethash.Config{ CacheDir: "ethash", @@ -77,7 +83,7 @@ var DefaultConfig = Config{ }, TxPool: core.DefaultTxPoolConfig, RPCGasCap: 25000000, - GPO: DefaultFullGPOConfig, + GPO: FullNodeGPO, RPCTxFeeCap: 1, // 1 ether } @@ -89,21 +95,22 @@ func init() { } } if runtime.GOOS == "darwin" { - DefaultConfig.Ethash.DatasetDir = filepath.Join(home, "Library", "Ethash") + Defaults.Ethash.DatasetDir = filepath.Join(home, "Library", "Ethash") } else if runtime.GOOS == "windows" { localappdata := os.Getenv("LOCALAPPDATA") if localappdata != "" { - DefaultConfig.Ethash.DatasetDir = filepath.Join(localappdata, "Ethash") + Defaults.Ethash.DatasetDir = filepath.Join(localappdata, "Ethash") } else { - DefaultConfig.Ethash.DatasetDir = filepath.Join(home, "AppData", "Local", "Ethash") + Defaults.Ethash.DatasetDir = filepath.Join(home, "AppData", "Local", "Ethash") } } else { - DefaultConfig.Ethash.DatasetDir = filepath.Join(home, ".ethash") + Defaults.Ethash.DatasetDir = filepath.Join(home, ".ethash") } } //go:generate gencodec -type Config -formats toml -out gen_config.go +// Config contains configuration options for of the ETH and LES protocols. type Config struct { // The genesis block, which is inserted if the database is empty. // If nil, the Ethereum main net block is used. @@ -190,3 +197,36 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` } + +// CreateConsensusEngine creates a consensus engine for the given chain configuration. +func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { + // If proof-of-authority is requested, set it up + if chainConfig.Clique != nil { + return clique.New(chainConfig.Clique, db) + } + // Otherwise assume proof-of-work + switch config.PowMode { + case ethash.ModeFake: + log.Warn("Ethash used in fake mode") + return ethash.NewFaker() + case ethash.ModeTest: + log.Warn("Ethash used in test mode") + return ethash.NewTester(nil, noverify) + case ethash.ModeShared: + log.Warn("Ethash used in shared mode") + return ethash.NewShared() + default: + engine := ethash.New(ethash.Config{ + CacheDir: stack.ResolvePath(config.CacheDir), + CachesInMem: config.CachesInMem, + CachesOnDisk: config.CachesOnDisk, + CachesLockMmap: config.CachesLockMmap, + DatasetDir: config.DatasetDir, + DatasetsInMem: config.DatasetsInMem, + DatasetsOnDisk: config.DatasetsOnDisk, + DatasetsLockMmap: config.DatasetsLockMmap, + }, notify, noverify) + engine.SetThreads(-1) // Disable CPU mining + return engine + } +} diff --git a/eth/gen_config.go b/eth/ethconfig/gen_config.go similarity index 99% rename from eth/gen_config.go rename to eth/ethconfig/gen_config.go index e68b29ce5e..5814b81b09 100644 --- a/eth/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -1,6 +1,6 @@ // Code generated by github.com/fjl/gencodec. DO NOT EDIT. -package eth +package ethconfig import ( "time" diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index d700022e8f..8b175ee066 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" @@ -195,7 +196,7 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { t.Fatalf("can't create new node: %v", err) } // Create Ethereum Service - config := ð.Config{Genesis: genesis} + config := ðconfig.Config{Genesis: genesis} config.Ethash.PowMode = ethash.ModeFake ethservice, err := eth.New(n, config) if err != nil { diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index a88c9b30b1..2f3b230329 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -191,7 +192,7 @@ func createNode(t *testing.T, gqlEnabled bool) *node.Node { func createGQLService(t *testing.T, stack *node.Node) { // create backend - ethConf := ð.Config{ + ethConf := ðconfig.Config{ Genesis: &core.Genesis{ Config: params.AllEthashProtocolChanges, GasLimit: 11500000, diff --git a/les/api_test.go b/les/api_test.go index 2895264f67..f7017c5d98 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -492,14 +493,14 @@ func testSim(t *testing.T, serverCount, clientCount int, serverDir, clientDir [] } func newLesClientService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - config := eth.DefaultConfig + config := ethconfig.Defaults config.SyncMode = downloader.LightSync config.Ethash.PowMode = ethash.ModeFake return New(stack, &config) } func newLesServerService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { - config := eth.DefaultConfig + config := ethconfig.Defaults config.SyncMode = downloader.FullSync config.LightServ = testServerCapacity config.LightPeers = testMaxClients diff --git a/les/client.go b/les/client.go index d8cb6e385a..1b26e9a9b5 100644 --- a/les/client.go +++ b/les/client.go @@ -30,8 +30,8 @@ import ( "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" @@ -76,7 +76,7 @@ type LightEthereum struct { } // New creates an instance of the light client. -func New(stack *node.Node, config *eth.Config) (*LightEthereum, error) { +func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/") if err != nil { return nil, err @@ -105,9 +105,9 @@ func New(stack *node.Node, config *eth.Config) (*LightEthereum, error) { eventMux: stack.EventMux(), reqDist: newRequestDistributor(peers, &mclock.System{}), accountManager: stack.AccountManager(), - engine: eth.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: eth.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), + bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), valueTracker: lpc.NewValueTracker(lespayDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), p2pServer: stack.Server(), p2pConfig: &stack.Config().P2P, diff --git a/les/commons.go b/les/commons.go index 73334497ad..a2fce1dc97 100644 --- a/les/commons.go +++ b/les/commons.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/checkpointoracle" @@ -48,7 +48,7 @@ type chainReader interface { // lesCommons contains fields needed by both server and client. type lesCommons struct { genesis common.Hash - config *eth.Config + config *ethconfig.Config chainConfig *params.ChainConfig iConfig *light.IndexerConfig chainDb ethdb.Database @@ -138,7 +138,7 @@ func (c *lesCommons) localCheckpoint(index uint64) params.TrustedCheckpoint { } // setupOracle sets up the checkpoint oracle contract client. -func (c *lesCommons) setupOracle(node *node.Node, genesis common.Hash, ethconfig *eth.Config) *checkpointoracle.CheckpointOracle { +func (c *lesCommons) setupOracle(node *node.Node, genesis common.Hash, ethconfig *ethconfig.Config) *checkpointoracle.CheckpointOracle { config := ethconfig.CheckpointOracle if config == nil { // Try loading default config. diff --git a/les/costtracker.go b/les/costtracker.go index 0558779bc5..43e32a5b2d 100644 --- a/les/costtracker.go +++ b/les/costtracker.go @@ -24,7 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/log" @@ -137,7 +137,7 @@ type costTracker struct { // newCostTracker creates a cost tracker and loads the cost factor statistics from the database. // It also returns the minimum capacity that can be assigned to any peer. -func newCostTracker(db ethdb.Database, config *eth.Config) (*costTracker, uint64) { +func newCostTracker(db ethdb.Database, config *ethconfig.Config) (*costTracker, uint64) { utilTarget := float64(config.LightServ) * flowcontrol.FixedPointMultiplier / 100 ct := &costTracker{ db: db, diff --git a/les/server.go b/les/server.go index 6b12b6f8f3..44495eb311 100644 --- a/les/server.go +++ b/les/server.go @@ -22,7 +22,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" lps "github.com/ethereum/go-ethereum/les/lespay/server" "github.com/ethereum/go-ethereum/light" @@ -50,6 +52,15 @@ func init() { priorityPoolSetup.Connect(balanceTrackerSetup.BalanceField, balanceTrackerSetup.UpdateFlag) // NodeBalance implements nodePriority } +type ethBackend interface { + ArchiveMode() bool + BlockChain() *core.BlockChain + BloomIndexer() *core.ChainIndexer + ChainDb() ethdb.Database + Synced() bool + TxPool() *core.TxPool +} + type LesServer struct { lesCommons @@ -73,7 +84,7 @@ type LesServer struct { p2pSrv *p2p.Server } -func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesServer, error) { +func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*LesServer, error) { ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup) // Calculate the number of threads used to service the light client // requests based on the user-specified value. diff --git a/les/test_helper.go b/les/test_helper.go index 04482ba68e..e3f0616a88 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/les/checkpointoracle" @@ -163,7 +163,7 @@ func prepare(n int, backend *backends.SimulatedBackend) { func testIndexers(db ethdb.Database, odr light.OdrBackend, config *light.IndexerConfig, disablePruning bool) []*core.ChainIndexer { var indexers [3]*core.ChainIndexer indexers[0] = light.NewChtIndexer(db, odr, config.ChtSize, config.ChtConfirms, disablePruning) - indexers[1] = eth.NewBloomIndexer(db, config.BloomSize, config.BloomConfirms) + indexers[1] = core.NewBloomIndexer(db, config.BloomSize, config.BloomConfirms) indexers[2] = light.NewBloomTrieIndexer(db, odr, config.BloomSize, config.BloomTrieSize, disablePruning) // make bloomTrieIndexer as a child indexer of bloom indexer. indexers[1].AddChildIndexer(indexers[2]) @@ -204,7 +204,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index client := &LightEthereum{ lesCommons: lesCommons{ genesis: genesis.Hash(), - config: ð.Config{LightPeers: 100, NetworkId: NetworkId}, + config: ðconfig.Config{LightPeers: 100, NetworkId: NetworkId}, chainConfig: params.AllEthashProtocolChanges, iConfig: light.TestClientIndexerConfig, chainDb: db, @@ -269,7 +269,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da server := &LesServer{ lesCommons: lesCommons{ genesis: genesis.Hash(), - config: ð.Config{LightPeers: 100, NetworkId: NetworkId}, + config: ðconfig.Config{LightPeers: 100, NetworkId: NetworkId}, chainConfig: params.AllEthashProtocolChanges, iConfig: light.TestServerIndexerConfig, chainDb: db, diff --git a/miner/stress_clique.go b/miner/stress_clique.go index a0fd596098..c585e0b1f6 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -185,7 +185,7 @@ func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { return nil, nil, err } // Create and register the backend - ethBackend, err := eth.New(stack, ð.Config{ + ethBackend, err := eth.New(stack, ðconfig.Config{ Genesis: genesis, NetworkId: genesis.Config.ChainID.Uint64(), SyncMode: downloader.FullSync, diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 1713af9d23..0b838d48b9 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -162,7 +162,7 @@ func makeMiner(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { if err != nil { return nil, nil, err } - ethBackend, err := eth.New(stack, ð.Config{ + ethBackend, err := eth.New(stack, ðconfig.Config{ Genesis: genesis, NetworkId: genesis.Config.ChainID.Uint64(), SyncMode: downloader.FullSync, diff --git a/mobile/geth.go b/mobile/geth.go index b561e33675..704d432e04 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -25,8 +25,8 @@ import ( "path/filepath" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/internal/debug" @@ -182,7 +182,7 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { } // Register the Ethereum protocol if requested if config.EthereumEnabled { - ethConf := eth.DefaultConfig + ethConf := ethconfig.Defaults ethConf.Genesis = genesis ethConf.SyncMode = downloader.LightSync ethConf.NetworkId = uint64(config.EthereumNetworkID) From fba5a63afe0993da70896c763b4fdfa953e066ff Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 5 Feb 2021 13:51:53 +0100 Subject: [PATCH 117/709] internal/ethapi: fix typo in comment (#22271) --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index ed109cd201..d3c007b9bf 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -386,7 +386,7 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs } // SignTransaction will create a transaction from the given arguments and -// tries to sign it with the key associated with args.To. If the given passwd isn't +// tries to sign it with the key associated with args.From. If the given passwd isn't // able to decrypt the key it fails. The transaction is returned in RLP-form, not broadcast // to other nodes func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs, passwd string) (*SignTransactionResult, error) { From 7ed860d4f17884ac9f1f6b927244a70e2e92eb94 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 5 Feb 2021 14:15:22 +0100 Subject: [PATCH 118/709] eth: don't wait for snap registration if we're not running snap (#22272) Prevents a situation where we (not running snap) connects with a peer running snap, and get stalled waiting for snap registration to succeed (which will never happen), which cause a waitgroup wait to halt shutdown --- eth/peerset.go | 4 ++-- p2p/peer.go | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/eth/peerset.go b/eth/peerset.go index f0657e140b..1e864a8e46 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -73,7 +73,7 @@ func newPeerSet() *peerSet { func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { // Reject the peer if it advertises `snap` without `eth` as `snap` is only a // satellite protocol meaningful with the chain selection of `eth` - if !peer.SupportsCap(eth.ProtocolName, eth.ProtocolVersions) { + if !peer.RunningCap(eth.ProtocolName, eth.ProtocolVersions) { return errSnapWithoutEth } // Ensure nobody can double connect @@ -101,7 +101,7 @@ func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { // If the peer does not support a compatible `snap`, don't wait - if !peer.SupportsCap(snap.ProtocolName, snap.ProtocolVersions) { + if !peer.RunningCap(snap.ProtocolName, snap.ProtocolVersions) { return nil, nil } // Ensure nobody can double connect diff --git a/p2p/peer.go b/p2p/peer.go index 08881e2583..8ebc858392 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -158,15 +158,14 @@ func (p *Peer) Caps() []Cap { return p.rw.caps } -// SupportsCap returns true if the peer supports any of the enumerated versions -// of a specific protocol. -func (p *Peer) SupportsCap(protocol string, versions []uint) bool { - for _, cap := range p.rw.caps { - if cap.Name == protocol { - for _, ver := range versions { - if cap.Version == ver { - return true - } +// RunningCap returns true if the peer is actively connected using any of the +// enumerated versions of a specific protocol, meaning that at least one of the +// versions is supported by both this node and the peer p. +func (p *Peer) RunningCap(protocol string, versions []uint) bool { + if proto, ok := p.running[protocol]; ok { + for _, ver := range versions { + if proto.Version == ver { + return true } } } From e74bd587f730fcdb5a9b625390da8aa85a2cbbc8 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 5 Feb 2021 19:44:34 +0100 Subject: [PATCH 119/709] consensus: remove seal verification from the consensus engine interface (#22274) --- consensus/clique/clique.go | 6 ------ consensus/consensus.go | 4 ---- consensus/ethash/algorithm_test.go | 2 +- consensus/ethash/consensus.go | 8 +------- consensus/ethash/ethash_test.go | 4 ++-- 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 61d4358b4e..c62e180faa 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -434,12 +434,6 @@ func (c *Clique) VerifyUncles(chain consensus.ChainReader, block *types.Block) e return nil } -// VerifySeal implements consensus.Engine, checking whether the signature contained -// in the header satisfies the consensus protocol requirements. -func (c *Clique) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error { - return c.verifySeal(chain, header, nil) -} - // verifySeal checks whether the signature contained in the header satisfies the // consensus protocol requirements. The method accepts an optional list of parent // headers that aren't yet part of the local blockchain to generate the snapshots diff --git a/consensus/consensus.go b/consensus/consensus.go index f7a4d0ff0b..2a5aac945d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -77,10 +77,6 @@ type Engine interface { // rules of a given engine. VerifyUncles(chain ChainReader, block *types.Block) error - // VerifySeal checks whether the crypto seal on a header is valid according to - // the consensus rules of the given engine. - VerifySeal(chain ChainHeaderReader, header *types.Header) error - // Prepare initializes the consensus fields of a block header according to the // rules of a particular engine. The changes are executed inline. Prepare(chain ChainHeaderReader, header *types.Header) error diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index 51fb6b124d..663687b81c 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -731,7 +731,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { defer pend.Done() ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false) defer ethash.Close() - if err := ethash.VerifySeal(nil, block.Header()); err != nil { + if err := ethash.verifySeal(nil, block.Header(), false); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) } }(i) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 1b801b253f..011a5688ef 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -288,7 +288,7 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa } // Verify the engine specific seal securing the block if seal { - if err := ethash.VerifySeal(chain, header); err != nil { + if err := ethash.verifySeal(chain, header, false); err != nil { return err } } @@ -488,12 +488,6 @@ var FrontierDifficultyCalulator = calcDifficultyFrontier var HomesteadDifficultyCalulator = calcDifficultyHomestead var DynamicDifficultyCalculator = makeDifficultyCalculator -// VerifySeal implements consensus.Engine, checking whether the given block satisfies -// the PoW difficulty requirements. -func (ethash *Ethash) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error { - return ethash.verifySeal(chain, header, false) -} - // verifySeal checks whether a block satisfies the PoW difficulty requirements, // either using the usual ethash cache for it, or alternatively using a full DAG // to make remote mining fast. diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index adbf1ccfeb..2639707eb2 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -46,7 +46,7 @@ func TestTestMode(t *testing.T) { case block := <-results: header.Nonce = types.EncodeNonce(block.Nonce()) header.MixDigest = block.MixDigest() - if err := ethash.VerifySeal(nil, header); err != nil { + if err := ethash.verifySeal(nil, header, false); err != nil { t.Fatalf("unexpected verification error: %v", err) } case <-time.NewTimer(4 * time.Second).C: @@ -86,7 +86,7 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { block = 0 } header := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} - e.VerifySeal(nil, header) + e.verifySeal(nil, header, false) } } From 994cdc69c8ab79b97f490c64721f2908df2070d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sun, 7 Feb 2021 20:13:59 +0200 Subject: [PATCH 120/709] cmd/utils: enable snapshots by default --- cmd/utils/flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1ea26be457..87295fb2f4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -208,9 +208,9 @@ var ( Usage: `Blockchain garbage collection mode ("full", "archive")`, Value: "full", } - SnapshotFlag = cli.BoolFlag{ + SnapshotFlag = cli.BoolTFlag{ Name: "snapshot", - Usage: `Enables snapshot-database mode -- experimental work in progress feature`, + Usage: `Enables snapshot-database mode (default = enable)`, } TxLookupLimitFlag = cli.Int64Flag{ Name: "txlookuplimit", @@ -1579,7 +1579,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 } - if !ctx.GlobalIsSet(SnapshotFlag.Name) { + if !ctx.GlobalBool(SnapshotFlag.Name) { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { log.Info("Snap sync requested, enabling --snapshot") @@ -1893,7 +1893,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B cache.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if !ctx.GlobalIsSet(SnapshotFlag.Name) { + if !ctx.GlobalBool(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { From 477fd420b393e3e5b962ac8c5af90a1add54d9d7 Mon Sep 17 00:00:00 2001 From: isdyaufh8o7cq Date: Mon, 8 Feb 2021 11:36:49 +0100 Subject: [PATCH 121/709] metrics: fix cast omission in cpu_syscall.go (#22262) fixes an regression which caused build failure on certain platforms --- metrics/cpu_syscall.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/cpu_syscall.go b/metrics/cpu_syscall.go index 106637af5a..e245453e82 100644 --- a/metrics/cpu_syscall.go +++ b/metrics/cpu_syscall.go @@ -31,5 +31,5 @@ func getProcessCPUTime() int64 { log.Warn("Failed to retrieve CPU time", "err", err) return 0 } - return (usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert + return int64(usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert } From d86906f1e6040e4e57c164fc5dfab0f97329b229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 8 Feb 2021 13:03:06 +0200 Subject: [PATCH 122/709] params: just to make snapshots a bit more official --- params/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/version.go b/params/version.go index b6d6bd3f1d..3447f0f283 100644 --- a/params/version.go +++ b/params/version.go @@ -22,8 +22,8 @@ import ( const ( VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 26 // Patch version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) From f566dd305e7db3a629a783ce89697f49c4ba4a75 Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 8 Feb 2021 19:16:30 +0800 Subject: [PATCH 123/709] all: bloom-filter based pruning mechanism (#21724) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cmd, core, tests: initial state pruner core: fix db inspector cmd/geth: add verify-state cmd/geth: add verification tool core/rawdb: implement flatdb cmd, core: fix rebase core/state: use new contract code layout core/state/pruner: avoid deleting genesis state cmd/geth: add helper function core, cmd: fix extract genesis core: minor fixes contracts: remove useless core/state/snapshot: plugin stacktrie core: polish core/state/snapshot: iterate storage concurrently core/state/snapshot: fix iteration core: add comments core/state/snapshot: polish code core/state: polish core/state/snapshot: rebase core/rawdb: add comments core/rawdb: fix tests core/rawdb: improve tests core/state/snapshot: fix concurrent iteration core/state: run pruning during the recovery core, trie: implement martin's idea core, eth: delete flatdb and polish pruner trie: fix import core/state/pruner: add log core/state/pruner: fix issues core/state/pruner: don't read back core/state/pruner: fix contract code write core/state/pruner: check root node presence cmd, core: polish log core/state: use HEAD-127 as the target core/state/snapshot: improve tests cmd/geth: fix verification tool cmd/geth: use HEAD as the verification default target all: replace the bloomfilter with martin's fork cmd, core: polish code core, cmd: forcibly delete state root core/state/pruner: add hash64 core/state/pruner: fix blacklist core/state: remove blacklist cmd, core: delete trie clean cache before pruning cmd, core: fix lint cmd, core: fix rebase core/state: fix the special case for clique networks core/state/snapshot: remove useless code core/state/pruner: capping the snapshot after pruning cmd, core, eth: fixes core/rawdb: update db inspector cmd/geth: polish code core/state/pruner: fsync bloom filter cmd, core: print warning log core/state/pruner: adjust the parameters for bloom filter cmd, core: create the bloom filter by size core: polish core/state/pruner: sanitize invalid bloomfilter size cmd: address comments cmd/geth: address comments cmd/geth: address comment core/state/pruner: address comments core/state/pruner: rename homedir to datadir cmd, core: address comments core/state/pruner: address comment core/state: address comments core, cmd, tests: address comments core: address comments core/state/pruner: release the iterator after each commit core/state/pruner: improve pruner cmd, core: adjust bloom paramters core/state/pruner: fix lint core/state/pruner: fix tests core: fix rebase core/state/pruner: remove atomic rename core/state/pruner: address comments all: run go mod tidy core/state/pruner: avoid false-positive for the middle state roots core/state/pruner: add checks for middle roots cmd/geth: replace crit with error * core/state/pruner: fix lint * core: drop legacy bloom filter * core/state/snapshot: improve pruner * core/state/snapshot: polish concurrent logs to report ETA vs. hashes * core/state/pruner: add progress report for pruning and compaction too * core: fix snapshot test API * core/state: fix some pruning logs * core/state/pruner: support recovering from bloom flush fail Co-authored-by: Péter Szilágyi --- cmd/geth/main.go | 3 + cmd/geth/snapshot.go | 437 ++++++++++++++++++++++ cmd/geth/usage.go | 1 + cmd/utils/flags.go | 5 + core/blockchain.go | 2 +- core/blockchain_snapshot_test.go | 3 +- core/genesis.go | 4 - core/rawdb/database.go | 19 +- core/rawdb/schema.go | 10 +- core/state/pruner/bloom.go | 132 +++++++ core/state/pruner/pruner.go | 537 +++++++++++++++++++++++++++ core/state/snapshot/conversion.go | 284 +++++++++----- core/state/snapshot/snapshot.go | 78 +++- core/state/snapshot/snapshot_test.go | 67 ++++ eth/backend.go | 4 + go.mod | 16 +- go.sum | 11 +- tests/block_test_util.go | 3 +- tests/state_test_util.go | 2 +- trie/stacktrie.go | 21 +- 20 files changed, 1491 insertions(+), 148 deletions(-) create mode 100644 cmd/geth/snapshot.go create mode 100644 core/state/pruner/bloom.go create mode 100644 core/state/pruner/pruner.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 11829cbad9..0236ffb7d3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -107,6 +107,7 @@ var ( utils.UltraLightFractionFlag, utils.UltraLightOnlyAnnounceFlag, utils.WhitelistFlag, + utils.BloomFilterSizeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, @@ -255,6 +256,8 @@ func init() { dumpConfigCommand, // See cmd/utils/flags_legacy.go utils.ShowDeprecated, + // See snapshot.go + snapshotCommand, } sort.Sort(cli.CommandsByName(app.Commands)) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go new file mode 100644 index 0000000000..6805a42585 --- /dev/null +++ b/cmd/geth/snapshot.go @@ -0,0 +1,437 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "errors" + "time" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/pruner" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + cli "gopkg.in/urfave/cli.v1" +) + +var ( + // emptyRoot is the known root hash of an empty trie. + emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + + // emptyCode is the known hash of the empty EVM bytecode. + emptyCode = crypto.Keccak256(nil) +) + +var ( + snapshotCommand = cli.Command{ + Name: "snapshot", + Usage: "A set of commands based on the snapshot", + Category: "MISCELLANEOUS COMMANDS", + Description: "", + Subcommands: []cli.Command{ + { + Name: "prune-state", + Usage: "Prune stale ethereum state data based on the snapshot", + ArgsUsage: "", + Action: utils.MigrateFlags(pruneState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.LegacyTestnetFlag, + utils.CacheTrieJournalFlag, + utils.BloomFilterSizeFlag, + }, + Description: ` +geth snapshot prune-state +will prune historical state data with the help of the state snapshot. +All trie nodes and contract codes that do not belong to the specified +version state will be deleted from the database. After pruning, only +two version states are available: genesis and the specific one. + +The default pruning target is the HEAD-127 state. + +WARNING: It's necessary to delete the trie clean cache after the pruning. +If you specify another directory for the trie clean cache via "--cache.trie.journal" +during the use of Geth, please also specify it here for correct deletion. Otherwise +the trie clean cache with default directory will be deleted. +`, + }, + { + Name: "verify-state", + Usage: "Recalculate state hash based on the snapshot for verification", + ArgsUsage: "", + Action: utils.MigrateFlags(verifyState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.LegacyTestnetFlag, + }, + Description: ` +geth snapshot verify-state +will traverse the whole accounts and storages set based on the specified +snapshot and recalculate the root hash of state for verification. +In other words, this command does the snapshot to trie conversion. +`, + }, + { + Name: "traverse-state", + Usage: "Traverse the state with given root hash for verification", + ArgsUsage: "", + Action: utils.MigrateFlags(traverseState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.LegacyTestnetFlag, + }, + Description: ` +geth snapshot traverse-state +will traverse the whole state from the given state root and will abort if any +referenced trie node or contract code is missing. This command can be used for +state integrity verification. The default checking target is the HEAD state. + +It's also usable without snapshot enabled. +`, + }, + { + Name: "traverse-rawstate", + Usage: "Traverse the state with given root hash for verification", + ArgsUsage: "", + Action: utils.MigrateFlags(traverseRawState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.LegacyTestnetFlag, + }, + Description: ` +geth snapshot traverse-rawstate +will traverse the whole state from the given root and will abort if any referenced +trie node or contract code is missing. This command can be used for state integrity +verification. The default checking target is the HEAD state. It's basically identical +to traverse-state, but the check granularity is smaller. + +It's also usable without snapshot enabled. +`, + }, + }, + } +) + +func pruneState(ctx *cli.Context) error { + stack, config := makeConfigNode(ctx) + defer stack.Close() + + chain, chaindb := utils.MakeChain(ctx, stack, true) + defer chaindb.Close() + + pruner, err := pruner.NewPruner(chaindb, chain.CurrentBlock().Header(), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) + if err != nil { + log.Error("Failed to open snapshot tree", "error", err) + return err + } + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + var targetRoot common.Hash + if ctx.NArg() == 1 { + targetRoot, err = parseRoot(ctx.Args()[0]) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + } + if err = pruner.Prune(targetRoot); err != nil { + log.Error("Failed to prune state", "error", err) + return err + } + return nil +} + +func verifyState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, chaindb := utils.MakeChain(ctx, stack, true) + defer chaindb.Close() + + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, chain.CurrentBlock().Root(), false, false, false) + if err != nil { + log.Error("Failed to open snapshot tree", "error", err) + return err + } + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + var root = chain.CurrentBlock().Root() + if ctx.NArg() == 1 { + root, err = parseRoot(ctx.Args()[0]) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + } + if err := snaptree.Verify(root); err != nil { + log.Error("Failed to verfiy state", "error", err) + return err + } + log.Info("Verified the state") + return nil +} + +// traverseState is a helper function used for pruning verification. +// Basically it just iterates the trie, ensure all nodes and associated +// contract codes are present. +func traverseState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, chaindb := utils.MakeChain(ctx, stack, true) + defer chaindb.Close() + + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + // Use the HEAD root as the default + head := chain.CurrentBlock() + if head == nil { + log.Error("Head block is missing") + return errors.New("head block is missing") + } + var ( + root common.Hash + err error + ) + if ctx.NArg() == 1 { + root, err = parseRoot(ctx.Args()[0]) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + log.Info("Start traversing the state", "root", root) + } else { + root = head.Root() + log.Info("Start traversing the state", "root", root, "number", head.NumberU64()) + } + triedb := trie.NewDatabase(chaindb) + t, err := trie.NewSecure(root, triedb) + if err != nil { + log.Error("Failed to open trie", "root", root, "error", err) + return err + } + var ( + accounts int + slots int + codes int + lastReport time.Time + start = time.Now() + ) + accIter := trie.NewIterator(t.NodeIterator(nil)) + for accIter.Next() { + accounts += 1 + var acc state.Account + if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { + log.Error("Invalid account encountered during traversal", "error", err) + return err + } + if acc.Root != emptyRoot { + storageTrie, err := trie.NewSecure(acc.Root, triedb) + if err != nil { + log.Error("Failed to open storage trie", "root", acc.Root, "error", err) + return err + } + storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) + for storageIter.Next() { + slots += 1 + } + if storageIter.Err != nil { + log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err) + return storageIter.Err + } + } + if !bytes.Equal(acc.CodeHash, emptyCode) { + code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) + if len(code) == 0 { + log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash)) + return errors.New("missing code") + } + codes += 1 + } + if time.Since(lastReport) > time.Second*8 { + log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + if accIter.Err != nil { + log.Error("Failed to traverse state trie", "root", root, "error", accIter.Err) + return accIter.Err + } + log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + +// traverseRawState is a helper function used for pruning verification. +// Basically it just iterates the trie, ensure all nodes and associated +// contract codes are present. It's basically identical to traverseState +// but it will check each trie node. +func traverseRawState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, chaindb := utils.MakeChain(ctx, stack, true) + defer chaindb.Close() + + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + // Use the HEAD root as the default + head := chain.CurrentBlock() + if head == nil { + log.Error("Head block is missing") + return errors.New("head block is missing") + } + var ( + root common.Hash + err error + ) + if ctx.NArg() == 1 { + root, err = parseRoot(ctx.Args()[0]) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + log.Info("Start traversing the state", "root", root) + } else { + root = head.Root() + log.Info("Start traversing the state", "root", root, "number", head.NumberU64()) + } + triedb := trie.NewDatabase(chaindb) + t, err := trie.NewSecure(root, triedb) + if err != nil { + log.Error("Failed to open trie", "root", root, "error", err) + return err + } + var ( + nodes int + accounts int + slots int + codes int + lastReport time.Time + start = time.Now() + ) + accIter := t.NodeIterator(nil) + for accIter.Next(true) { + nodes += 1 + node := accIter.Hash() + + if node != (common.Hash{}) { + // Check the present for non-empty hash node(embedded node doesn't + // have their own hash). + blob := rawdb.ReadTrieNode(chaindb, node) + if len(blob) == 0 { + log.Error("Missing trie node(account)", "hash", node) + return errors.New("missing account") + } + } + // If it's a leaf node, yes we are touching an account, + // dig into the storage trie further. + if accIter.Leaf() { + accounts += 1 + var acc state.Account + if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { + log.Error("Invalid account encountered during traversal", "error", err) + return errors.New("invalid account") + } + if acc.Root != emptyRoot { + storageTrie, err := trie.NewSecure(acc.Root, triedb) + if err != nil { + log.Error("Failed to open storage trie", "root", acc.Root, "error", err) + return errors.New("missing storage trie") + } + storageIter := storageTrie.NodeIterator(nil) + for storageIter.Next(true) { + nodes += 1 + node := storageIter.Hash() + + // Check the present for non-empty hash node(embedded node doesn't + // have their own hash). + if node != (common.Hash{}) { + blob := rawdb.ReadTrieNode(chaindb, node) + if len(blob) == 0 { + log.Error("Missing trie node(storage)", "hash", node) + return errors.New("missing storage") + } + } + // Bump the counter if it's leaf node. + if storageIter.Leaf() { + slots += 1 + } + } + if storageIter.Error() != nil { + log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error()) + return storageIter.Error() + } + } + if !bytes.Equal(acc.CodeHash, emptyCode) { + code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) + if len(code) == 0 { + log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey())) + return errors.New("missing code") + } + codes += 1 + } + if time.Since(lastReport) > time.Second*8 { + log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + } + if accIter.Error() != nil { + log.Error("Failed to traverse state trie", "root", root, "error", accIter.Error()) + return accIter.Error() + } + log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + +func parseRoot(input string) (common.Hash, error) { + var h common.Hash + if err := h.UnmarshalText([]byte(input)); err != nil { + return h, err + } + return h, nil +} diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 0ed31d7da6..55e934d31b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -245,6 +245,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ Name: "MISC", Flags: []cli.Flag{ utils.SnapshotFlag, + utils.BloomFilterSizeFlag, cli.HelpFlag, }, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 87295fb2f4..f9c5553961 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -225,6 +225,11 @@ var ( Name: "whitelist", Usage: "Comma separated block number-to-hash mappings to enforce (=)", } + BloomFilterSizeFlag = cli.Uint64Flag{ + Name: "bloomfilter.size", + Usage: "Megabytes of memory allocated to bloom-filter for pruning", + Value: 2048, + } // Light server and client settings LightServeFlag = cli.IntFlag{ Name: "light.serve", diff --git a/core/blockchain.go b/core/blockchain.go index c05ebfd549..7dd1097e98 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -372,7 +372,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer) recover = true } - bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, recover) + bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) } // Take ownership of this particular state go bc.update() diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index cb634a451d..96a5c7a8d4 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" @@ -163,7 +162,7 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [ } // Check the snapshot, ensure it's integrated - if err := snapshot.VerifyState(chain.snaps, block.Root()); err != nil { + if err := chain.snaps.Verify(block.Root()); err != nil { t.Errorf("The disk layer is not integrated %v", err) } } diff --git a/core/genesis.go b/core/genesis.go index a01eee9eb2..462aec45bd 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -171,7 +171,6 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig } return genesis.Config, block.Hash(), nil } - // We have the genesis block in database(perhaps in ancient database) // but the corresponding state is missing. header := rawdb.ReadHeader(db, stored, 0) @@ -190,7 +189,6 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig } return genesis.Config, block.Hash(), nil } - // Check whether the genesis block is already written. if genesis != nil { hash := genesis.ToBlock(nil).Hash() @@ -198,7 +196,6 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig return genesis.Config, hash, &GenesisMismatchError{stored, hash} } } - // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) if err := newcfg.CheckConfigForkOrder(); err != nil { @@ -216,7 +213,6 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig if genesis == nil && stored != params.MainnetGenesisHash { return storedcfg, stored, nil } - // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. height := rawdb.ReadHeaderNumber(db, rawdb.ReadHeadHeaderHash(db)) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index a2cc9c7be9..1f8c3f4542 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -335,7 +335,7 @@ func InspectDatabase(db ethdb.Database) error { hashNumPairings.Add(size) case len(key) == common.HashLength: tries.Add(size) - case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength: + case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: codes.Add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): txLookups.Add(size) @@ -347,15 +347,26 @@ func InspectDatabase(db ethdb.Database) error { preimages.Add(size) case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): bloomBits.Add(size) + case bytes.HasPrefix(key, BloomBitsIndexPrefix): + bloomBits.Add(size) case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: cliqueSnaps.Add(size) - case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: + case bytes.HasPrefix(key, []byte("cht-")) || + bytes.HasPrefix(key, []byte("chtIndexV2-")) || + bytes.HasPrefix(key, []byte("chtRootV2-")): // Canonical hash trie chtTrieNodes.Add(size) - case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: + case bytes.HasPrefix(key, []byte("blt-")) || + bytes.HasPrefix(key, []byte("bltIndex-")) || + bytes.HasPrefix(key, []byte("bltRoot-")): // Bloomtrie sub bloomTrieNodes.Add(size) default: var accounted bool - for _, meta := range [][]byte{databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey, uncleanShutdownKey, badBlockKey} { + for _, meta := range [][]byte{ + databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey, + fastTrieProgressKey, snapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, + snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, + badBlockKey, + } { if bytes.Equal(key, meta) { metadata.Add(size) accounted = true diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 9749a30d8a..0b411057f8 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -85,7 +85,7 @@ var ( bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value - codePrefix = []byte("c") // codePrefix + code hash -> account code + CodePrefix = []byte("c") // CodePrefix + code hash -> account code preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage configPrefix = []byte("ethereum-config-") // config prefix for the db @@ -209,16 +209,16 @@ func preimageKey(hash common.Hash) []byte { return append(preimagePrefix, hash.Bytes()...) } -// codeKey = codePrefix + hash +// codeKey = CodePrefix + hash func codeKey(hash common.Hash) []byte { - return append(codePrefix, hash.Bytes()...) + return append(CodePrefix, hash.Bytes()...) } // IsCodeKey reports whether the given byte slice is the key of contract code, // if so return the raw code hash as well. func IsCodeKey(key []byte) (bool, []byte) { - if bytes.HasPrefix(key, codePrefix) && len(key) == common.HashLength+len(codePrefix) { - return true, key[len(codePrefix):] + if bytes.HasPrefix(key, CodePrefix) && len(key) == common.HashLength+len(CodePrefix) { + return true, key[len(CodePrefix):] } return false, nil } diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go new file mode 100644 index 0000000000..4aeeb176e8 --- /dev/null +++ b/core/state/pruner/bloom.go @@ -0,0 +1,132 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pruner + +import ( + "encoding/binary" + "errors" + "os" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/log" + bloomfilter "github.com/holiman/bloomfilter/v2" +) + +// stateBloomHasher is a wrapper around a byte blob to satisfy the interface API +// requirements of the bloom library used. It's used to convert a trie hash or +// contract code hash into a 64 bit mini hash. +type stateBloomHasher []byte + +func (f stateBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } +func (f stateBloomHasher) Sum(b []byte) []byte { panic("not implemented") } +func (f stateBloomHasher) Reset() { panic("not implemented") } +func (f stateBloomHasher) BlockSize() int { panic("not implemented") } +func (f stateBloomHasher) Size() int { return 8 } +func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } + +// stateBloom is a bloom filter used during the state convesion(snapshot->state). +// The keys of all generated entries will be recorded here so that in the pruning +// stage the entries belong to the specific version can be avoided for deletion. +// +// The false-positive is allowed here. The "false-positive" entries means they +// actually don't belong to the specific version but they are not deleted in the +// pruning. The downside of the false-positive allowance is we may leave some "dangling" +// nodes in the disk. But in practice the it's very unlike the dangling node is +// state root. So in theory this pruned state shouldn't be visited anymore. Another +// potential issue is for fast sync. If we do another fast sync upon the pruned +// database, it's problematic which will stop the expansion during the syncing. +// TODO address it @rjl493456442 @holiman @karalabe. +// +// After the entire state is generated, the bloom filter should be persisted into +// the disk. It indicates the whole generation procedure is finished. +type stateBloom struct { + bloom *bloomfilter.Filter +} + +// newStateBloomWithSize creates a brand new state bloom for state generation. +// The bloom filter will be created by the passing bloom filter size. According +// to the https://hur.st/bloomfilter/?n=600000000&p=&m=2048MB&k=4, the parameters +// are picked so that the false-positive rate for mainnet is low enough. +func newStateBloomWithSize(size uint64) (*stateBloom, error) { + bloom, err := bloomfilter.New(size*1024*1024*8, 4) + if err != nil { + return nil, err + } + log.Info("Initialized state bloom", "size", common.StorageSize(float64(bloom.M()/8))) + return &stateBloom{bloom: bloom}, nil +} + +// NewStateBloomFromDisk loads the state bloom from the given file. +// In this case the assumption is held the bloom filter is complete. +func NewStateBloomFromDisk(filename string) (*stateBloom, error) { + bloom, _, err := bloomfilter.ReadFile(filename) + if err != nil { + return nil, err + } + return &stateBloom{bloom: bloom}, nil +} + +// Commit flushes the bloom filter content into the disk and marks the bloom +// as complete. +func (bloom *stateBloom) Commit(filename, tempname string) error { + // Write the bloom out into a temporary file + _, err := bloom.bloom.WriteFile(tempname) + if err != nil { + return err + } + // Ensure the file is synced to disk + f, err := os.Open(tempname) + if err != nil { + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + f.Close() + + // Move the teporary file into it's final location + return os.Rename(tempname, filename) +} + +// Put implements the KeyValueWriter interface. But here only the key is needed. +func (bloom *stateBloom) Put(key []byte, value []byte) error { + // If the key length is not 32bytes, ensure it's contract code + // entry with new scheme. + if len(key) != common.HashLength { + isCode, codeKey := rawdb.IsCodeKey(key) + if !isCode { + return errors.New("invalid entry") + } + bloom.bloom.Add(stateBloomHasher(codeKey)) + return nil + } + bloom.bloom.Add(stateBloomHasher(key)) + return nil +} + +// Delete removes the key from the key-value data store. +func (bloom *stateBloom) Delete(key []byte) error { panic("not supported") } + +// Contain is the wrapper of the underlying contains function which +// reports whether the key is contained. +// - If it says yes, the key may be contained +// - If it says no, the key is definitely not contained. +func (bloom *stateBloom) Contain(key []byte) (bool, error) { + return bloom.bloom.Contains(stateBloomHasher(key)), nil +} diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go new file mode 100644 index 0000000000..1cb65b38b1 --- /dev/null +++ b/core/state/pruner/pruner.go @@ -0,0 +1,537 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pruner + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +const ( + // stateBloomFilePrefix is the filename prefix of state bloom filter. + stateBloomFilePrefix = "statebloom" + + // stateBloomFilePrefix is the filename suffix of state bloom filter. + stateBloomFileSuffix = "bf.gz" + + // stateBloomFileTempSuffix is the filename suffix of state bloom filter + // while it is being written out to detect write aborts. + stateBloomFileTempSuffix = ".tmp" + + // rangeCompactionThreshold is the minimal deleted entry number for + // triggering range compaction. It's a quite arbitrary number but just + // to avoid triggering range compaction because of small deletion. + rangeCompactionThreshold = 100000 +) + +var ( + // emptyRoot is the known root hash of an empty trie. + emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + + // emptyCode is the known hash of the empty EVM bytecode. + emptyCode = crypto.Keccak256(nil) +) + +// Pruner is an offline tool to prune the stale state with the +// help of the snapshot. The workflow of pruner is very simple: +// +// - iterate the snapshot, reconstruct the relevant state +// - iterate the database, delete all other state entries which +// don't belong to the target state and the genesis state +// +// It can take several hours(around 2 hours for mainnet) to finish +// the whole pruning work. It's recommended to run this offline tool +// periodically in order to release the disk usage and improve the +// disk read performance to some extent. +type Pruner struct { + db ethdb.Database + stateBloom *stateBloom + datadir string + trieCachePath string + headHeader *types.Header + snaptree *snapshot.Tree +} + +// NewPruner creates the pruner instance. +func NewPruner(db ethdb.Database, headHeader *types.Header, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) { + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headHeader.Root, false, false, false) + if err != nil { + return nil, err // The relevant snapshot(s) might not exist + } + // Sanitize the bloom filter size if it's too small. + if bloomSize < 256 { + log.Warn("Sanitizing bloomfilter size", "provided(MB)", bloomSize, "updated(MB)", 256) + bloomSize = 256 + } + stateBloom, err := newStateBloomWithSize(bloomSize) + if err != nil { + return nil, err + } + return &Pruner{ + db: db, + stateBloom: stateBloom, + datadir: datadir, + trieCachePath: trieCachePath, + headHeader: headHeader, + snaptree: snaptree, + }, nil +} + +func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[common.Hash]struct{}, start time.Time) error { + // Delete all stale trie nodes in the disk. With the help of state bloom + // the trie nodes(and codes) belong to the active state will be filtered + // out. A very small part of stale tries will also be filtered because of + // the false-positive rate of bloom filter. But the assumption is held here + // that the false-positive is low enough(~0.05%). The probablity of the + // dangling node is the state root is super low. So the dangling nodes in + // theory will never ever be visited again. + var ( + count int + size common.StorageSize + pstart = time.Now() + logged = time.Now() + batch = maindb.NewBatch() + iter = maindb.NewIterator(nil, nil) + ) + for iter.Next() { + key := iter.Key() + + // All state entries don't belong to specific state and genesis are deleted here + // - trie node + // - legacy contract code + // - new-scheme contract code + isCode, codeKey := rawdb.IsCodeKey(key) + if len(key) == common.HashLength || isCode { + checkKey := key + if isCode { + checkKey = codeKey + } + if _, exist := middleStateRoots[common.BytesToHash(checkKey)]; exist { + log.Debug("Forcibly delete the middle state roots", "hash", common.BytesToHash(checkKey)) + } else { + if ok, err := stateBloom.Contain(checkKey); err != nil { + return err + } else if ok { + continue + } + } + count += 1 + size += common.StorageSize(len(key) + len(iter.Value())) + batch.Delete(key) + + var eta time.Duration // Realistically will never remain uninited + if done := binary.BigEndian.Uint64(key[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) + speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + eta = time.Duration(left/speed) * time.Millisecond + } + if time.Since(logged) > 8*time.Second { + log.Info("Pruning state data", "nodes", count, "size", size, + "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta)) + logged = time.Now() + } + // Recreate the iterator after every batch commit in order + // to allow the underlying compactor to delete the entries. + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + + iter.Release() + iter = maindb.NewIterator(nil, key) + } + } + } + if batch.ValueSize() > 0 { + batch.Write() + batch.Reset() + } + iter.Release() + log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + + // Start compactions, will remove the deleted data from the disk immediately. + // Note for small pruning, the compaction is skipped. + if count >= rangeCompactionThreshold { + cstart := time.Now() + + for b := byte(0); b < byte(16); b++ { + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", b, b+1), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := maindb.Compact([]byte{b}, []byte{b + 1}); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + } + log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + +// Prune deletes all historical state nodes except the nodes belong to the +// specified state version. If user doesn't specify the state version, use +// the bottom-most snapshot diff layer as the target. +func (p *Pruner) Prune(root common.Hash) error { + // If the state bloom filter is already committed previously, + // reuse it for pruning instead of generating a new one. It's + // mandatory because a part of state may already be deleted, + // the recovery procedure is necessary. + _, stateBloomRoot, err := findBloomFilter(p.datadir) + if err != nil { + return err + } + if stateBloomRoot != (common.Hash{}) { + return RecoverPruning(p.datadir, p.db, p.trieCachePath) + } + // If the target state root is not specified, use the HEAD-127 as the + // target. The reason for picking it is: + // - in most of the normal cases, the related state is available + // - the probability of this layer being reorg is very low + var layers []snapshot.Snapshot + if root == (common.Hash{}) { + // Retrieve all snapshot layers from the current HEAD. + // In theory there are 128 difflayers + 1 disk layer present, + // so 128 diff layers are expected to be returned. + layers = p.snaptree.Snapshots(p.headHeader.Root, 128, true) + if len(layers) != 128 { + // Reject if the accumulated diff layers are less than 128. It + // means in most of normal cases, there is no associated state + // with bottom-most diff layer. + return errors.New("the snapshot difflayers are less than 128") + } + // Use the bottom-most diff layer as the target + root = layers[len(layers)-1].Root() + } + // Ensure the root is really present. The weak assumption + // is the presence of root can indicate the presence of the + // entire trie. + if blob := rawdb.ReadTrieNode(p.db, root); len(blob) == 0 { + // The special case is for clique based networks(rinkeby, goerli + // and some other private networks), it's possible that two + // consecutive blocks will have same root. In this case snapshot + // difflayer won't be created. So HEAD-127 may not paired with + // head-127 layer. Instead the paired layer is higher than the + // bottom-most diff layer. Try to find the bottom-most snapshot + // layer with state available. + // + // Note HEAD and HEAD-1 is ignored. Usually there is the associated + // state available, but we don't want to use the topmost state + // as the pruning target. + var found bool + for i := len(layers) - 2; i >= 2; i-- { + if blob := rawdb.ReadTrieNode(p.db, layers[i].Root()); len(blob) != 0 { + root = layers[i].Root() + found = true + log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i) + break + } + } + if !found { + if len(layers) > 0 { + return errors.New("no snapshot paired state") + } + return fmt.Errorf("associated state[%x] is not present", root) + } + } else { + if len(layers) > 0 { + log.Info("Selecting bottom-most difflayer as the pruning target", "root", root, "height", p.headHeader.Number.Uint64()-127) + } else { + log.Info("Selecting user-specified state as the pruning target", "root", root) + } + } + // Before start the pruning, delete the clean trie cache first. + // It's necessary otherwise in the next restart we will hit the + // deleted state root in the "clean cache" so that the incomplete + // state is picked for usage. + deleteCleanTrieCache(p.trieCachePath) + + // All the state roots of the middle layer should be forcibly pruned, + // otherwise the dangling state will be left. + middleRoots := make(map[common.Hash]struct{}) + for _, layer := range layers { + if layer.Root() == root { + break + } + middleRoots[layer.Root()] = struct{}{} + } + // Traverse the target state, re-construct the whole state trie and + // commit to the given bloom filter. + start := time.Now() + if err := snapshot.GenerateTrie(p.snaptree, root, p.db, p.stateBloom); err != nil { + return err + } + // Traverse the genesis, put all genesis state entries into the + // bloom filter too. + if err := extractGenesis(p.db, p.stateBloom); err != nil { + return err + } + filterName := bloomFilterName(p.datadir, root) + + log.Info("Writing state bloom to disk", "name", filterName) + if err := p.stateBloom.Commit(filterName, filterName+stateBloomFileTempSuffix); err != nil { + return err + } + log.Info("State bloom filter committed", "name", filterName) + + if err := prune(p.db, p.stateBloom, middleRoots, start); err != nil { + return err + } + // Pruning is done, now drop the "useless" layers from the snapshot. + // Firstly, flushing the target layer into the disk. After that all + // diff layers below the target will all be merged into the disk. + if err := p.snaptree.Cap(root, 0); err != nil { + return err + } + // Secondly, flushing the snapshot journal into the disk. All diff + // layers upon the target layer are dropped silently. Eventually the + // entire snapshot tree is converted into a single disk layer with + // the pruning target as the root. + if _, err := p.snaptree.Journal(root); err != nil { + return err + } + // Delete the state bloom, it marks the entire pruning procedure is + // finished. If any crashes or manual exit happens before this, + // `RecoverPruning` will pick it up in the next restarts to redo all + // the things. + os.RemoveAll(filterName) + return nil +} + +// RecoverPruning will resume the pruning procedure during the system restart. +// This function is used in this case: user tries to prune state data, but the +// system was interrupted midway because of crash or manual-kill. In this case +// if the bloom filter for filtering active state is already constructed, the +// pruning can be resumed. What's more if the bloom filter is constructed, the +// pruning **has to be resumed**. Otherwise a lot of dangling nodes may be left +// in the disk. +func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) error { + stateBloomPath, stateBloomRoot, err := findBloomFilter(datadir) + if err != nil { + return err + } + if stateBloomPath == "" { + return nil // nothing to recover + } + headHeader, err := getHeadHeader(db) + if err != nil { + return err + } + // Initialize the snapshot tree in recovery mode to handle this special case: + // - Users run the `prune-state` command multiple times + // - Neither these `prune-state` running is finished(e.g. interrupted manually) + // - The state bloom filter is already generated, a part of state is deleted, + // so that resuming the pruning here is mandatory + // - The state HEAD is rewound already because of multiple incomplete `prune-state` + // In this case, even the state HEAD is not exactly matched with snapshot, it + // still feasible to recover the pruning correctly. + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headHeader.Root, false, false, true) + if err != nil { + return err // The relevant snapshot(s) might not exist + } + stateBloom, err := NewStateBloomFromDisk(stateBloomPath) + if err != nil { + return err + } + log.Info("Loaded state bloom filter", "path", stateBloomPath) + + // Before start the pruning, delete the clean trie cache first. + // It's necessary otherwise in the next restart we will hit the + // deleted state root in the "clean cache" so that the incomplete + // state is picked for usage. + deleteCleanTrieCache(trieCachePath) + + // All the state roots of the middle layers should be forcibly pruned, + // otherwise the dangling state will be left. + var ( + found bool + layers = snaptree.Snapshots(headHeader.Root, 128, true) + middleRoots = make(map[common.Hash]struct{}) + ) + for _, layer := range layers { + if layer.Root() == stateBloomRoot { + found = true + break + } + middleRoots[layer.Root()] = struct{}{} + } + if !found { + log.Error("Pruning target state is not existent") + return errors.New("non-existent target state") + } + if err := prune(db, stateBloom, middleRoots, time.Now()); err != nil { + return err + } + // Pruning is done, now drop the "useless" layers from the snapshot. + // Firstly, flushing the target layer into the disk. After that all + // diff layers below the target will all be merged into the disk. + if err := snaptree.Cap(stateBloomRoot, 0); err != nil { + return err + } + // Secondly, flushing the snapshot journal into the disk. All diff + // layers upon are dropped silently. Eventually the entire snapshot + // tree is converted into a single disk layer with the pruning target + // as the root. + if _, err := snaptree.Journal(stateBloomRoot); err != nil { + return err + } + // Delete the state bloom, it marks the entire pruning procedure is + // finished. If any crashes or manual exit happens before this, + // `RecoverPruning` will pick it up in the next restarts to redo all + // the things. + os.RemoveAll(stateBloomPath) + return nil +} + +// extractGenesis loads the genesis state and commits all the state entries +// into the given bloomfilter. +func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { + genesisHash := rawdb.ReadCanonicalHash(db, 0) + if genesisHash == (common.Hash{}) { + return errors.New("missing genesis hash") + } + genesis := rawdb.ReadBlock(db, genesisHash, 0) + if genesis == nil { + return errors.New("missing genesis block") + } + t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db)) + if err != nil { + return err + } + accIter := t.NodeIterator(nil) + for accIter.Next(true) { + hash := accIter.Hash() + + // Embedded nodes don't have hash. + if hash != (common.Hash{}) { + stateBloom.Put(hash.Bytes(), nil) + } + // If it's a leaf node, yes we are touching an account, + // dig into the storage trie further. + if accIter.Leaf() { + var acc state.Account + if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { + return err + } + if acc.Root != emptyRoot { + storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db)) + if err != nil { + return err + } + storageIter := storageTrie.NodeIterator(nil) + for storageIter.Next(true) { + hash := storageIter.Hash() + if hash != (common.Hash{}) { + stateBloom.Put(hash.Bytes(), nil) + } + } + if storageIter.Error() != nil { + return storageIter.Error() + } + } + if !bytes.Equal(acc.CodeHash, emptyCode) { + stateBloom.Put(acc.CodeHash, nil) + } + } + } + return accIter.Error() +} + +func bloomFilterName(datadir string, hash common.Hash) string { + return filepath.Join(datadir, fmt.Sprintf("%s.%s.%s", stateBloomFilePrefix, hash.Hex(), stateBloomFileSuffix)) +} + +func isBloomFilter(filename string) (bool, common.Hash) { + filename = filepath.Base(filename) + if strings.HasPrefix(filename, stateBloomFilePrefix) && strings.HasSuffix(filename, stateBloomFileSuffix) { + return true, common.HexToHash(filename[len(stateBloomFilePrefix)+1 : len(filename)-len(stateBloomFileSuffix)-1]) + } + return false, common.Hash{} +} + +func findBloomFilter(datadir string) (string, common.Hash, error) { + var ( + stateBloomPath string + stateBloomRoot common.Hash + ) + if err := filepath.Walk(datadir, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + ok, root := isBloomFilter(path) + if ok { + stateBloomPath = path + stateBloomRoot = root + } + } + return nil + }); err != nil { + return "", common.Hash{}, err + } + return stateBloomPath, stateBloomRoot, nil +} + +func getHeadHeader(db ethdb.Database) (*types.Header, error) { + headHeaderHash := rawdb.ReadHeadBlockHash(db) + if headHeaderHash == (common.Hash{}) { + return nil, errors.New("empty head block hash") + } + headHeaderNumber := rawdb.ReadHeaderNumber(db, headHeaderHash) + if headHeaderNumber == nil { + return nil, errors.New("empty head block number") + } + headHeader := rawdb.ReadHeader(db, headHeaderHash, *headHeaderNumber) + if headHeader == nil { + return nil, errors.New("empty head header") + } + return headHeader, nil +} + +const warningLog = ` + +WARNING! + +The clean trie cache is not found. Please delete it by yourself after the +pruning. Remember don't start the Geth without deleting the clean trie cache +otherwise the entire database may be damaged! + +Check the command description "geth snapshot prune-state --help" for more details. +` + +func deleteCleanTrieCache(path string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + log.Warn(warningLog) + return + } + os.RemoveAll(path) + log.Info("Deleted trie clean cache", "path", path) +} diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 4ec229b7ac..bb87ecddf1 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -18,12 +18,17 @@ package snapshot import ( "bytes" + "encoding/binary" + "errors" "fmt" + "math" + "runtime" "sync" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -38,46 +43,56 @@ type trieKV struct { type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(in chan trieKV, out chan common.Hash) + trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. - leafCallbackFn func(hash common.Hash, stat *generateStats) common.Hash + leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) ) // GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(it, common.Hash{}, stdGenerate, nil, &generateStats{start: time.Now()}, true) + return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) } // GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(it, account, stdGenerate, nil, &generateStats{start: time.Now()}, true) + return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true) } -// VerifyState takes the whole snapshot tree as the input, traverses all the accounts -// as well as the corresponding storages and compares the re-computed hash with the -// original one(state root and the storage root). -func VerifyState(snaptree *Tree, root common.Hash) error { +// GenerateTrie takes the whole snapshot tree as the input, traverses all the +// accounts as well as the corresponding storages and regenerate the whole state +// (account trie + all storage tries). +func GenerateTrie(snaptree *Tree, root common.Hash, src ethdb.Database, dst ethdb.KeyValueWriter) error { + // Traverse all state by snapshot, re-generate the whole state trie acctIt, err := snaptree.AccountIterator(root, common.Hash{}) if err != nil { - return err + return err // The required snapshot might not exist. } defer acctIt.Release() - got, err := generateTrieRoot(acctIt, common.Hash{}, stdGenerate, func(account common.Hash, stat *generateStats) common.Hash { - storageIt, err := snaptree.StorageIterator(root, account, common.Hash{}) + got, err := generateTrieRoot(dst, acctIt, common.Hash{}, stackTrieGenerate, func(dst ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + // Migrate the code first, commit the contract code into the tmp db. + if codeHash != emptyCode { + code := rawdb.ReadCode(src, codeHash) + if len(code) == 0 { + return common.Hash{}, errors.New("failed to read contract code") + } + rawdb.WriteCode(dst, codeHash, code) + } + // Then migrate all storage trie nodes into the tmp db. + storageIt, err := snaptree.StorageIterator(root, accountHash, common.Hash{}) if err != nil { - return common.Hash{} + return common.Hash{}, err } defer storageIt.Release() - hash, err := generateTrieRoot(storageIt, account, stdGenerate, nil, stat, false) + hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false) if err != nil { - return common.Hash{} + return common.Hash{}, err } - return hash - }, &generateStats{start: time.Now()}, true) + return hash, nil + }, newGenerateStats(), true) if err != nil { return err @@ -91,23 +106,64 @@ func VerifyState(snaptree *Tree, root common.Hash) error { // generateStats is a collection of statistics gathered by the trie generator // for logging purposes. type generateStats struct { - accounts uint64 - slots uint64 - curAccount common.Hash - curSlot common.Hash - start time.Time - lock sync.RWMutex + head common.Hash + start time.Time + + accounts uint64 // Number of accounts done (including those being crawled) + slots uint64 // Number of storage slots done (including those being crawled) + + slotsStart map[common.Hash]time.Time // Start time for account slot crawling + slotsHead map[common.Hash]common.Hash // Slot head for accounts being crawled + + lock sync.RWMutex } -// progress records the progress trie generator made recently. -func (stat *generateStats) progress(accounts, slots uint64, curAccount common.Hash, curSlot common.Hash) { +// newGenerateStats creates a new generator stats. +func newGenerateStats() *generateStats { + return &generateStats{ + slotsStart: make(map[common.Hash]time.Time), + slotsHead: make(map[common.Hash]common.Hash), + start: time.Now(), + } +} + +// progressAccounts updates the generator stats for the account range. +func (stat *generateStats) progressAccounts(account common.Hash, done uint64) { stat.lock.Lock() defer stat.lock.Unlock() - stat.accounts += accounts - stat.slots += slots - stat.curAccount = curAccount - stat.curSlot = curSlot + stat.accounts += done + stat.head = account +} + +// finishAccounts updates the gemerator stats for the finished account range. +func (stat *generateStats) finishAccounts(done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.accounts += done +} + +// progressContract updates the generator stats for a specific in-progress contract. +func (stat *generateStats) progressContract(account common.Hash, slot common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.slots += done + stat.slotsHead[account] = slot + if _, ok := stat.slotsStart[account]; !ok { + stat.slotsStart[account] = time.Now() + } +} + +// finishContract updates the generator stats for a specific just-finished contract. +func (stat *generateStats) finishContract(account common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.slots += done + delete(stat.slotsHead, account) + delete(stat.slotsStart, account) } // report prints the cumulative progress statistic smartly. @@ -115,22 +171,39 @@ func (stat *generateStats) report() { stat.lock.RLock() defer stat.lock.RUnlock() - var ctx []interface{} - if stat.curSlot != (common.Hash{}) { - ctx = append(ctx, []interface{}{ - "in", stat.curAccount, - "at", stat.curSlot, - }...) - } else { - ctx = append(ctx, []interface{}{"at", stat.curAccount}...) + ctx := []interface{}{ + "accounts", stat.accounts, + "slots", stat.slots, + "elapsed", common.PrettyDuration(time.Since(stat.start)), } - // Add the usual measurements - ctx = append(ctx, []interface{}{"accounts", stat.accounts}...) - if stat.slots != 0 { - ctx = append(ctx, []interface{}{"slots", stat.slots}...) + if stat.accounts > 0 { + // If there's progress on the account trie, estimate the time to finish crawling it + if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 { + var ( + left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts + speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + eta = time.Duration(left/speed) * time.Millisecond + ) + // If there are large contract crawls in progress, estimate their finish time + for acc, head := range stat.slotsHead { + start := stat.slotsStart[acc] + if done := binary.BigEndian.Uint64(head[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8]) + speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + // Override the ETA if larger than the largest until now + if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA { + eta = slotETA + } + } + } + ctx = append(ctx, []interface{}{ + "eta", common.PrettyDuration(eta), + }...) + } } - ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) - log.Info("Generating trie hash from snapshot", ctx...) + log.Info("Iterating state snapshot", ctx...) } // reportDone prints the last log when the whole generation is finished. @@ -144,13 +217,32 @@ func (stat *generateStats) reportDone() { ctx = append(ctx, []interface{}{"slots", stat.slots}...) } ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) - log.Info("Generated trie hash from snapshot", ctx...) + log.Info("Iterated snapshot", ctx...) +} + +// runReport periodically prints the progress information. +func runReport(stats *generateStats, stop chan bool) { + timer := time.NewTimer(0) + defer timer.Stop() + + for { + select { + case <-timer.C: + stats.report() + timer.Reset(time.Second * 8) + case success := <-stop: + if success { + stats.reportDone() + } + return + } + } } // generateTrieRoot generates the trie hash based on the snapshot iterator. // It can be used for generating account trie, storage trie or even the // whole state which connects the accounts and the corresponding storages. -func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { +func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { var ( in = make(chan trieKV) // chan to pass leaves out = make(chan common.Hash, 1) // chan to collect result @@ -161,46 +253,43 @@ func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieGenerato wg.Add(1) go func() { defer wg.Done() - generatorFn(in, out) + generatorFn(db, in, out) }() - // Spin up a go-routine for progress logging if report && stats != nil { wg.Add(1) go func() { defer wg.Done() - - timer := time.NewTimer(0) - defer timer.Stop() - - for { - select { - case <-timer.C: - stats.report() - timer.Reset(time.Second * 8) - case success := <-stoplog: - if success { - stats.reportDone() - } - return - } - } + runReport(stats, stoplog) }() } + // Create a semaphore to assign tasks and collect results through. We'll pre- + // fill it with nils, thus using the same channel for both limiting concurrent + // processing and gathering results. + threads := runtime.NumCPU() + results := make(chan error, threads) + for i := 0; i < threads; i++ { + results <- nil // fill the semaphore + } // stop is a helper function to shutdown the background threads // and return the re-generated trie hash. - stop := func(success bool) common.Hash { + stop := func(fail error) (common.Hash, error) { close(in) result := <-out - stoplog <- success + for i := 0; i < threads; i++ { + if err := <-results; err != nil && fail == nil { + fail = err + } + } + stoplog <- fail == nil + wg.Wait() - return result + return result, fail } var ( logged = time.Now() processed = uint64(0) leaf trieKV - last common.Hash ) // Start to feed leaves for it.Next() { @@ -212,26 +301,35 @@ func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieGenerato if leafCallback == nil { fullData, err = FullAccountRLP(it.(AccountIterator).Account()) if err != nil { - stop(false) - return common.Hash{}, err + return stop(err) } } else { + // Wait until the semaphore allows us to continue, aborting if + // a sub-task failed + if err := <-results; err != nil { + results <- nil // stop will drain the results, add a noop back for this error we just consumed + return stop(err) + } + // Fetch the next account and process it concurrently account, err := FullAccount(it.(AccountIterator).Account()) if err != nil { - stop(false) - return common.Hash{}, err - } - // Apply the leaf callback. Normally the callback is used to traverse - // the storage trie and re-generate the subtrie root. - subroot := leafCallback(it.Hash(), stats) - if !bytes.Equal(account.Root, subroot.Bytes()) { - stop(false) - return common.Hash{}, fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot) + return stop(err) } + go func(hash common.Hash) { + subroot, err := leafCallback(db, hash, common.BytesToHash(account.CodeHash), stats) + if err != nil { + results <- err + return + } + if !bytes.Equal(account.Root, subroot.Bytes()) { + results <- fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot) + return + } + results <- nil + }(it.Hash()) fullData, err = rlp.EncodeToBytes(account) if err != nil { - stop(false) - return common.Hash{}, err + return stop(err) } } leaf = trieKV{it.Hash(), fullData} @@ -244,32 +342,34 @@ func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieGenerato processed++ if time.Since(logged) > 3*time.Second && stats != nil { if account == (common.Hash{}) { - stats.progress(processed, 0, it.Hash(), common.Hash{}) + stats.progressAccounts(it.Hash(), processed) } else { - stats.progress(0, processed, account, it.Hash()) + stats.progressContract(account, it.Hash(), processed) } logged, processed = time.Now(), 0 } - last = it.Hash() } // Commit the last part statistic. if processed > 0 && stats != nil { if account == (common.Hash{}) { - stats.progress(processed, 0, last, common.Hash{}) + stats.finishAccounts(processed) } else { - stats.progress(0, processed, account, last) + stats.finishContract(account, processed) } } - result := stop(true) - return result, nil + return stop(nil) } -// stdGenerate is a very basic hexary trie builder which uses the same Trie -// as the rest of geth, with no enhancements or optimizations -func stdGenerate(in chan trieKV, out chan common.Hash) { - t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New())) +func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { + t := trie.NewStackTrie(db) for leaf := range in { t.TryUpdate(leaf.key[:], leaf.value) } - out <- t.Hash() + var root common.Hash + if db == nil { + root = t.Hash() + } else { + root, _ = t.Commit() + } + out <- root } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 443fc8e5c7..df2b1ed330 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -178,7 +178,7 @@ type Tree struct { // store, on a background thread. If the memory layers from the journal is not // continuous with disk layer or the journal is missing, all diffs will be discarded // iff it's in "recovery" mode, otherwise rebuild is mandatory. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool, recovery bool) *Tree { +func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ diskdb: diskdb, @@ -192,16 +192,19 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm // Attempt to load a previously persisted snapshot and rebuild one if failed head, err := loadSnapshot(diskdb, triedb, cache, root, recovery) if err != nil { - log.Warn("Failed to load snapshot, regenerating", "err", err) - snap.Rebuild(root) - return snap + if rebuild { + log.Warn("Failed to load snapshot, regenerating", "err", err) + snap.Rebuild(root) + return snap, nil + } + return nil, err // Bail out the error, don't rebuild automatically. } // Existing snapshot loaded, seed all the layers for head != nil { snap.layers[head.Root()] = head head = head.Parent() } - return snap + return snap, nil } // waitBuild blocks until the snapshot finishes rebuilding. This method is meant @@ -234,6 +237,39 @@ func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot { return t.layers[blockRoot] } +// Snapshots returns all visited layers from the topmost layer with specific +// root and traverses downward. The layer amount is limited by the given number. +// If nodisk is set, then disk layer is excluded. +func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot { + t.lock.RLock() + defer t.lock.RUnlock() + + if limits == 0 { + return nil + } + layer := t.layers[root] + if layer == nil { + return nil + } + var ret []Snapshot + for { + if _, isdisk := layer.(*diskLayer); isdisk && nodisk { + break + } + ret = append(ret, layer) + limits -= 1 + if limits == 0 { + break + } + parent := layer.Parent() + if parent == nil { + break + } + layer = parent + } + return ret +} + // Update adds a new snapshot into the tree, if that can be linked to an existing // old parent. It is disallowed to insert a disk layer (the origin of all). func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { @@ -681,6 +717,38 @@ func (t *Tree) StorageIterator(root common.Hash, account common.Hash, seek commo return newFastStorageIterator(t, root, account, seek) } +// Verify iterates the whole state(all the accounts as well as the corresponding storages) +// with the specific root and compares the re-computed hash with the original one. +func (t *Tree) Verify(root common.Hash) error { + acctIt, err := t.AccountIterator(root, common.Hash{}) + if err != nil { + return err + } + defer acctIt.Release() + + got, err := generateTrieRoot(nil, acctIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + storageIt, err := t.StorageIterator(root, accountHash, common.Hash{}) + if err != nil { + return common.Hash{}, err + } + defer storageIt.Release() + + hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true) + + if err != nil { + return err + } + if got != root { + return fmt.Errorf("state root hash mismatch: got %x, want %x", got, root) + } + return nil +} + // disklayer is an internal helper function to return the disk layer. // The lock of snapTree is assumed to be held already. func (t *Tree) disklayer() *diskLayer { diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index ca4fa0a055..a315fd216c 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -17,6 +17,7 @@ package snapshot import ( + "encoding/binary" "fmt" "math/big" "math/rand" @@ -369,3 +370,69 @@ func TestPostCapBasicDataAccess(t *testing.T) { t.Error("expected error capping the disk layer, got none") } } + +// TestSnaphots tests the functionality for retrieveing the snapshot +// with given head root and the desired depth. +func TestSnaphots(t *testing.T) { + // setAccount is a helper to construct a random account entry and assign it to + // an account slot in a snapshot + setAccount := func(accKey string) map[common.Hash][]byte { + return map[common.Hash][]byte{ + common.HexToHash(accKey): randomAccount(), + } + } + makeRoot := func(height uint64) common.Hash { + var buffer [8]byte + binary.BigEndian.PutUint64(buffer[:], height) + return common.BytesToHash(buffer[:]) + } + // Create a starting base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Construct the snapshots with 128 layers + var ( + last = common.HexToHash("0x01") + head common.Hash + ) + // Flush another 128 layers, one diff will be flatten into the parent. + for i := 0; i < 128; i++ { + head = makeRoot(uint64(i + 2)) + snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) + last = head + snaps.Cap(head, 128) // 129 layers(128 diffs + 1 disk) are allowed, 129th is the persistent layer + } + var cases = []struct { + headRoot common.Hash + limit int + nodisk bool + expected int + expectBottom common.Hash + }{ + {head, 0, false, 0, common.Hash{}}, + {head, 64, false, 64, makeRoot(127 + 2 - 63)}, + {head, 128, false, 128, makeRoot(2)}, // All diff layers + {head, 129, true, 128, makeRoot(2)}, // All diff layers + {head, 129, false, 129, common.HexToHash("0x01")}, // All diff layers + disk layer + } + for _, c := range cases { + layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) + if len(layers) != c.expected { + t.Fatalf("Returned snapshot layers are mismatched, want %v, got %v", c.expected, len(layers)) + } + if len(layers) == 0 { + continue + } + bottommost := layers[len(layers)-1] + if bottommost.Root() != c.expectBottom { + t.Fatalf("Snapshot mismatch, want %v, get %v", c.expectBottom, bottommost.Root()) + } + } +} diff --git a/eth/backend.go b/eth/backend.go index 51ad0ccf3f..4170ecc94e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -131,6 +132,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Initialised chain configuration", "config", chainConfig) + if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { + log.Error("Failed to recover state", "error", err) + } eth := &Ethereum{ config: config, chainDb: chainDb, diff --git a/go.mod b/go.mod index 5fd3f772a4..cddce4e0d8 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,7 @@ module github.com/ethereum/go-ethereum go 1.13 require ( - github.com/Azure/azure-pipeline-go v0.2.2 // indirect github.com/Azure/azure-storage-blob-go v0.7.0 - github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.5.7 github.com/aws/aws-sdk-go v1.25.48 github.com/btcsuite/btcd v0.20.1-beta @@ -15,15 +12,12 @@ require ( github.com/consensys/gurvy v0.3.8 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea - github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 github.com/edsrzf/mmap-go v1.0.0 - github.com/fatih/color v1.3.0 + github.com/fatih/color v1.7.0 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/go-ole/go-ole v1.2.1 // indirect - github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 @@ -39,15 +33,13 @@ require ( github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 - github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 - github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 - github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 + github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v2.20.5+incompatible @@ -56,12 +48,10 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect - golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 - gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index 1ac0cf81f2..4b788824c7 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,7 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -253,8 +254,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 h1:HarGZ5h9HD9LgEg1yRVMXyfiw4wlXiLiYM2oMjeA/SE= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= @@ -263,6 +264,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= @@ -275,12 +277,10 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJye github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -298,7 +298,6 @@ github.com/kilic/bls12-381 v0.0.0-20201226121925-69dacb279461/go.mod h1:vDTTHJON github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -401,7 +400,6 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -440,13 +438,11 @@ github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57N github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -570,7 +566,6 @@ golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/tests/block_test_util.go b/tests/block_test_util.go index c043f0b3eb..745208b5b8 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" @@ -147,7 +146,7 @@ func (t *BlockTest) Run(snapshotter bool) error { } // Cross-check the snapshot-to-hash against the trie hash if snapshotter { - if err := snapshot.VerifyState(chain.Snapshots(), chain.CurrentBlock().Root()); err != nil { + if err := chain.Snapshots().Verify(chain.CurrentBlock().Root()); err != nil { return err } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 7bd2e801e4..53c7d955be 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -236,7 +236,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps = snapshot.New(db, sdb.TrieDB(), 1, root, false, false) + snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, root, false, true, false) } statedb, _ = state.New(root, sdb, snaps) return snaps, statedb diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 575a04022f..a198eb204b 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -35,7 +35,7 @@ var stPool = sync.Pool{ }, } -func stackTrieFromPool(db ethdb.KeyValueStore) *StackTrie { +func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie { st := stPool.Get().(*StackTrie) st.db = db return st @@ -50,24 +50,23 @@ func returnToPool(st *StackTrie) { // in order. Once it determines that a subtree will no longer be inserted // into, it will hash it and free up the memory it uses. type StackTrie struct { - nodeType uint8 // node type (as in branch, ext, leaf) - val []byte // value contained by this node if it's a leaf - key []byte // key chunk covered by this (full|ext) node - keyOffset int // offset of the key chunk inside a full key - children [16]*StackTrie // list of children (for fullnodes and exts) - - db ethdb.KeyValueStore // Pointer to the commit db, can be nil + nodeType uint8 // node type (as in branch, ext, leaf) + val []byte // value contained by this node if it's a leaf + key []byte // key chunk covered by this (full|ext) node + keyOffset int // offset of the key chunk inside a full key + children [16]*StackTrie // list of children (for fullnodes and exts) + db ethdb.KeyValueWriter // Pointer to the commit db, can be nil } // NewStackTrie allocates and initializes an empty trie. -func NewStackTrie(db ethdb.KeyValueStore) *StackTrie { +func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie { return &StackTrie{ nodeType: emptyNode, db: db, } } -func newLeaf(ko int, key, val []byte, db ethdb.KeyValueStore) *StackTrie { +func newLeaf(ko int, key, val []byte, db ethdb.KeyValueWriter) *StackTrie { st := stackTrieFromPool(db) st.nodeType = leafNode st.keyOffset = ko @@ -76,7 +75,7 @@ func newLeaf(ko int, key, val []byte, db ethdb.KeyValueStore) *StackTrie { return st } -func newExt(ko int, key []byte, child *StackTrie, db ethdb.KeyValueStore) *StackTrie { +func newExt(ko int, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { st := stackTrieFromPool(db) st.nodeType = extNode st.keyOffset = ko From 2728672c28183da21028379ea5497debe92325b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 8 Feb 2021 17:02:30 +0200 Subject: [PATCH 124/709] core/state/pruner: fix compaction after pruning --- core/state/pruner/pruner.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 1cb65b38b1..84a8952b57 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -188,8 +188,15 @@ func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[c cstart := time.Now() for b := byte(0); b < byte(16); b++ { - log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", b, b+1), "elapsed", common.PrettyDuration(time.Since(cstart))) - if err := maindb.Compact([]byte{b}, []byte{b + 1}); err != nil { + var ( + start = []byte{b << 4} + end = []byte{(b+1)<<4 - 1} + ) + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if b == 15 { + end = nil + } + if err := maindb.Compact(start, end); err != nil { log.Error("Database compaction failed", "error", err) return err } @@ -229,7 +236,7 @@ func (p *Pruner) Prune(root common.Hash) error { // Reject if the accumulated diff layers are less than 128. It // means in most of normal cases, there is no associated state // with bottom-most diff layer. - return errors.New("the snapshot difflayers are less than 128") + return fmt.Errorf("snapshot not old enough yet: need %d more blocks", 128-len(layers)) } // Use the bottom-most diff layer as the target root = layers[len(layers)-1].Root() From 74dbc20260caac0159ca59bfad1f41321130e676 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 8 Feb 2021 20:31:52 +0100 Subject: [PATCH 125/709] core/state/pruner: fix compaction range error --- core/state/pruner/pruner.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 84a8952b57..1fbfa55b6a 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -186,16 +186,15 @@ func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[c // Note for small pruning, the compaction is skipped. if count >= rangeCompactionThreshold { cstart := time.Now() - - for b := byte(0); b < byte(16); b++ { + for b := 0x00; b <= 0xf0; b += 0x10 { var ( - start = []byte{b << 4} - end = []byte{(b+1)<<4 - 1} + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} ) - log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) - if b == 15 { + if b == 0xf0 { end = nil } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) if err := maindb.Compact(start, end); err != nil { log.Error("Database compaction failed", "error", err) return err From 27786671d28705159f15cd458045d29d732110e5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 9 Feb 2021 10:42:55 +0100 Subject: [PATCH 126/709] internal/debug: add switch to format logs with json (#22207) adds a flag --log.json which if enabled makes the client format logs with JSON. --- internal/debug/flags.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index e3a01378ca..fc4ea9f518 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -41,6 +41,10 @@ var ( Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", Value: 3, } + logjsonFlag = cli.BoolFlag{ + Name: "log.json", + Usage: "Format logs with JSON", + } vmoduleFlag = cli.StringFlag{ Name: "vmodule", Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", @@ -114,7 +118,7 @@ var ( // Flags holds all command-line flags required for debugging. var Flags = []cli.Flag{ - verbosityFlag, vmoduleFlag, backtraceAtFlag, debugFlag, + verbosityFlag, logjsonFlag, vmoduleFlag, backtraceAtFlag, debugFlag, pprofFlag, pprofAddrFlag, pprofPortFlag, memprofilerateFlag, blockprofilerateFlag, cpuprofileFlag, traceFlag, } @@ -124,24 +128,29 @@ var DeprecatedFlags = []cli.Flag{ legacyBlockprofilerateFlag, legacyCpuprofileFlag, } -var ( - ostream log.Handler - glogger *log.GlogHandler -) +var glogger *log.GlogHandler func init() { - usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" - output := io.Writer(os.Stderr) - if usecolor { - output = colorable.NewColorableStderr() - } - ostream = log.StreamHandler(output, log.TerminalFormat(usecolor)) - glogger = log.NewGlogHandler(ostream) + glogger = log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) + glogger.Verbosity(log.LvlInfo) + log.Root().SetHandler(glogger) } // Setup initializes profiling and logging based on the CLI flags. // It should be called as early as possible in the program. func Setup(ctx *cli.Context) error { + var ostream log.Handler + output := io.Writer(os.Stderr) + if ctx.GlobalBool(logjsonFlag.Name) { + ostream = log.StreamHandler(output, log.JSONFormat()) + } else { + usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" + if usecolor { + output = colorable.NewColorableStderr() + } + ostream = log.StreamHandler(output, log.TerminalFormat(usecolor)) + } + glogger.SetHandler(ostream) // logging log.PrintOrigins(ctx.GlobalBool(debugFlag.Name)) glogger.Verbosity(log.Lvl(ctx.GlobalInt(verbosityFlag.Name))) From cb3c7e431978f0bd5efb19b378aa9e42d940986d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 10 Feb 2021 13:12:13 +0100 Subject: [PATCH 127/709] accounts/abi/bind: fixed unpacking error (#22230) There was a dormant error with structured inputs that failed unpacking. This commit fixes the error by switching casting to the better abi.ConvertType function. It also adds a test for calling a view function that returns a struct --- accounts/abi/abi_test.go | 3 ++ accounts/abi/bind/bind_test.go | 64 ++++++++++++++++++++++++++++++++++ accounts/abi/bind/template.go | 2 +- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index ad8acdf522..a022ec5f9d 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -43,6 +43,7 @@ const jsondata = ` { "type" : "function", "name" : "uint64[2]", "inputs" : [ { "name" : "inputs", "type" : "uint64[2]" } ] }, { "type" : "function", "name" : "uint64[]", "inputs" : [ { "name" : "inputs", "type" : "uint64[]" } ] }, { "type" : "function", "name" : "int8", "inputs" : [ { "name" : "inputs", "type" : "int8" } ] }, + { "type" : "function", "name" : "bytes32", "inputs" : [ { "name" : "inputs", "type" : "bytes32" } ] }, { "type" : "function", "name" : "foo", "inputs" : [ { "name" : "inputs", "type" : "uint32" } ] }, { "type" : "function", "name" : "bar", "inputs" : [ { "name" : "inputs", "type" : "uint32" }, { "name" : "string", "type" : "uint16" } ] }, { "type" : "function", "name" : "slice", "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] }, @@ -68,6 +69,7 @@ var ( String, _ = NewType("string", "", nil) Bool, _ = NewType("bool", "", nil) Bytes, _ = NewType("bytes", "", nil) + Bytes32, _ = NewType("bytes32", "", nil) Address, _ = NewType("address", "", nil) Uint64Arr, _ = NewType("uint64[]", "", nil) AddressArr, _ = NewType("address[]", "", nil) @@ -98,6 +100,7 @@ var methods = map[string]Method{ "uint64[]": NewMethod("uint64[]", "uint64[]", Function, "", false, false, []Argument{{"inputs", Uint64Arr, false}}, nil), "uint64[2]": NewMethod("uint64[2]", "uint64[2]", Function, "", false, false, []Argument{{"inputs", Uint64Arr2, false}}, nil), "int8": NewMethod("int8", "int8", Function, "", false, false, []Argument{{"inputs", Int8, false}}, nil), + "bytes32": NewMethod("bytes32", "bytes32", Function, "", false, false, []Argument{{"inputs", Bytes32, false}}, nil), "foo": NewMethod("foo", "foo", Function, "", false, false, []Argument{{"inputs", Uint32, false}}, nil), "bar": NewMethod("bar", "bar", Function, "", false, false, []Argument{{"inputs", Uint32, false}, {"string", Uint16, false}}, nil), "slice": NewMethod("slice", "slice", Function, "", false, false, []Argument{{"inputs", Uint32Arr2, false}}, nil), diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index db87703d03..400545fd23 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -529,6 +529,70 @@ var bindTests = []struct { nil, nil, }, + // Tests that structs are correctly unpacked + { + + `Structs`, + ` + pragma solidity ^0.6.5; + pragma experimental ABIEncoderV2; + contract Structs { + struct A { + bytes32 B; + } + + function F() public view returns (A[] memory a, uint256[] memory c, bool[] memory d) { + A[] memory a = new A[](2); + a[0].B = bytes32(uint256(1234) << 96); + uint256[] memory c; + bool[] memory d; + return (a, c, d); + } + + function G() public view returns (A[] memory a) { + A[] memory a = new A[](2); + a[0].B = bytes32(uint256(1234) << 96); + return a; + } + } + `, + []string{`608060405234801561001057600080fd5b50610278806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806328811f591461003b5780636fecb6231461005b575b600080fd5b610043610070565b604051610052939291906101a0565b60405180910390f35b6100636100d6565b6040516100529190610186565b604080516002808252606082810190935282918291829190816020015b610095610131565b81526020019060019003908161008d575050805190915061026960611b9082906000906100be57fe5b60209081029190910101515293606093508392509050565b6040805160028082526060828101909352829190816020015b6100f7610131565b8152602001906001900390816100ef575050805190915061026960611b90829060009061012057fe5b602090810291909101015152905090565b60408051602081019091526000815290565b815260200190565b6000815180845260208085019450808401835b8381101561017b578151518752958201959082019060010161015e565b509495945050505050565b600060208252610199602083018461014b565b9392505050565b6000606082526101b3606083018661014b565b6020838203818501528186516101c98185610239565b91508288019350845b818110156101f3576101e5838651610143565b9484019492506001016101d2565b505084810360408601528551808252908201925081860190845b8181101561022b57825115158552938301939183019160010161020d565b509298975050505050505050565b9081526020019056fea2646970667358221220eb85327e285def14230424c52893aebecec1e387a50bb6b75fc4fdbed647f45f64736f6c63430006050033`}, + []string{`[{"inputs":[],"name":"F","outputs":[{"components":[{"internalType":"bytes32","name":"B","type":"bytes32"}],"internalType":"structStructs.A[]","name":"a","type":"tuple[]"},{"internalType":"uint256[]","name":"c","type":"uint256[]"},{"internalType":"bool[]","name":"d","type":"bool[]"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"G","outputs":[{"components":[{"internalType":"bytes32","name":"B","type":"bytes32"}],"internalType":"structStructs.A[]","name":"a","type":"tuple[]"}],"stateMutability":"view","type":"function"}]`}, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, + ` + // Generate a new random account and a funded simulator + key, _ := crypto.GenerateKey() + auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) + defer sim.Close() + + // Deploy a structs method invoker contract and execute its default method + _, _, structs, err := DeployStructs(auth, sim) + if err != nil { + t.Fatalf("Failed to deploy defaulter contract: %v", err) + } + sim.Commit() + opts := bind.CallOpts{} + if _, err := structs.F(&opts); err != nil { + t.Fatalf("Failed to invoke F method: %v", err) + } + if _, err := structs.G(&opts); err != nil { + t.Fatalf("Failed to invoke G method: %v", err) + } + `, + nil, + nil, + nil, + nil, + }, // Tests that non-existent contracts are reported as such (though only simulator test) { `NonExistent`, diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index 351eabd258..e9bdd3d414 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -308,7 +308,7 @@ var ( return *outstruct, err } {{range $i, $t := .Normalized.Outputs}} - outstruct.{{.Name}} = out[{{$i}}].({{bindtype .Type $structs}}){{end}} + outstruct.{{.Name}} = *abi.ConvertType(out[{{$i}}], new({{bindtype .Type $structs}})).(*{{bindtype .Type $structs}}){{end}} return *outstruct, err {{else}} From 409b16e5abac3a48c21142fdfa68d33cf6c95fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 8 Feb 2021 20:44:05 +0200 Subject: [PATCH 128/709] cmd/utils, eth/ethconfig: unindex txs older than ~1 year --- cmd/utils/flags.go | 6 +++--- eth/ethconfig/config.go | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f9c5553961..0d7b0e1bf5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -212,10 +212,10 @@ var ( Name: "snapshot", Usage: `Enables snapshot-database mode (default = enable)`, } - TxLookupLimitFlag = cli.Int64Flag{ + TxLookupLimitFlag = cli.Uint64Flag{ Name: "txlookuplimit", - Usage: "Number of recent blocks to maintain transactions index by-hash for (default = index all blocks)", - Value: 0, + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", + Value: ethconfig.Defaults.TxLookupLimit, } LightKDFFlag = cli.BoolFlag{ Name: "lightkdf", diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 9147a602d5..e192e4d333 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -66,6 +66,7 @@ var Defaults = Config{ DatasetsLockMmap: false, }, NetworkId: 1, + TxLookupLimit: 2350000, LightPeers: 100, UltraLightFraction: 75, DatabaseCache: 512, From 111abdcfbdc3c73b527589dce7863d3b93eca91d Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 11 Feb 2021 12:09:13 +0100 Subject: [PATCH 129/709] cmd/devp2p: fix documentation for eth-test (#22298) --- cmd/devp2p/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/devp2p/README.md b/cmd/devp2p/README.md index e1372d0158..ca0b852fda 100644 --- a/cmd/devp2p/README.md +++ b/cmd/devp2p/README.md @@ -94,10 +94,12 @@ To run the eth protocol test suite against your implementation, the node needs t geth --datadir --nodiscover --nat=none --networkid 19763 --verbosity 5 ``` -Then, run the following command, replacing `` with the enode of the geth node: +Then, run the following command, replacing `` with the enode of the geth node: ``` - devp2p rlpx eth-test cmd/devp2p/internal/ethtest/testdata/fullchain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json + devp2p rlpx eth-test cmd/devp2p/internal/ethtest/testdata/chain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json ``` + +Repeat the above process (re-initialising the node) in order to run the Eth Protocol test suite again. [eth]: https://github.com/ethereum/devp2p/blob/master/caps/eth.md [dns-tutorial]: https://geth.ethereum.org/docs/developers/dns-discovery-setup From ef227c5f42a2e180b0e3b57d38ef5018fc8733d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 12 Feb 2021 12:45:34 +0200 Subject: [PATCH 130/709] core: fix temp memory blowup caused by defers holding on to state --- core/blockchain.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 7dd1097e98..d65ce4f048 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1805,6 +1805,17 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er return it.index, err } // No validation errors for the first block (or chain prefix skipped) + var activeState *state.StateDB + defer func() { + // The chain importer is starting and stopping trie prefetchers. If a bad + // block or other error is hit however, an early return may not properly + // terminate the background threads. This defer ensures that we clean up + // and dangling prefetcher, without defering each and holding on live refs. + if activeState != nil { + activeState.StopPrefetcher() + } + }() + for ; block != nil && err == nil || err == ErrKnownBlock; block, err = it.next() { // If the chain is terminating, stop processing blocks if bc.insertStopped() { @@ -1867,7 +1878,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er } // Enable prefetching to pull in trie node paths while processing transactions statedb.StartPrefetcher("chain") - defer statedb.StopPrefetcher() // stopped on write anyway, defer meant to catch early error returns + activeState = statedb // If we have a followup block, run that against the current state to pre-cache // transactions and probabilistically some of the account/storage trie nodes. From 7d1b711c7d0f27efd7772c81bb73b9b29720515a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 12 Feb 2021 20:48:18 +0100 Subject: [PATCH 131/709] les: enable les/4 and add tests (#22321) --- les/handler_test.go | 10 ++++++++++ les/odr_test.go | 5 +++++ les/protocol.go | 4 ++-- les/request_test.go | 4 ++++ les/test_helper.go | 12 ++++++++++-- 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/les/handler_test.go b/les/handler_test.go index f5cbeb8efc..83be312081 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -49,6 +49,7 @@ func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{} // Tests that block headers can be retrieved from a remote chain based on user queries. func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func TestGetBlockHeadersLes3(t *testing.T) { testGetBlockHeaders(t, 3) } +func TestGetBlockHeadersLes4(t *testing.T) { testGetBlockHeaders(t, 4) } func testGetBlockHeaders(t *testing.T, protocol int) { server, tearDown := newServerEnv(t, downloader.MaxHeaderFetch+15, protocol, nil, false, true, 0) @@ -178,6 +179,7 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Tests that block contents can be retrieved from a remote chain based on their hashes. func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func TestGetBlockBodiesLes3(t *testing.T) { testGetBlockBodies(t, 3) } +func TestGetBlockBodiesLes4(t *testing.T) { testGetBlockBodies(t, 4) } func testGetBlockBodies(t *testing.T, protocol int) { server, tearDown := newServerEnv(t, downloader.MaxBlockFetch+15, protocol, nil, false, true, 0) @@ -255,6 +257,7 @@ func testGetBlockBodies(t *testing.T, protocol int) { // Tests that the contract codes can be retrieved based on account addresses. func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func TestGetCodeLes3(t *testing.T) { testGetCode(t, 3) } +func TestGetCodeLes4(t *testing.T) { testGetCode(t, 4) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment @@ -285,6 +288,7 @@ func testGetCode(t *testing.T, protocol int) { // Tests that the stale contract codes can't be retrieved based on account addresses. func TestGetStaleCodeLes2(t *testing.T) { testGetStaleCode(t, 2) } func TestGetStaleCodeLes3(t *testing.T) { testGetStaleCode(t, 3) } +func TestGetStaleCodeLes4(t *testing.T) { testGetStaleCode(t, 4) } func testGetStaleCode(t *testing.T, protocol int) { server, tearDown := newServerEnv(t, core.TriesInMemory+4, protocol, nil, false, true, 0) @@ -309,6 +313,7 @@ func testGetStaleCode(t *testing.T, protocol int) { // Tests that the transaction receipts can be retrieved based on hashes. func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func TestGetReceiptLes3(t *testing.T) { testGetReceipt(t, 3) } +func TestGetReceiptLes4(t *testing.T) { testGetReceipt(t, 4) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment @@ -336,6 +341,7 @@ func testGetReceipt(t *testing.T, protocol int) { // Tests that trie merkle proofs can be retrieved func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func TestGetProofsLes3(t *testing.T) { testGetProofs(t, 3) } +func TestGetProofsLes4(t *testing.T) { testGetProofs(t, 4) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment @@ -371,6 +377,7 @@ func testGetProofs(t *testing.T, protocol int) { // Tests that the stale contract codes can't be retrieved based on account addresses. func TestGetStaleProofLes2(t *testing.T) { testGetStaleProof(t, 2) } func TestGetStaleProofLes3(t *testing.T) { testGetStaleProof(t, 3) } +func TestGetStaleProofLes4(t *testing.T) { testGetStaleProof(t, 4) } func testGetStaleProof(t *testing.T, protocol int) { server, tearDown := newServerEnv(t, core.TriesInMemory+4, protocol, nil, false, true, 0) @@ -407,6 +414,7 @@ func testGetStaleProof(t *testing.T, protocol int) { // Tests that CHT proofs can be correctly retrieved. func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) } func TestGetCHTProofsLes3(t *testing.T) { testGetCHTProofs(t, 3) } +func TestGetCHTProofsLes4(t *testing.T) { testGetCHTProofs(t, 4) } func testGetCHTProofs(t *testing.T, protocol int) { config := light.TestServerIndexerConfig @@ -454,6 +462,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { func TestGetBloombitsProofsLes2(t *testing.T) { testGetBloombitsProofs(t, 2) } func TestGetBloombitsProofsLes3(t *testing.T) { testGetBloombitsProofs(t, 3) } +func TestGetBloombitsProofsLes4(t *testing.T) { testGetBloombitsProofs(t, 4) } // Tests that bloombits proofs can be correctly retrieved. func testGetBloombitsProofs(t *testing.T, protocol int) { @@ -503,6 +512,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { func TestTransactionStatusLes2(t *testing.T) { testTransactionStatus(t, 2) } func TestTransactionStatusLes3(t *testing.T) { testTransactionStatus(t, 3) } +func TestTransactionStatusLes4(t *testing.T) { testTransactionStatus(t, 4) } func testTransactionStatus(t *testing.T, protocol int) { server, tearDown := newServerEnv(t, 0, protocol, nil, false, true, 0) diff --git a/les/odr_test.go b/les/odr_test.go index c157507dd2..2a70a296c8 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -40,6 +40,7 @@ type odrTestFn func(ctx context.Context, db ethdb.Database, config *params.Chain func TestOdrGetBlockLes2(t *testing.T) { testOdr(t, 2, 1, true, odrGetBlock) } func TestOdrGetBlockLes3(t *testing.T) { testOdr(t, 3, 1, true, odrGetBlock) } +func TestOdrGetBlockLes4(t *testing.T) { testOdr(t, 4, 1, true, odrGetBlock) } func odrGetBlock(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { var block *types.Block @@ -57,6 +58,7 @@ func odrGetBlock(ctx context.Context, db ethdb.Database, config *params.ChainCon func TestOdrGetReceiptsLes2(t *testing.T) { testOdr(t, 2, 1, true, odrGetReceipts) } func TestOdrGetReceiptsLes3(t *testing.T) { testOdr(t, 3, 1, true, odrGetReceipts) } +func TestOdrGetReceiptsLes4(t *testing.T) { testOdr(t, 4, 1, true, odrGetReceipts) } func odrGetReceipts(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { var receipts types.Receipts @@ -78,6 +80,7 @@ func odrGetReceipts(ctx context.Context, db ethdb.Database, config *params.Chain func TestOdrAccountsLes2(t *testing.T) { testOdr(t, 2, 1, true, odrAccounts) } func TestOdrAccountsLes3(t *testing.T) { testOdr(t, 3, 1, true, odrAccounts) } +func TestOdrAccountsLes4(t *testing.T) { testOdr(t, 4, 1, true, odrAccounts) } func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { dummyAddr := common.HexToAddress("1234567812345678123456781234567812345678") @@ -107,6 +110,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon func TestOdrContractCallLes2(t *testing.T) { testOdr(t, 2, 2, true, odrContractCall) } func TestOdrContractCallLes3(t *testing.T) { testOdr(t, 3, 2, true, odrContractCall) } +func TestOdrContractCallLes4(t *testing.T) { testOdr(t, 4, 2, true, odrContractCall) } type callmsg struct { types.Message @@ -159,6 +163,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai func TestOdrTxStatusLes2(t *testing.T) { testOdr(t, 2, 1, false, odrTxStatus) } func TestOdrTxStatusLes3(t *testing.T) { testOdr(t, 3, 1, false, odrTxStatus) } +func TestOdrTxStatusLes4(t *testing.T) { testOdr(t, 4, 1, false, odrTxStatus) } func odrTxStatus(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { var txs types.Transactions diff --git a/les/protocol.go b/les/protocol.go index 39d9f5152f..9eb6ec7471 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -39,8 +39,8 @@ const ( // Supported versions of the les protocol (first is primary) var ( - ClientProtocolVersions = []uint{lpv2, lpv3} - ServerProtocolVersions = []uint{lpv2, lpv3} + ClientProtocolVersions = []uint{lpv2, lpv3, lpv4} + ServerProtocolVersions = []uint{lpv2, lpv3, lpv4} AdvertiseProtocolVersions = []uint{lpv2} // clients are searching for the first advertised protocol in the list ) diff --git a/les/request_test.go b/les/request_test.go index 4851274382..b054fd88df 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -38,6 +38,7 @@ type accessTestFn func(db ethdb.Database, bhash common.Hash, number uint64) ligh func TestBlockAccessLes2(t *testing.T) { testAccess(t, 2, tfBlockAccess) } func TestBlockAccessLes3(t *testing.T) { testAccess(t, 3, tfBlockAccess) } +func TestBlockAccessLes4(t *testing.T) { testAccess(t, 4, tfBlockAccess) } func tfBlockAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { return &light.BlockRequest{Hash: bhash, Number: number} @@ -45,6 +46,7 @@ func tfBlockAccess(db ethdb.Database, bhash common.Hash, number uint64) light.Od func TestReceiptsAccessLes2(t *testing.T) { testAccess(t, 2, tfReceiptsAccess) } func TestReceiptsAccessLes3(t *testing.T) { testAccess(t, 3, tfReceiptsAccess) } +func TestReceiptsAccessLes4(t *testing.T) { testAccess(t, 4, tfReceiptsAccess) } func tfReceiptsAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { return &light.ReceiptsRequest{Hash: bhash, Number: number} @@ -52,6 +54,7 @@ func tfReceiptsAccess(db ethdb.Database, bhash common.Hash, number uint64) light func TestTrieEntryAccessLes2(t *testing.T) { testAccess(t, 2, tfTrieEntryAccess) } func TestTrieEntryAccessLes3(t *testing.T) { testAccess(t, 3, tfTrieEntryAccess) } +func TestTrieEntryAccessLes4(t *testing.T) { testAccess(t, 4, tfTrieEntryAccess) } func tfTrieEntryAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { @@ -62,6 +65,7 @@ func tfTrieEntryAccess(db ethdb.Database, bhash common.Hash, number uint64) ligh func TestCodeAccessLes2(t *testing.T) { testAccess(t, 2, tfCodeAccess) } func TestCodeAccessLes3(t *testing.T) { testAccess(t, 3, tfCodeAccess) } +func TestCodeAccessLes4(t *testing.T) { testAccess(t, 4, tfCodeAccess) } func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrRequest { number := rawdb.ReadHeaderNumber(db, bhash) diff --git a/les/test_helper.go b/les/test_helper.go index e3f0616a88..1afc9be7d8 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/contracts/checkpointoracle/contract" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -344,7 +345,8 @@ func newTestPeer(t *testing.T, name string, version int, handler *serverHandler, head = handler.blockchain.CurrentHeader() td = handler.blockchain.GetTd(head.Hash(), head.Number.Uint64()) ) - tp.handshake(t, td, head.Hash(), head.Number.Uint64(), genesis.Hash(), testCostList(testCost)) + forkID := forkid.NewID(handler.blockchain.Config(), genesis.Hash(), head.Number.Uint64()) + tp.handshake(t, td, head.Hash(), head.Number.Uint64(), genesis.Hash(), forkID, testCostList(testCost)) } return tp, errCh } @@ -402,7 +404,7 @@ func newTestPeerPair(name string, version int, server *serverHandler, client *cl // handshake simulates a trivial handshake that expects the same state from the // remote side as we are simulating locally. -func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, costList RequestCostList) { +func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, costList RequestCostList) { var expList keyValueList expList = expList.add("protocolVersion", uint64(p.cpeer.version)) expList = expList.add("networkId", uint64(NetworkId)) @@ -410,6 +412,9 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu expList = expList.add("headHash", head) expList = expList.add("headNum", headNum) expList = expList.add("genesisHash", genesis) + if p.cpeer.version >= lpv4 { + expList = expList.add("forkID", &forkID) + } sendList := make(keyValueList, len(expList)) copy(sendList, expList) expList = expList.add("serveHeaders", nil) @@ -417,6 +422,9 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu expList = expList.add("serveStateSince", uint64(0)) expList = expList.add("serveRecentState", uint64(core.TriesInMemory-4)) expList = expList.add("txRelay", nil) + if p.cpeer.version >= lpv4 { + expList = expList.add("recentTxLookup", uint64(0)) + } expList = expList.add("flowControl/BL", testBufLimit) expList = expList.add("flowControl/MRR", testBufRecharge) expList = expList.add("flowControl/MRC", costList) From 08c878acd235fdc908b3a7a3c43dfc9fc5e9b2ef Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 15 Feb 2021 19:37:09 +0100 Subject: [PATCH 132/709] cmd/utils: add workaround for FreeBSD statfs quirk (#22310) Make geth build on FreeBSD, fixes #22309. --- cmd/utils/diskusage.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index a822118a39..da696de6bf 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -31,5 +31,12 @@ func getFreeDiskSpace(path string) (uint64, error) { } // Available blocks * size per block = available space in bytes - return stat.Bavail * uint64(stat.Bsize), nil + var bavail = stat.Bavail + if stat.Bavail < 0 { + // FreeBSD can have a negative number of blocks available + // because of the grace limit. + bavail = 0 + } + //nolint:unconvert + return uint64(bavail) * uint64(stat.Bsize), nil } From 77787802fe8f8415638480066ecace73037f1eed Mon Sep 17 00:00:00 2001 From: Alex Mazalov Date: Mon, 15 Feb 2021 18:47:47 +0000 Subject: [PATCH 133/709] cmd/geth: fix js unclean shutdown (#22302) --- cmd/geth/consolecmd.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 5369612e87..cc88b9eec4 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -19,10 +19,8 @@ package main import ( "fmt" "os" - "os/signal" "path/filepath" "strings" - "syscall" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" @@ -218,13 +216,10 @@ func ephemeralConsole(ctx *cli.Context) error { utils.Fatalf("Failed to execute %s: %v", file, err) } } - // Wait for pending callbacks, but stop for Ctrl-C. - abort := make(chan os.Signal, 1) - signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM) go func() { - <-abort - os.Exit(0) + stack.Wait() + console.Stop(false) }() console.Stop(true) From e991bdae2458dbee5a28addd188a897858aa34dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 16 Feb 2021 10:44:38 +0200 Subject: [PATCH 134/709] trie: fix bloom crash on fast sync restart --- trie/sync.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/trie/sync.go b/trie/sync.go index 05b9f55d33..dd8279b665 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -313,11 +313,15 @@ func (s *Sync) Commit(dbw ethdb.Batch) error { // Dump the membatch into a database dbw for key, value := range s.membatch.nodes { rawdb.WriteTrieNode(dbw, key, value) - s.bloom.Add(key[:]) + if s.bloom != nil { + s.bloom.Add(key[:]) + } } for key, value := range s.membatch.codes { rawdb.WriteCode(dbw, key, value) - s.bloom.Add(key[:]) + if s.bloom != nil { + s.bloom.Add(key[:]) + } } // Drop the membatch data and return s.membatch = newSyncMemBatch() From f4fcd4f506661c7cece755b90b8a84e51d5925ac Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 16 Feb 2021 10:40:59 +0100 Subject: [PATCH 135/709] rpc: increase the number of subscriptions in storm test (#22316) --- rpc/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/client_test.go b/rpc/client_test.go index 5b1f960352..5d301a07a7 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -427,7 +427,7 @@ func TestClientNotificationStorm(t *testing.T) { } doTest(8000, false) - doTest(23000, true) + doTest(24000, true) } func TestClientSetHeader(t *testing.T) { From 9ec3329899a0ff62ed2f83c61b50140881a577a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 16 Feb 2021 09:04:07 +0200 Subject: [PATCH 136/709] core/state/snapshot: ensure Cap retains a min number of layers --- core/state/snapshot/snapshot.go | 47 ++++-------- core/state/snapshot/snapshot_test.go | 111 ++++++++++++--------------- 2 files changed, 65 insertions(+), 93 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index df2b1ed330..aa5f5900b0 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -300,6 +300,12 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m // Cap traverses downwards the snapshot tree from a head block hash until the // number of allowed layers are crossed. All layers beyond the permitted number // are flattened downwards. +// +// Note, the final diff layer count in general will be one more than the amount +// requested. This happens because the bottom-most diff layer is the accumulator +// which may or may not overflow and cascade to disk. Since this last layer's +// survival is only known *after* capping, we need to omit it from the count if +// we want to ensure that *at least* the requested number of diff layers remain. func (t *Tree) Cap(root common.Hash, layers int) error { // Retrieve the head snapshot to cap from snap := t.Snapshot(root) @@ -324,10 +330,7 @@ func (t *Tree) Cap(root common.Hash, layers int) error { // Flattening the bottom-most diff layer requires special casing since there's // no child to rewire to the grandparent. In that case we can fake a temporary // child for the capping and then remove it. - var persisted *diskLayer - - switch layers { - case 0: + if layers == 0 { // If full commit was requested, flatten the diffs and merge onto disk diff.lock.RLock() base := diffToDisk(diff.flatten().(*diffLayer)) @@ -336,33 +339,9 @@ func (t *Tree) Cap(root common.Hash, layers int) error { // Replace the entire snapshot tree with the flat base t.layers = map[common.Hash]snapshot{base.root: base} return nil - - case 1: - // If full flattening was requested, flatten the diffs but only merge if the - // memory limit was reached - var ( - bottom *diffLayer - base *diskLayer - ) - diff.lock.RLock() - bottom = diff.flatten().(*diffLayer) - if bottom.memory >= aggregatorMemoryLimit { - base = diffToDisk(bottom) - } - diff.lock.RUnlock() - - // If all diff layers were removed, replace the entire snapshot tree - if base != nil { - t.layers = map[common.Hash]snapshot{base.root: base} - return nil - } - // Merge the new aggregated layer into the snapshot tree, clean stales below - t.layers[bottom.root] = bottom - - default: - // Many layers requested to be retained, cap normally - persisted = t.cap(diff, layers) } + persisted := t.cap(diff, layers) + // Remove any layer that is stale or links into a stale layer children := make(map[common.Hash][]common.Hash) for root, snap := range t.layers { @@ -405,9 +384,15 @@ func (t *Tree) Cap(root common.Hash, layers int) error { // layer limit is reached, memory cap is also enforced (but not before). // // The method returns the new disk layer if diffs were persisted into it. +// +// Note, the final diff layer count in general will be one more than the amount +// requested. This happens because the bottom-most diff layer is the accumulator +// which may or may not overflow and cascade to disk. Since this last layer's +// survival is only known *after* capping, we need to omit it from the count if +// we want to ensure that *at least* the requested number of diff layers remain. func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer { // Dive until we run out of layers or reach the persistent database - for ; layers > 2; layers-- { + for i := 0; i < layers-1; i++ { // If we still have diff layers below, continue down if parent, ok := diff.parent.(*diffLayer); ok { diff = parent diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index a315fd216c..4b787cfe2e 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -162,57 +162,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) aggregatorMemoryLimit = 0 - if err := snaps.Cap(common.HexToHash("0x03"), 2); err != nil { - t.Fatalf("failed to merge diff layer onto disk: %v", err) - } - // Since the base layer was modified, ensure that data retrievald on the external reference fail - if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { - t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) - } - if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { - t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) - } - if n := len(snaps.layers); n != 2 { - t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2) - fmt.Println(snaps.layers) - } -} - -// Tests that if a diff layer becomes stale, no active external references will -// be returned with junk data. This version of the test flattens every diff layer -// to check internal corner case around the bottom-most memory accumulator. -func TestDiffLayerExternalInvalidationFullFlatten(t *testing.T) { - // Create an empty base layer and a snapshot tree out of it - base := &diskLayer{ - diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), - cache: fastcache.New(1024 * 500), - } - snaps := &Tree{ - layers: map[common.Hash]snapshot{ - base.root: base, - }, - } - // Commit two diffs on top and retrieve a reference to the bottommost - accounts := map[common.Hash][]byte{ - common.HexToHash("0xa1"): randomAccount(), - } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { - t.Fatalf("failed to create a diff layer: %v", err) - } - if n := len(snaps.layers); n != 3 { - t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3) - } - ref := snaps.Snapshot(common.HexToHash("0x02")) - - // Flatten the diff layer into the bottom accumulator if err := snaps.Cap(common.HexToHash("0x03"), 1); err != nil { - t.Fatalf("failed to flatten diff layer into accumulator: %v", err) + t.Fatalf("failed to merge accumulator onto disk: %v", err) } - // Since the accumulator diff layer was modified, ensure that data retrievald on the external reference fail + // Since the base layer was modified, ensure that data retrievald on the external reference fail if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) } @@ -267,7 +220,7 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { t.Errorf("layers modified, got %d exp %d", got, exp) } // Flatten the diff layer into the bottom accumulator - if err := snaps.Cap(common.HexToHash("0x04"), 2); err != nil { + if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil { t.Fatalf("failed to flatten diff layer into accumulator: %v", err) } // Since the accumulator diff layer was modified, ensure that data retrievald on the external reference fail @@ -389,7 +342,7 @@ func TestSnaphots(t *testing.T) { // Create a starting base layer and a snapshot tree out of it base := &diskLayer{ diskdb: rawdb.NewMemoryDatabase(), - root: common.HexToHash("0x01"), + root: makeRoot(1), cache: fastcache.New(1024 * 500), } snaps := &Tree{ @@ -397,17 +350,16 @@ func TestSnaphots(t *testing.T) { base.root: base, }, } - // Construct the snapshots with 128 layers + // Construct the snapshots with 129 layers, flattening whatever's above that var ( last = common.HexToHash("0x01") head common.Hash ) - // Flush another 128 layers, one diff will be flatten into the parent. - for i := 0; i < 128; i++ { + for i := 0; i < 129; i++ { head = makeRoot(uint64(i + 2)) snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) last = head - snaps.Cap(head, 128) // 129 layers(128 diffs + 1 disk) are allowed, 129th is the persistent layer + snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) } var cases = []struct { headRoot common.Hash @@ -417,22 +369,57 @@ func TestSnaphots(t *testing.T) { expectBottom common.Hash }{ {head, 0, false, 0, common.Hash{}}, - {head, 64, false, 64, makeRoot(127 + 2 - 63)}, - {head, 128, false, 128, makeRoot(2)}, // All diff layers - {head, 129, true, 128, makeRoot(2)}, // All diff layers - {head, 129, false, 129, common.HexToHash("0x01")}, // All diff layers + disk layer + {head, 64, false, 64, makeRoot(129 + 2 - 64)}, + {head, 128, false, 128, makeRoot(3)}, // Normal diff layers, no accumulator + {head, 129, true, 129, makeRoot(2)}, // All diff layers, including accumulator + {head, 130, false, 130, makeRoot(1)}, // All diff layers + disk layer + } + for i, c := range cases { + layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) + if len(layers) != c.expected { + t.Errorf("non-overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers)) + } + if len(layers) == 0 { + continue + } + bottommost := layers[len(layers)-1] + if bottommost.Root() != c.expectBottom { + t.Errorf("non-overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root()) + } + } + // Above we've tested the normal capping, which leaves the accumulator live. + // Test that if the bottommost accumulator diff layer overflows the allowed + // memory limit, the snapshot tree gets capped to one less layer. + // Commit the diff layer onto the disk and ensure it's persisted + defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) + aggregatorMemoryLimit = 0 + + snaps.Cap(head, 128) // 129 (128 diffs + 1 overflown accumulator + 1 disk) + + cases = []struct { + headRoot common.Hash + limit int + nodisk bool + expected int + expectBottom common.Hash + }{ + {head, 0, false, 0, common.Hash{}}, + {head, 64, false, 64, makeRoot(129 + 2 - 64)}, + {head, 128, false, 128, makeRoot(3)}, // All diff layers, accumulator was flattened + {head, 129, true, 128, makeRoot(3)}, // All diff layers, accumulator was flattened + {head, 130, false, 129, makeRoot(2)}, // All diff layers + disk layer } - for _, c := range cases { + for i, c := range cases { layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) if len(layers) != c.expected { - t.Fatalf("Returned snapshot layers are mismatched, want %v, got %v", c.expected, len(layers)) + t.Errorf("overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers)) } if len(layers) == 0 { continue } bottommost := layers[len(layers)-1] if bottommost.Root() != c.expectBottom { - t.Fatalf("Snapshot mismatch, want %v, get %v", c.expectBottom, bottommost.Root()) + t.Errorf("overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root()) } } } From bfdff4c5b83cc09b2f91377f87e7757ddbe7fd63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 16 Feb 2021 16:11:33 +0200 Subject: [PATCH 137/709] eth: fix snap sync cancellation --- eth/downloader/downloader.go | 3 +-- eth/protocols/snap/protocol.go | 1 - eth/protocols/snap/sync.go | 6 +++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 421803876e..416a387e31 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -346,7 +346,6 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode case nil, errBusy, errCanceled: return err } - if errors.Is(err, errInvalidChain) || errors.Is(err, errBadPeer) || errors.Is(err, errTimeout) || errors.Is(err, errStallingPeer) || errors.Is(err, errUnsyncedPeer) || errors.Is(err, errEmptyHeaderSet) || errors.Is(err, errPeersUnavailable) || errors.Is(err, errTooOld) || errors.Is(err, errInvalidAncestor) { @@ -1764,7 +1763,7 @@ func (d *Downloader) processFastSyncContent() error { }() closeOnErr := func(s *stateSync) { - if err := s.Wait(); err != nil && err != errCancelStateFetch && err != errCanceled { + if err := s.Wait(); err != nil && err != errCancelStateFetch && err != errCanceled && err != snap.ErrCancelled { d.queue.Close() // wake up Results } } diff --git a/eth/protocols/snap/protocol.go b/eth/protocols/snap/protocol.go index a74142cafb..5528e9212e 100644 --- a/eth/protocols/snap/protocol.go +++ b/eth/protocols/snap/protocol.go @@ -61,7 +61,6 @@ var ( errDecode = errors.New("invalid message") errInvalidMsgCode = errors.New("invalid message code") errBadRequest = errors.New("bad request") - errCancelled = errors.New("sync cancelled") ) // Packet represents a p2p message in the `snap` protocol. diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 422cdf8f72..c31e4a5dae 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -88,6 +88,10 @@ var ( requestTimeout = 10 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? ) +// ErrCancelled is returned from snap syncing if the operation was prematurely +// terminated. +var ErrCancelled = errors.New("sync cancelled") + // accountRequest tracks a pending account range request to ensure responses are // to actual requests and to validate any security constraints. // @@ -615,7 +619,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { case id := <-peerDrop: s.revertRequests(id) case <-cancel: - return errCancelled + return ErrCancelled case req := <-s.accountReqFails: s.revertAccountRequest(req) From f9445e93bb72aedec953e65734ec18b4e1eaac3d Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 16 Feb 2021 15:23:03 +0100 Subject: [PATCH 138/709] cmd/devp2p/internal/ethtest: use shared message types (#22315) This updates the eth protocol test suite to use the message type definitions of the 'production' protocol implementation in eth/protocols/eth. --- cmd/devp2p/internal/ethtest/chain_test.go | 7 +- cmd/devp2p/internal/ethtest/suite.go | 19 +++-- cmd/devp2p/internal/ethtest/transaction.go | 4 +- cmd/devp2p/internal/ethtest/types.go | 90 +++++----------------- 4 files changed, 36 insertions(+), 84 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain_test.go b/cmd/devp2p/internal/ethtest/chain_test.go index ac3907ce8d..5e4289d80a 100644 --- a/cmd/devp2p/internal/ethtest/chain_test.go +++ b/cmd/devp2p/internal/ethtest/chain_test.go @@ -21,6 +21,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/p2p" "github.com/stretchr/testify/assert" ) @@ -93,7 +94,7 @@ func TestChain_GetHeaders(t *testing.T) { }{ { req: GetBlockHeaders{ - Origin: hashOrNumber{ + Origin: eth.HashOrNumber{ Number: uint64(2), }, Amount: uint64(5), @@ -110,7 +111,7 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: hashOrNumber{ + Origin: eth.HashOrNumber{ Number: uint64(chain.Len() - 1), }, Amount: uint64(3), @@ -125,7 +126,7 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: hashOrNumber{ + Origin: eth.HashOrNumber{ Hash: chain.Head().Hash(), }, Amount: uint64(1), diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 5d0cdda720..edf7bb7e31 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -24,6 +24,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -143,7 +144,7 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { // get block headers req := &GetBlockHeaders{ - Origin: hashOrNumber{ + Origin: eth.HashOrNumber{ Hash: s.chain.blocks[1].Hash(), }, Amount: 2, @@ -157,8 +158,8 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { switch msg := conn.ReadAndServe(s.chain, timeout).(type) { case *BlockHeaders: - headers := msg - for _, header := range *headers { + headers := *msg + for _, header := range headers { num := header.Number.Uint64() t.Logf("received header (%d): %s", num, pretty.Sdump(header)) assert.Equal(t, s.chain.blocks[int(num)].Header(), header) @@ -179,7 +180,10 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { conn.handshake(t) conn.statusExchange(t, s.chain, nil) // create block bodies request - req := &GetBlockBodies{s.chain.blocks[54].Hash(), s.chain.blocks[75].Hash()} + req := &GetBlockBodies{ + s.chain.blocks[54].Hash(), + s.chain.blocks[75].Hash(), + } if err := conn.Write(req); err != nil { t.Fatalf("could not write to connection: %v", err) } @@ -357,10 +361,9 @@ func (s *Suite) waitAnnounce(t *utesting.T, conn *Conn, blockAnnouncement *NewBl "wrong TD in announcement", ) case *NewBlockHashes: - hashes := *msg - t.Logf("received NewBlockHashes message: %s", pretty.Sdump(hashes)) - assert.Equal(t, - blockAnnouncement.Block.Hash(), hashes[0].Hash, + message := *msg + t.Logf("received NewBlockHashes message: %s", pretty.Sdump(message)) + assert.Equal(t, blockAnnouncement.Block.Hash(), message[0].Hash, "wrong block hash in announcement", ) default: diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 4aaab8bf97..effcc3af29 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -32,7 +32,7 @@ func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) { sendConn := s.setupConnection(t) t.Logf("sending tx: %v %v %v\n", tx.Hash().String(), tx.GasPrice(), tx.Gas()) // Send the transaction - if err := sendConn.Write(Transactions([]*types.Transaction{tx})); err != nil { + if err := sendConn.Write(&Transactions{tx}); err != nil { t.Fatal(err) } time.Sleep(100 * time.Millisecond) @@ -70,7 +70,7 @@ func sendFailingTx(t *utesting.T, s *Suite, tx *types.Transaction) { t.Logf("unexpected message, logging: %v", pretty.Sdump(msg)) } // Send the transaction - if err := sendConn.Write(Transactions([]*types.Transaction{tx})); err != nil { + if err := sendConn.Write(&Transactions{tx}); err != nil { t.Fatal(err) } // Wait for another transaction announcement diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index f768d61d5a..b901f50700 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -19,15 +19,12 @@ package ethtest import ( "crypto/ecdsa" "fmt" - "io" - "math/big" "reflect" "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" @@ -81,102 +78,48 @@ type Pong struct{} func (p Pong) Code() int { return 0x03 } // Status is the network packet for the status message for eth/64 and later. -type Status struct { - ProtocolVersion uint32 - NetworkID uint64 - TD *big.Int - Head common.Hash - Genesis common.Hash - ForkID forkid.ID -} +type Status eth.StatusPacket func (s Status) Code() int { return 16 } // NewBlockHashes is the network packet for the block announcements. -type NewBlockHashes []struct { - Hash common.Hash // Hash of one particular block being announced - Number uint64 // Number of one particular block being announced -} +type NewBlockHashes eth.NewBlockHashesPacket func (nbh NewBlockHashes) Code() int { return 17 } -type Transactions []*types.Transaction +type Transactions eth.TransactionsPacket func (t Transactions) Code() int { return 18 } // GetBlockHeaders represents a block header query. -type GetBlockHeaders struct { - Origin hashOrNumber // Block from which to retrieve headers - Amount uint64 // Maximum number of headers to retrieve - Skip uint64 // Blocks to skip between consecutive headers - Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) -} +type GetBlockHeaders eth.GetBlockHeadersPacket func (g GetBlockHeaders) Code() int { return 19 } -type BlockHeaders []*types.Header +type BlockHeaders eth.BlockHeadersPacket func (bh BlockHeaders) Code() int { return 20 } // GetBlockBodies represents a GetBlockBodies request -type GetBlockBodies []common.Hash +type GetBlockBodies eth.GetBlockBodiesPacket func (gbb GetBlockBodies) Code() int { return 21 } // BlockBodies is the network packet for block content distribution. -type BlockBodies []*types.Body +type BlockBodies eth.BlockBodiesPacket func (bb BlockBodies) Code() int { return 22 } // NewBlock is the network packet for the block propagation message. -type NewBlock struct { - Block *types.Block - TD *big.Int -} +type NewBlock eth.NewBlockPacket func (nb NewBlock) Code() int { return 23 } // NewPooledTransactionHashes is the network packet for the tx hash propagation message. -type NewPooledTransactionHashes [][32]byte +type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket func (nb NewPooledTransactionHashes) Code() int { return 24 } -// HashOrNumber is a combined field for specifying an origin block. -type hashOrNumber struct { - Hash common.Hash // Block hash from which to retrieve headers (excludes Number) - Number uint64 // Block hash from which to retrieve headers (excludes Hash) -} - -// EncodeRLP is a specialized encoder for hashOrNumber to encode only one of the -// two contained union fields. -func (hn *hashOrNumber) EncodeRLP(w io.Writer) error { - if hn.Hash == (common.Hash{}) { - return rlp.Encode(w, hn.Number) - } - if hn.Number != 0 { - return fmt.Errorf("both origin hash (%x) and number (%d) provided", hn.Hash, hn.Number) - } - return rlp.Encode(w, hn.Hash) -} - -// DecodeRLP is a specialized decoder for hashOrNumber to decode the contents -// into either a block hash or a block number. -func (hn *hashOrNumber) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - origin, err := s.Raw() - if err == nil { - switch { - case size == 32: - err = rlp.DecodeBytes(origin, &hn.Hash) - case size <= 8: - err = rlp.DecodeBytes(origin, &hn.Number) - default: - err = fmt.Errorf("invalid input size %d for origin", size) - } - } - return err -} - // Conn represents an individual connection with a peer type Conn struct { *rlpx.Conn @@ -221,7 +164,7 @@ func (c *Conn) Read() Message { default: return errorf("invalid message code: %d", code) } - + // if message is devp2p, decode here if err := rlp.DecodeBytes(rawData, msg); err != nil { return errorf("could not rlp decode message: %v", err) } @@ -256,7 +199,12 @@ func (c *Conn) ReadAndServe(chain *Chain, timeout time.Duration) Message { } func (c *Conn) Write(msg Message) error { - payload, err := rlp.EncodeToBytes(msg) + // check if message is eth protocol message + var ( + payload []byte + err error + ) + payload, err = rlp.EncodeToBytes(msg) if err != nil { return err } @@ -363,7 +311,7 @@ loop: } } - if err := c.Write(*status); err != nil { + if err := c.Write(status); err != nil { t.Fatalf("could not write to connection: %v", err) } @@ -378,7 +326,7 @@ func (c *Conn) waitForBlock(block *types.Block) error { timeout := time.Now().Add(20 * time.Second) c.SetReadDeadline(timeout) for { - req := &GetBlockHeaders{Origin: hashOrNumber{Hash: block.Hash()}, Amount: 1} + req := &GetBlockHeaders{Origin: eth.HashOrNumber{Hash: block.Hash()}, Amount: 1} if err := c.Write(req); err != nil { return err } From e01096f531862b982833732514376cead8d58e82 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 17 Feb 2021 14:59:00 +0100 Subject: [PATCH 139/709] eth/handler, broadcast: optimize tx broadcast mechanism (#22176) This PR optimizes the broadcast loop. Instead of iterating twice through a given set of transactions to weed out which peers have and which do not have a tx, to send/announce transactions, we do it only once. --- eth/handler.go | 56 ++++++++++++++++++---------------- eth/protocols/eth/broadcast.go | 12 ++++---- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index a5a62b894d..13fa701935 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -456,44 +456,51 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { } } -// BroadcastTransactions will propagate a batch of transactions to all peers which are not known to +// BroadcastTransactions will propagate a batch of transactions +// - To a square root of all peers +// - And, separately, as announcements to all peers which are not known to // already have the given transaction. -func (h *handler) BroadcastTransactions(txs types.Transactions, propagate bool) { +func (h *handler) BroadcastTransactions(txs types.Transactions) { var ( - txset = make(map[*ethPeer][]common.Hash) - annos = make(map[*ethPeer][]common.Hash) + annoCount int // Count of announcements made + annoPeers int + directCount int // Count of the txs sent directly to peers + directPeers int // Count of the peers that were sent transactions directly + + txset = make(map[*ethPeer][]common.Hash) // Set peer->hash to transfer directly + annos = make(map[*ethPeer][]common.Hash) // Set peer->hash to announce + ) // Broadcast transactions to a batch of peers not knowing about it - if propagate { - for _, tx := range txs { - peers := h.peers.peersWithoutTransaction(tx.Hash()) - - // Send the block to a subset of our peers - transfer := peers[:int(math.Sqrt(float64(len(peers))))] - for _, peer := range transfer { - txset[peer] = append(txset[peer], tx.Hash()) - } - log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(transfer)) - } - for peer, hashes := range txset { - peer.AsyncSendTransactions(hashes) - } - return - } - // Otherwise only broadcast the announcement to peers for _, tx := range txs { peers := h.peers.peersWithoutTransaction(tx.Hash()) - for _, peer := range peers { + // Send the tx unconditionally to a subset of our peers + numDirect := int(math.Sqrt(float64(len(peers)))) + for _, peer := range peers[:numDirect] { + txset[peer] = append(txset[peer], tx.Hash()) + } + // For the remaining peers, send announcement only + for _, peer := range peers[numDirect:] { annos[peer] = append(annos[peer], tx.Hash()) } } + for peer, hashes := range txset { + directPeers++ + directCount += len(hashes) + peer.AsyncSendTransactions(hashes) + } for peer, hashes := range annos { + annoPeers++ + annoCount += len(hashes) if peer.Version() >= eth.ETH65 { peer.AsyncSendPooledTransactionHashes(hashes) } else { peer.AsyncSendTransactions(hashes) } } + log.Debug("Transaction broadcast", "txs", len(txs), + "announce packs", annoPeers, "announced hashes", annoCount, + "tx packs", directPeers, "broadcast txs", directCount) } // minedBroadcastLoop sends mined blocks to connected peers. @@ -511,13 +518,10 @@ func (h *handler) minedBroadcastLoop() { // txBroadcastLoop announces new transactions to connected peers. func (h *handler) txBroadcastLoop() { defer h.wg.Done() - for { select { case event := <-h.txsCh: - h.BroadcastTransactions(event.Txs, true) // First propagate transactions to peers - h.BroadcastTransactions(event.Txs, false) // Only then announce to the rest - + h.BroadcastTransactions(event.Txs) case <-h.txsSub.Err(): return } diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 74ec2f0654..328396d510 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -142,18 +142,18 @@ func (p *Peer) announceTransactions() { if done == nil && len(queue) > 0 { // Pile transaction hashes until we reach our allowed network limit var ( - hashes []common.Hash + count int pending []common.Hash size common.StorageSize ) - for i := 0; i < len(queue) && size < maxTxPacketSize; i++ { - if p.txpool.Get(queue[i]) != nil { - pending = append(pending, queue[i]) + for count = 0; count < len(queue) && size < maxTxPacketSize; count++ { + if p.txpool.Get(queue[count]) != nil { + pending = append(pending, queue[count]) size += common.HashLength } - hashes = append(hashes, queue[i]) } - queue = queue[:copy(queue, queue[len(hashes):])] + // Shift and trim queue + queue = queue[:copy(queue, queue[count:])] // If there's anything available to transfer, fire up an async writer if len(pending) > 0 { From 52e5c38aa5dcc01566bb6d05a5312b5b642899b4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 18 Feb 2021 09:05:47 +0100 Subject: [PATCH 140/709] core/state: copy the snap when copying the state (#22340) * core/state: copy the snap when copying the state * core/state: deep-copy snap stuff during state Copy --- core/state/statedb.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/state/statedb.go b/core/state/statedb.go index 49f457a99f..8fd066e24f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -728,6 +728,31 @@ func (s *StateDB) Copy() *StateDB { if s.prefetcher != nil { state.prefetcher = s.prefetcher.copy() } + if s.snaps != nil { + // In order for the miner to be able to use and make additions + // to the snapshot tree, we need to copy that aswell. + // Otherwise, any block mined by ourselves will cause gaps in the tree, + // and force the miner to operate trie-backed only + state.snaps = s.snaps + state.snap = s.snap + // deep copy needed + state.snapDestructs = make(map[common.Hash]struct{}) + for k, v := range s.snapDestructs { + state.snapDestructs[k] = v + } + state.snapAccounts = make(map[common.Hash][]byte) + for k, v := range s.snapAccounts { + state.snapAccounts[k] = v + } + state.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + for k, v := range s.snapStorage { + temp := make(map[common.Hash][]byte) + for kk, vv := range v { + temp[kk] = vv + } + state.snapStorage[k] = temp + } + } return state } From 9ec32a9e7b2a39103c905d57e270d99463e6aa99 Mon Sep 17 00:00:00 2001 From: Or Neeman Date: Thu, 18 Feb 2021 03:19:49 -0600 Subject: [PATCH 141/709] rlp: handle case of normal EOF in Stream.readFull (#22336) io.Reader may return n > 0 and io.EOF at the end of the input stream. readFull did not handle this correctly, looking only at the error. This fixes it to check for n == len(buf) as well. --- rlp/decode.go | 8 +++++++- rlp/decode_test.go | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/rlp/decode.go b/rlp/decode.go index 5f3f5eedfd..79b7ef0626 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -952,7 +952,13 @@ func (s *Stream) readFull(buf []byte) (err error) { n += nn } if err == io.EOF { - err = io.ErrUnexpectedEOF + if n < len(buf) { + err = io.ErrUnexpectedEOF + } else { + // Readers are allowed to give EOF even though the read succeeded. + // In such cases, we discard the EOF, like io.ReadFull() does. + err = nil + } } return err } diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 167e9974b9..d94c3969b2 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -665,6 +665,26 @@ func TestDecodeWithByteReader(t *testing.T) { }) } +func testDecodeWithEncReader(t *testing.T, n int) { + s := strings.Repeat("0", n) + _, r, _ := EncodeToReader(s) + var decoded string + err := Decode(r, &decoded) + if err != nil { + t.Errorf("Unexpected decode error with n=%v: %v", n, err) + } + if decoded != s { + t.Errorf("Decode mismatch with n=%v", n) + } +} + +// This is a regression test checking that decoding from encReader +// works for RLP values of size 8192 bytes or more. +func TestDecodeWithEncReader(t *testing.T) { + testDecodeWithEncReader(t, 8188) // length with header is 8191 + testDecodeWithEncReader(t, 8189) // length with header is 8192 +} + // plainReader reads from a byte slice but does not // implement ReadByte. It is also not recognized by the // size validation. This is useful to test how the decoder From b1835b3855ebee0aa8c63d18b8f0671168ceced5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 18 Feb 2021 10:40:19 +0100 Subject: [PATCH 142/709] node: always show websocket url in logs (#22307) --- node/rpcstack.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/node/rpcstack.go b/node/rpcstack.go index d693bb0bbd..56e23cc5c7 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -144,13 +144,15 @@ func (h *httpServer) start() error { h.listener = listener go h.server.Serve(listener) - // if server is websocket only, return after logging - if h.wsAllowed() && !h.rpcAllowed() { + if h.wsAllowed() { url := fmt.Sprintf("ws://%v", listener.Addr()) if h.wsConfig.prefix != "" { url += h.wsConfig.prefix } h.log.Info("WebSocket enabled", "url", url) + } + // if server is websocket only, return after logging + if !h.rpcAllowed() { return nil } // Log http endpoint. From 6ec15610443b28eabf665199f1dc5be2b3e3f7cb Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 18 Feb 2021 17:54:29 +0100 Subject: [PATCH 143/709] eth: implement eth66 (#22241) * eth/protocols/eth: split up the eth protocol handlers * eth/protocols/eth: define eth-66 protocol messages * eth/protocols/eth: poc implement getblockheaders on eth/66 * eth/protocols/eth: implement remaining eth-66 handlers * eth/protocols: define handler map for eth 66 * eth/downloader: use protocol constants from eth package * eth/protocols/eth: add ETH66 capability * eth/downloader: tests for eth66 * eth/downloader: fix error in tests * eth/protocols/eth: use eth66 for outgoing requests * eth/protocols/eth: remove unused error type * eth/protocols/eth: define protocol length * eth/protocols/eth: fix pooled tx over eth66 * protocols/eth/handlers: revert behavioural change which caused tests to fail * eth/downloader: fix failing test * eth/protocols/eth: add testcases + fix flaw with header requests * eth/protocols: change comments * eth/protocols/eth: review fixes + fixed flaw in RequestOneHeader * eth/protocols: documentation * eth/protocols/eth: review concerns about types --- eth/downloader/downloader_test.go | 185 ++++++++--- eth/downloader/peer.go | 9 +- eth/protocols/eth/handler.go | 399 ++++------------------ eth/protocols/eth/handlers.go | 510 +++++++++++++++++++++++++++++ eth/protocols/eth/peer.go | 113 ++++++- eth/protocols/eth/protocol.go | 95 +++++- eth/protocols/eth/protocol_test.go | 200 +++++++++++ p2p/message.go | 4 + 8 files changed, 1120 insertions(+), 395 deletions(-) create mode 100644 eth/protocols/eth/handlers.go diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 5de1ef3f8e..2917116144 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -515,18 +515,18 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } -// Tests that simple synchronization against a canonical chain works correctly. -// In this test common ancestor lookup should be short circuited and not require -// binary searching. -func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonicalSynchronisation(t, 64, FullSync) } -func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonicalSynchronisation(t, 64, FastSync) } -func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonicalSynchronisation(t, 65, FullSync) } -func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonicalSynchronisation(t, 65, FastSync) } -func TestCanonicalSynchronisation65Light(t *testing.T) { - testCanonicalSynchronisation(t, 65, LightSync) -} +func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonSync(t, 64, FullSync) } +func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonSync(t, 64, FastSync) } + +func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, 65, FullSync) } +func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, 65, FastSync) } +func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, 65, LightSync) } + +func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, 66, FullSync) } +func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, 66, FastSync) } +func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, 66, LightSync) } -func testCanonicalSynchronisation(t *testing.T, protocol uint, mode SyncMode) { +func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -547,9 +547,13 @@ func testCanonicalSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling64Full(t *testing.T) { testThrottling(t, 64, FullSync) } func TestThrottling64Fast(t *testing.T) { testThrottling(t, 64, FastSync) } + func TestThrottling65Full(t *testing.T) { testThrottling(t, 65, FullSync) } func TestThrottling65Fast(t *testing.T) { testThrottling(t, 65, FastSync) } +func TestThrottling66Full(t *testing.T) { testThrottling(t, 66, FullSync) } +func TestThrottling66Fast(t *testing.T) { testThrottling(t, 66, FastSync) } + func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -629,12 +633,17 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync64Full(t *testing.T) { testForkedSync(t, 64, FullSync) } -func TestForkedSync64Fast(t *testing.T) { testForkedSync(t, 64, FastSync) } +func TestForkedSync64Full(t *testing.T) { testForkedSync(t, 64, FullSync) } +func TestForkedSync64Fast(t *testing.T) { testForkedSync(t, 64, FastSync) } + func TestForkedSync65Full(t *testing.T) { testForkedSync(t, 65, FullSync) } func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, 65, FastSync) } func TestForkedSync65Light(t *testing.T) { testForkedSync(t, 65, LightSync) } +func TestForkedSync66Full(t *testing.T) { testForkedSync(t, 66, FullSync) } +func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, 66, FastSync) } +func TestForkedSync66Light(t *testing.T) { testForkedSync(t, 66, LightSync) } + func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -660,12 +669,17 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync64Full(t *testing.T) { testHeavyForkedSync(t, 64, FullSync) } -func TestHeavyForkedSync64Fast(t *testing.T) { testHeavyForkedSync(t, 64, FastSync) } +func TestHeavyForkedSync64Full(t *testing.T) { testHeavyForkedSync(t, 64, FullSync) } +func TestHeavyForkedSync64Fast(t *testing.T) { testHeavyForkedSync(t, 64, FastSync) } + func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, 65, FullSync) } func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, 65, FastSync) } func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, 65, LightSync) } +func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, 66, FullSync) } +func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, 66, FastSync) } +func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, 66, LightSync) } + func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -693,12 +707,17 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync64Full(t *testing.T) { testBoundedForkedSync(t, 64, FullSync) } -func TestBoundedForkedSync64Fast(t *testing.T) { testBoundedForkedSync(t, 64, FastSync) } +func TestBoundedForkedSync64Full(t *testing.T) { testBoundedForkedSync(t, 64, FullSync) } +func TestBoundedForkedSync64Fast(t *testing.T) { testBoundedForkedSync(t, 64, FastSync) } + func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, 65, FullSync) } func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, 65, FastSync) } func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, 65, LightSync) } +func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, 66, FullSync) } +func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, 66, FastSync) } +func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, 66, LightSync) } + func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -725,12 +744,17 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync64Full(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FullSync) } -func TestBoundedHeavyForkedSync64Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FastSync) } +func TestBoundedHeavyForkedSync64Full(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FullSync) } +func TestBoundedHeavyForkedSync64Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FastSync) } + func TestBoundedHeavyForkedSync65Full(t *testing.T) { testBoundedHeavyForkedSync(t, 65, FullSync) } func TestBoundedHeavyForkedSync65Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 65, FastSync) } func TestBoundedHeavyForkedSync65Light(t *testing.T) { testBoundedHeavyForkedSync(t, 65, LightSync) } +func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, 66, FullSync) } +func TestBoundedHeavyForkedSync66Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 66, FastSync) } +func TestBoundedHeavyForkedSync66Light(t *testing.T) { testBoundedHeavyForkedSync(t, 66, LightSync) } + func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() tester := newTester() @@ -775,12 +799,17 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) } -func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) } +func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) } +func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) } + func TestCancel65Full(t *testing.T) { testCancel(t, 65, FullSync) } func TestCancel65Fast(t *testing.T) { testCancel(t, 65, FastSync) } func TestCancel65Light(t *testing.T) { testCancel(t, 65, LightSync) } +func TestCancel66Full(t *testing.T) { testCancel(t, 66, FullSync) } +func TestCancel66Fast(t *testing.T) { testCancel(t, 66, FastSync) } +func TestCancel66Light(t *testing.T) { testCancel(t, 66, LightSync) } + func testCancel(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -806,12 +835,17 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, 64, FullSync) } -func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) } +func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, 64, FullSync) } +func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) } + func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, 65, FullSync) } func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, 65, FastSync) } func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, 65, LightSync) } +func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, 66, FullSync) } +func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, 66, FastSync) } +func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, 66, LightSync) } + func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -834,12 +868,17 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, 64, FullSync) } -func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) } +func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, 64, FullSync) } +func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) } + func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, 65, FullSync) } func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, 65, FastSync) } func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, 65, LightSync) } +func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, 66, FullSync) } +func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, 66, FastSync) } +func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, 66, LightSync) } + func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -850,9 +889,9 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 63", 63, chain) tester.newPeer("peer 64", 64, chain) tester.newPeer("peer 65", 65, chain) + tester.newPeer("peer 66", 66, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -861,7 +900,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{63, 64, 65} { + for _, version := range []int{64, 65, 66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -871,12 +910,17 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, FullSync) } -func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) } +func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, FullSync) } +func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) } + func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, 65, FullSync) } func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, 65, FastSync) } func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, 65, LightSync) } +func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, 66, FullSync) } +func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, 66, FastSync) } +func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, 66, LightSync) } + func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -923,12 +967,17 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 64, FullSync) } -func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) } +func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 64, FullSync) } +func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) } + func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, 65, FullSync) } func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, 65, FastSync) } func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, 65, LightSync) } +func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, 66, FullSync) } +func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, 66, FastSync) } +func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, 66, LightSync) } + func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -953,12 +1002,17 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 64, FullSync) } -func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) } +func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 64, FullSync) } +func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) } + func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, 65, FullSync) } func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, 65, FastSync) } func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, 65, LightSync) } +func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, 66, FullSync) } +func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, 66, FastSync) } +func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, 66, LightSync) } + func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -990,6 +1044,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // sure no state was corrupted. func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) } func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, 65, FastSync) } +func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, 66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1079,12 +1134,17 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttack(t, 64, FullSync) } -func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } +func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttack(t, 64, FullSync) } +func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } + func TestHighTDStarvationAttack65Full(t *testing.T) { testHighTDStarvationAttack(t, 65, FullSync) } func TestHighTDStarvationAttack65Fast(t *testing.T) { testHighTDStarvationAttack(t, 65, FastSync) } func TestHighTDStarvationAttack65Light(t *testing.T) { testHighTDStarvationAttack(t, 65, LightSync) } +func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, 66, FullSync) } +func TestHighTDStarvationAttack66Fast(t *testing.T) { testHighTDStarvationAttack(t, 66, FastSync) } +func TestHighTDStarvationAttack66Light(t *testing.T) { testHighTDStarvationAttack(t, 66, LightSync) } + func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1101,6 +1161,7 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, 65) } +func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, 66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { t.Parallel() @@ -1152,12 +1213,17 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) } -func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) } +func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) } +func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) } + func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, 65, FullSync) } func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, 65, FastSync) } func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, 65, LightSync) } +func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, 66, FullSync) } +func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, 66, FastSync) } +func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, 66, LightSync) } + func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1234,12 +1300,17 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, FullSync) } -func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) } +func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, FullSync) } +func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) } + func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, 65, FullSync) } func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, 65, FastSync) } func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, 65, LightSync) } +func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, 66, FullSync) } +func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, 66, FastSync) } +func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, 66, LightSync) } + func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1308,12 +1379,17 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, FullSync) } -func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) } +func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, FullSync) } +func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) } + func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, 65, FullSync) } func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, 65, FastSync) } func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, 65, LightSync) } +func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, 66, FullSync) } +func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, 66, FastSync) } +func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, 66, LightSync) } + func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1379,12 +1455,17 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, FullSync) } -func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) } +func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, FullSync) } +func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) } + func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, 65, FullSync) } func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, 65, FastSync) } func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, 65, LightSync) } +func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, 66, FullSync) } +func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, 66, FastSync) } +func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, 66, LightSync) } + func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1454,12 +1535,17 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang64Full(t *testing.T) { testDeliverHeadersHang(t, 64, FullSync) } -func TestDeliverHeadersHang64Fast(t *testing.T) { testDeliverHeadersHang(t, 64, FastSync) } +func TestDeliverHeadersHang64Full(t *testing.T) { testDeliverHeadersHang(t, 64, FullSync) } +func TestDeliverHeadersHang64Fast(t *testing.T) { testDeliverHeadersHang(t, 64, FastSync) } + func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, 65, FullSync) } func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, 65, FastSync) } func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, 65, LightSync) } +func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, 66, FullSync) } +func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, 66, FastSync) } +func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, 66, LightSync) } + func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1613,12 +1699,17 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement64Full(t *testing.T) { testCheckpointEnforcement(t, 64, FullSync) } -func TestCheckpointEnforcement64Fast(t *testing.T) { testCheckpointEnforcement(t, 64, FastSync) } +func TestCheckpointEnforcement64Full(t *testing.T) { testCheckpointEnforcement(t, 64, FullSync) } +func TestCheckpointEnforcement64Fast(t *testing.T) { testCheckpointEnforcement(t, 64, FastSync) } + func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, 65, FullSync) } func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, 65, FastSync) } func TestCheckpointEnforcement65Light(t *testing.T) { testCheckpointEnforcement(t, 65, LightSync) } +func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, 66, FullSync) } +func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, 66, FastSync) } +func TestCheckpointEnforcement66Light(t *testing.T) { testCheckpointEnforcement(t, 66, LightSync) } + func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index ba90bf31cb..7852569d8e 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -29,6 +29,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" ) @@ -457,7 +458,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.headerThroughput } - return ps.idlePeers(64, 65, idle, throughput) + return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -471,7 +472,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.blockThroughput } - return ps.idlePeers(64, 65, idle, throughput) + return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -485,7 +486,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.receiptThroughput } - return ps.idlePeers(64, 65, idle, throughput) + return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -499,7 +500,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.stateThroughput } - return ps.idlePeers(64, 65, idle, throughput) + return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index e32008fb41..64648ed419 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -17,19 +17,17 @@ package eth import ( - "encoding/json" "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -166,6 +164,64 @@ func Handle(backend Backend, peer *Peer) error { } } +type msgHandler func(backend Backend, msg Decoder, peer *Peer) error +type Decoder interface { + Decode(val interface{}) error + Time() time.Time +} + +var eth64 = map[uint64]msgHandler{ + GetBlockHeadersMsg: handleGetBlockHeaders, + BlockHeadersMsg: handleBlockHeaders, + GetBlockBodiesMsg: handleGetBlockBodies, + BlockBodiesMsg: handleBlockBodies, + GetNodeDataMsg: handleGetNodeData, + NodeDataMsg: handleNodeData, + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, +} +var eth65 = map[uint64]msgHandler{ + // old 64 messages + GetBlockHeadersMsg: handleGetBlockHeaders, + BlockHeadersMsg: handleBlockHeaders, + GetBlockBodiesMsg: handleGetBlockBodies, + BlockBodiesMsg: handleBlockBodies, + GetNodeDataMsg: handleGetNodeData, + NodeDataMsg: handleNodeData, + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + // New eth65 messages + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + GetPooledTransactionsMsg: handleGetPooledTransactions, + PooledTransactionsMsg: handlePooledTransactions, +} + +var eth66 = map[uint64]msgHandler{ + // eth64 announcement messages (no id) + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + // eth65 announcement messages (no id) + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + // eth66 messages with request-id + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, +} + // handleMessage is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func handleMessage(backend Backend, peer *Peer) error { @@ -179,334 +235,15 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - // Handle the message depending on its contents - switch { - case msg.Code == StatusMsg: - // Status messages should never arrive after the handshake - return fmt.Errorf("%w: uncontrolled status message", errExtraStatusMsg) - - // Block header query, collect the requested headers and reply - case msg.Code == GetBlockHeadersMsg: - // Decode the complex header query - var query GetBlockHeadersPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - hashMode := query.Origin.Hash != (common.Hash{}) - first := true - maxNonCanonical := uint64(100) - - // Gather headers until the fetch or network limits is reached - var ( - bytes common.StorageSize - headers []*types.Header - unknown bool - lookups int - ) - for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit && - len(headers) < maxHeadersServe && lookups < 2*maxHeadersServe { - lookups++ - // Retrieve the next header satisfying the query - var origin *types.Header - if hashMode { - if first { - first = false - origin = backend.Chain().GetHeaderByHash(query.Origin.Hash) - if origin != nil { - query.Origin.Number = origin.Number.Uint64() - } - } else { - origin = backend.Chain().GetHeader(query.Origin.Hash, query.Origin.Number) - } - } else { - origin = backend.Chain().GetHeaderByNumber(query.Origin.Number) - } - if origin == nil { - break - } - headers = append(headers, origin) - bytes += estHeaderSize - - // Advance to the next header of the query - switch { - case hashMode && query.Reverse: - // Hash based traversal towards the genesis block - ancestor := query.Skip + 1 - if ancestor == 0 { - unknown = true - } else { - query.Origin.Hash, query.Origin.Number = backend.Chain().GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) - unknown = (query.Origin.Hash == common.Hash{}) - } - case hashMode && !query.Reverse: - // Hash based traversal towards the leaf block - var ( - current = origin.Number.Uint64() - next = current + query.Skip + 1 - ) - if next <= current { - infos, _ := json.MarshalIndent(peer.Peer.Info(), "", " ") - peer.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) - unknown = true - } else { - if header := backend.Chain().GetHeaderByNumber(next); header != nil { - nextHash := header.Hash() - expOldHash, _ := backend.Chain().GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) - if expOldHash == query.Origin.Hash { - query.Origin.Hash, query.Origin.Number = nextHash, next - } else { - unknown = true - } - } else { - unknown = true - } - } - case query.Reverse: - // Number based traversal towards the genesis block - if query.Origin.Number >= query.Skip+1 { - query.Origin.Number -= query.Skip + 1 - } else { - unknown = true - } - - case !query.Reverse: - // Number based traversal towards the leaf block - query.Origin.Number += query.Skip + 1 - } - } - return peer.SendBlockHeaders(headers) - - case msg.Code == BlockHeadersMsg: - // A batch of headers arrived to one of our previous requests - res := new(BlockHeadersPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) - - case msg.Code == GetBlockBodiesMsg: - // Decode the block body retrieval message - var query GetBlockBodiesPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Gather blocks until the fetch or network limits is reached - var ( - bytes int - bodies []rlp.RawValue - ) - for lookups, hash := range query { - if bytes >= softResponseLimit || len(bodies) >= maxBodiesServe || - lookups >= 2*maxBodiesServe { - break - } - if data := backend.Chain().GetBodyRLP(hash); len(data) != 0 { - bodies = append(bodies, data) - bytes += len(data) - } - } - return peer.SendBlockBodiesRLP(bodies) - - case msg.Code == BlockBodiesMsg: - // A batch of block bodies arrived to one of our previous requests - res := new(BlockBodiesPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) - - case msg.Code == GetNodeDataMsg: - // Decode the trie node data retrieval message - var query GetNodeDataPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Gather state data until the fetch or network limits is reached - var ( - bytes int - nodes [][]byte - ) - for lookups, hash := range query { - if bytes >= softResponseLimit || len(nodes) >= maxNodeDataServe || - lookups >= 2*maxNodeDataServe { - break - } - // Retrieve the requested state entry - if bloom := backend.StateBloom(); bloom != nil && !bloom.Contains(hash[:]) { - // Only lookup the trie node if there's chance that we actually have it - continue - } - entry, err := backend.Chain().TrieNode(hash) - if len(entry) == 0 || err != nil { - // Read the contract code with prefix only to save unnecessary lookups. - entry, err = backend.Chain().ContractCodeWithPrefix(hash) - } - if err == nil && len(entry) > 0 { - nodes = append(nodes, entry) - bytes += len(entry) - } - } - return peer.SendNodeData(nodes) - - case msg.Code == NodeDataMsg: - // A batch of node state data arrived to one of our previous requests - res := new(NodeDataPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) - - case msg.Code == GetReceiptsMsg: - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Gather state data until the fetch or network limits is reached - var ( - bytes int - receipts []rlp.RawValue - ) - for lookups, hash := range query { - if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || - lookups >= 2*maxReceiptsServe { - break - } - // Retrieve the requested block's receipts - results := backend.Chain().GetReceiptsByHash(hash) - if results == nil { - if header := backend.Chain().GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - continue - } - } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(results); err != nil { - log.Error("Failed to encode receipt", "err", err) - } else { - receipts = append(receipts, encoded) - bytes += len(encoded) - } - } - return peer.SendReceiptsRLP(receipts) - - case msg.Code == ReceiptsMsg: - // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) - - case msg.Code == NewBlockHashesMsg: - // A batch of new block announcements just arrived - ann := new(NewBlockHashesPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Mark the hashes as present at the remote node - for _, block := range *ann { - peer.markBlock(block.Hash) - } - // Deliver them all to the backend for queuing - return backend.Handle(peer, ann) - - case msg.Code == NewBlockMsg: - // Retrieve and decode the propagated block - ann := new(NewBlockPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { - log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) - break // TODO(karalabe): return error eventually, but wait a few releases - } - if hash := types.DeriveSha(ann.Block.Transactions(), trie.NewStackTrie(nil)); hash != ann.Block.TxHash() { - log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) - break // TODO(karalabe): return error eventually, but wait a few releases - } - if err := ann.sanityCheck(); err != nil { - return err - } - ann.Block.ReceivedAt = msg.ReceivedAt - ann.Block.ReceivedFrom = peer - - // Mark the peer as owning the block - peer.markBlock(ann.Block.Hash()) - - return backend.Handle(peer, ann) - - case msg.Code == NewPooledTransactionHashesMsg && peer.version >= ETH65: - // New transaction announcement arrived, make sure we have - // a valid and fresh chain to handle them - if !backend.AcceptTxs() { - break - } - ann := new(NewPooledTransactionHashesPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Schedule all the unknown hashes for retrieval - for _, hash := range *ann { - peer.markTransaction(hash) - } - return backend.Handle(peer, ann) - - case msg.Code == GetPooledTransactionsMsg && peer.version >= ETH65: - // Decode the pooled transactions retrieval message - var query GetPooledTransactionsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Gather transactions until the fetch or network limits is reached - var ( - bytes int - hashes []common.Hash - txs []rlp.RawValue - ) - for _, hash := range query { - if bytes >= softResponseLimit { - break - } - // Retrieve the requested transaction, skipping if unknown to us - tx := backend.TxPool().Get(hash) - if tx == nil { - continue - } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(tx); err != nil { - log.Error("Failed to encode transaction", "err", err) - } else { - hashes = append(hashes, hash) - txs = append(txs, encoded) - bytes += len(encoded) - } - } - return peer.SendPooledTransactionsRLP(hashes, txs) - - case msg.Code == TransactionsMsg || (msg.Code == PooledTransactionsMsg && peer.version >= ETH65): - // Transactions arrived, make sure we have a valid and fresh chain to handle them - if !backend.AcceptTxs() { - break - } - // Transactions can be processed, parse all of them and deliver to the pool - var txs []*types.Transaction - if err := msg.Decode(&txs); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("%w: transaction %d is nil", errDecode, i) - } - peer.markTransaction(tx.Hash()) - } - if msg.Code == PooledTransactionsMsg { - return backend.Handle(peer, (*PooledTransactionsPacket)(&txs)) - } - return backend.Handle(peer, (*TransactionsPacket)(&txs)) + var handlers = eth64 + if peer.Version() == ETH65 { + handlers = eth65 + } else if peer.Version() >= ETH66 { + handlers = eth66 + } - default: - return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) + if handler := handlers[msg.Code]; handler != nil { + return handler(backend, msg, peer) } - return nil + return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go new file mode 100644 index 0000000000..8433fa343a --- /dev/null +++ b/eth/protocols/eth/handlers.go @@ -0,0 +1,510 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// handleGetBlockHeaders handles Block header query, collect the requested headers and reply +func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { + // Decode the complex header query + var query GetBlockHeadersPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockHeadersQuery(backend, &query, peer) + return peer.SendBlockHeaders(response) +} + +// handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders +func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { + // Decode the complex header query + var query GetBlockHeadersPacket66 + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockHeadersQuery(backend, query.GetBlockHeadersPacket, peer) + return peer.ReplyBlockHeaders(query.RequestId, response) +} + +func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, peer *Peer) []*types.Header { + hashMode := query.Origin.Hash != (common.Hash{}) + first := true + maxNonCanonical := uint64(100) + + // Gather headers until the fetch or network limits is reached + var ( + bytes common.StorageSize + headers []*types.Header + unknown bool + lookups int + ) + for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit && + len(headers) < maxHeadersServe && lookups < 2*maxHeadersServe { + lookups++ + // Retrieve the next header satisfying the query + var origin *types.Header + if hashMode { + if first { + first = false + origin = backend.Chain().GetHeaderByHash(query.Origin.Hash) + if origin != nil { + query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = backend.Chain().GetHeader(query.Origin.Hash, query.Origin.Number) + } + } else { + origin = backend.Chain().GetHeaderByNumber(query.Origin.Number) + } + if origin == nil { + break + } + headers = append(headers, origin) + bytes += estHeaderSize + + // Advance to the next header of the query + switch { + case hashMode && query.Reverse: + // Hash based traversal towards the genesis block + ancestor := query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + query.Origin.Hash, query.Origin.Number = backend.Chain().GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) + unknown = (query.Origin.Hash == common.Hash{}) + } + case hashMode && !query.Reverse: + // Hash based traversal towards the leaf block + var ( + current = origin.Number.Uint64() + next = current + query.Skip + 1 + ) + if next <= current { + infos, _ := json.MarshalIndent(peer.Peer.Info(), "", " ") + peer.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) + unknown = true + } else { + if header := backend.Chain().GetHeaderByNumber(next); header != nil { + nextHash := header.Hash() + expOldHash, _ := backend.Chain().GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) + if expOldHash == query.Origin.Hash { + query.Origin.Hash, query.Origin.Number = nextHash, next + } else { + unknown = true + } + } else { + unknown = true + } + } + case query.Reverse: + // Number based traversal towards the genesis block + if query.Origin.Number >= query.Skip+1 { + query.Origin.Number -= query.Skip + 1 + } else { + unknown = true + } + + case !query.Reverse: + // Number based traversal towards the leaf block + query.Origin.Number += query.Skip + 1 + } + } + return headers +} + +func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block body retrieval message + var query GetBlockBodiesPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockBodiesQuery(backend, query, peer) + return peer.SendBlockBodiesRLP(response) +} + +func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block body retrieval message + var query GetBlockBodiesPacket66 + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockBodiesQuery(backend, query.GetBlockBodiesPacket, peer) + return peer.ReplyBlockBodiesRLP(query.RequestId, response) +} + +func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer *Peer) []rlp.RawValue { + // Gather blocks until the fetch or network limits is reached + var ( + bytes int + bodies []rlp.RawValue + ) + for lookups, hash := range query { + if bytes >= softResponseLimit || len(bodies) >= maxBodiesServe || + lookups >= 2*maxBodiesServe { + break + } + if data := backend.Chain().GetBodyRLP(hash); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) + } + } + return bodies +} + +func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { + // Decode the trie node data retrieval message + var query GetNodeDataPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetNodeDataQuery(backend, query, peer) + return peer.SendNodeData(response) +} + +func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { + // Decode the trie node data retrieval message + var query GetNodeDataPacket66 + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetNodeDataQuery(backend, query.GetNodeDataPacket, peer) + return peer.ReplyNodeData(query.RequestId, response) +} + +func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer) [][]byte { + // Gather state data until the fetch or network limits is reached + var ( + bytes int + nodes [][]byte + ) + for lookups, hash := range query { + if bytes >= softResponseLimit || len(nodes) >= maxNodeDataServe || + lookups >= 2*maxNodeDataServe { + break + } + // Retrieve the requested state entry + if bloom := backend.StateBloom(); bloom != nil && !bloom.Contains(hash[:]) { + // Only lookup the trie node if there's chance that we actually have it + continue + } + entry, err := backend.Chain().TrieNode(hash) + if len(entry) == 0 || err != nil { + // Read the contract code with prefix only to save unnecessary lookups. + entry, err = backend.Chain().ContractCodeWithPrefix(hash) + } + if err == nil && len(entry) > 0 { + nodes = append(nodes, entry) + bytes += len(entry) + } + } + return nodes +} + +func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block receipts retrieval message + var query GetReceiptsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetReceiptsQuery(backend, query, peer) + return peer.SendReceiptsRLP(response) +} + +func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block receipts retrieval message + var query GetReceiptsPacket66 + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetReceiptsQuery(backend, query.GetReceiptsPacket, peer) + return peer.ReplyReceiptsRLP(query.RequestId, response) +} + +func answerGetReceiptsQuery(backend Backend, query GetReceiptsPacket, peer *Peer) []rlp.RawValue { + // Gather state data until the fetch or network limits is reached + var ( + bytes int + receipts []rlp.RawValue + ) + for lookups, hash := range query { + if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || + lookups >= 2*maxReceiptsServe { + break + } + // Retrieve the requested block's receipts + results := backend.Chain().GetReceiptsByHash(hash) + if results == nil { + if header := backend.Chain().GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + continue + } + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(results); err != nil { + log.Error("Failed to encode receipt", "err", err) + } else { + receipts = append(receipts, encoded) + bytes += len(encoded) + } + } + return receipts +} + +func handleNewBlockhashes(backend Backend, msg Decoder, peer *Peer) error { + // A batch of new block announcements just arrived + ann := new(NewBlockHashesPacket) + if err := msg.Decode(ann); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Mark the hashes as present at the remote node + for _, block := range *ann { + peer.markBlock(block.Hash) + } + // Deliver them all to the backend for queuing + return backend.Handle(peer, ann) +} + +func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { + // Retrieve and decode the propagated block + ann := new(NewBlockPacket) + if err := msg.Decode(ann); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { + log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) + return nil // TODO(karalabe): return error eventually, but wait a few releases + } + if hash := types.DeriveSha(ann.Block.Transactions(), trie.NewStackTrie(nil)); hash != ann.Block.TxHash() { + log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) + return nil // TODO(karalabe): return error eventually, but wait a few releases + } + if err := ann.sanityCheck(); err != nil { + return err + } + ann.Block.ReceivedAt = msg.Time() + ann.Block.ReceivedFrom = peer + + // Mark the peer as owning the block + peer.markBlock(ann.Block.Hash()) + + return backend.Handle(peer, ann) +} + +func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { + // A batch of headers arrived to one of our previous requests + res := new(BlockHeadersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + +func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { + // A batch of headers arrived to one of our previous requests + res := new(BlockHeadersPacket66) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, &res.BlockHeadersPacket) +} + +func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { + // A batch of block bodies arrived to one of our previous requests + res := new(BlockBodiesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + +func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { + // A batch of block bodies arrived to one of our previous requests + res := new(BlockBodiesPacket66) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, &res.BlockBodiesPacket) +} + +func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { + // A batch of node state data arrived to one of our previous requests + res := new(NodeDataPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + +func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { + // A batch of node state data arrived to one of our previous requests + res := new(NodeDataPacket66) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, &res.NodeDataPacket) +} + +func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { + // A batch of receipts arrived to one of our previous requests + res := new(ReceiptsPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + +func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { + // A batch of receipts arrived to one of our previous requests + res := new(ReceiptsPacket66) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, &res.ReceiptsPacket) +} + +func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) error { + // New transaction announcement arrived, make sure we have + // a valid and fresh chain to handle them + if !backend.AcceptTxs() { + return nil + } + ann := new(NewPooledTransactionHashesPacket) + if err := msg.Decode(ann); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + // Schedule all the unknown hashes for retrieval + for _, hash := range *ann { + peer.markTransaction(hash) + } + return backend.Handle(peer, ann) +} + +func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Decode the pooled transactions retrieval message + var query GetPooledTransactionsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + hashes, txs := answerGetPooledTransactions(backend, query, peer) + return peer.SendPooledTransactionsRLP(hashes, txs) +} + +func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { + // Decode the pooled transactions retrieval message + var query GetPooledTransactionsPacket66 + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsPacket, peer) + return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs) +} + +func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsPacket, peer *Peer) ([]common.Hash, []rlp.RawValue) { + // Gather transactions until the fetch or network limits is reached + var ( + bytes int + hashes []common.Hash + txs []rlp.RawValue + ) + for _, hash := range query { + if bytes >= softResponseLimit { + break + } + // Retrieve the requested transaction, skipping if unknown to us + tx := backend.TxPool().Get(hash) + if tx == nil { + continue + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(tx); err != nil { + log.Error("Failed to encode transaction", "err", err) + } else { + hashes = append(hashes, hash) + txs = append(txs, encoded) + bytes += len(encoded) + } + } + return hashes, txs +} + +func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Transactions arrived, make sure we have a valid and fresh chain to handle them + if !backend.AcceptTxs() { + return nil + } + // Transactions can be processed, parse all of them and deliver to the pool + var txs TransactionsPacket + if err := msg.Decode(&txs); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + for i, tx := range txs { + // Validate and mark the remote transaction + if tx == nil { + return fmt.Errorf("%w: transaction %d is nil", errDecode, i) + } + peer.markTransaction(tx.Hash()) + } + return backend.Handle(peer, &txs) +} + +func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Transactions arrived, make sure we have a valid and fresh chain to handle them + if !backend.AcceptTxs() { + return nil + } + // Transactions can be processed, parse all of them and deliver to the pool + var txs PooledTransactionsPacket + if err := msg.Decode(&txs); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + for i, tx := range txs { + // Validate and mark the remote transaction + if tx == nil { + return fmt.Errorf("%w: transaction %d is nil", errDecode, i) + } + peer.markTransaction(tx.Hash()) + } + return backend.Handle(peer, &txs) +} + +func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { + // Transactions arrived, make sure we have a valid and fresh chain to handle them + if !backend.AcceptTxs() { + return nil + } + // Transactions can be processed, parse all of them and deliver to the pool + var txs PooledTransactionsPacket66 + if err := msg.Decode(&txs); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + for i, tx := range txs.PooledTransactionsPacket { + // Validate and mark the remote transaction + if tx == nil { + return fmt.Errorf("%w: transaction %d is nil", errDecode, i) + } + peer.markTransaction(tx.Hash()) + } + return backend.Handle(peer, &txs.PooledTransactionsPacket) +} diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 735ef78ce7..709fca8655 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -18,6 +18,7 @@ package eth import ( "math/big" + "math/rand" "sync" mapset "github.com/deckarep/golang-set" @@ -267,6 +268,22 @@ func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValu return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding } +// ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. +func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + // Not packed into PooledTransactionsPacket to avoid RLP decoding + return p2p.Send(p.rw, PooledTransactionsMsg, PooledTransactionsRLPPacket66{ + RequestId: id, + PooledTransactionsRLPPacket: txs, + }) +} + // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *Peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -308,7 +325,10 @@ func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { p.knownBlocks.Pop() } p.knownBlocks.Add(block.Hash()) - return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{block, td}) + return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{ + Block: block, + TD: td, + }) } // AsyncSendNewBlock queues an entire block for propagation to a remote peer. If @@ -331,9 +351,12 @@ func (p *Peer) SendBlockHeaders(headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) } -// SendBlockBodies sends a batch of block contents to the remote peer. -func (p *Peer) SendBlockBodies(bodies []*BlockBody) error { - return p2p.Send(p.rw, BlockBodiesMsg, BlockBodiesPacket(bodies)) +// ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. +func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { + return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: id, + BlockHeadersPacket: headers, + }) } // SendBlockBodiesRLP sends a batch of block contents to the remote peer from @@ -342,52 +365,98 @@ func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding } +// ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. +func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { + // Not packed into BlockBodiesPacket to avoid RLP decoding + return p2p.Send(p.rw, BlockBodiesMsg, BlockBodiesRLPPacket66{ + RequestId: id, + BlockBodiesRLPPacket: bodies, + }) +} + // SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the // hashes requested. func (p *Peer) SendNodeData(data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) } +// ReplyNodeData is the eth/66 response to GetNodeData. +func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { + return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ + RequestId: id, + NodeDataPacket: data, + }) +} + // SendReceiptsRLP sends a batch of transaction receipts, corresponding to the // ones requested from an already RLP encoded format. func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding } +// ReplyReceiptsRLP is the eth/66 response to GetReceipts. +func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { + return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ + RequestId: id, + ReceiptsRLPPacket: receipts, + }) +} + // RequestOneHeader is a wrapper around the header query functions to fetch a // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket{ + query := GetBlockHeadersPacket{ Origin: HashOrNumber{Hash: hash}, Amount: uint64(1), Skip: uint64(0), Reverse: false, - }) + } + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: rand.Uint64(), + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket{ + query := GetBlockHeadersPacket{ Origin: HashOrNumber{Hash: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse, - }) + } + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: rand.Uint64(), + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket{ + query := GetBlockHeadersPacket{ Origin: HashOrNumber{Number: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse, - }) + } + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: rand.Uint64(), + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // ExpectRequestHeadersByNumber is a testing method to mirror the recipient side @@ -406,6 +475,12 @@ func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: rand.Uint64(), + GetBlockBodiesPacket: hashes, + }) + } return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) } @@ -413,17 +488,35 @@ func (p *Peer) RequestBodies(hashes []common.Hash) error { // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: rand.Uint64(), + GetNodeDataPacket: hashes, + }) + } return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: rand.Uint64(), + GetReceiptsPacket: hashes, + }) + } return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + if p.Version() >= ETH66 { + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: rand.Uint64(), + GetPooledTransactionsPacket: hashes, + }) + } return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 9fff64b72a..7f1832754f 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -32,6 +32,7 @@ import ( const ( ETH64 = 64 ETH65 = 65 + ETH66 = 66 ) // ProtocolName is the official short name of the `eth` protocol used during @@ -40,11 +41,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH65, ETH64} +var ProtocolVersions = []uint{ETH66, ETH65, ETH64} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH65: 17, ETH64: 17} +var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17, ETH64: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -79,7 +80,6 @@ var ( errNetworkIDMismatch = errors.New("network ID mismatch") errGenesisMismatch = errors.New("genesis mismatch") errForkIDRejected = errors.New("fork ID rejected") - errExtraStatusMsg = errors.New("extra status message") ) // Packet represents a p2p message in the `eth` protocol. @@ -129,6 +129,12 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } +// GetBlockHeadersPacket represents a block header query over eth/66 +type GetBlockHeadersPacket66 struct { + RequestId uint64 + *GetBlockHeadersPacket +} + // HashOrNumber is a combined field for specifying an origin block. type HashOrNumber struct { Hash common.Hash // Block hash from which to retrieve headers (excludes Number) @@ -168,6 +174,12 @@ func (hn *HashOrNumber) DecodeRLP(s *rlp.Stream) error { // BlockHeadersPacket represents a block header response. type BlockHeadersPacket []*types.Header +// BlockHeadersPacket represents a block header response over eth/66. +type BlockHeadersPacket66 struct { + RequestId uint64 + BlockHeadersPacket +} + // NewBlockPacket is the network packet for the block propagation message. type NewBlockPacket struct { Block *types.Block @@ -190,9 +202,32 @@ func (request *NewBlockPacket) sanityCheck() error { // GetBlockBodiesPacket represents a block body query. type GetBlockBodiesPacket []common.Hash +// GetBlockBodiesPacket represents a block body query over eth/66. +type GetBlockBodiesPacket66 struct { + RequestId uint64 + GetBlockBodiesPacket +} + // BlockBodiesPacket is the network packet for block content distribution. type BlockBodiesPacket []*BlockBody +// BlockBodiesPacket is the network packet for block content distribution over eth/66. +type BlockBodiesPacket66 struct { + RequestId uint64 + BlockBodiesPacket +} + +// BlockBodiesRLPPacket is used for replying to block body requests, in cases +// where we already have them RLP-encoded, and thus can avoid the decode-encode +// roundtrip. +type BlockBodiesRLPPacket []rlp.RawValue + +// BlockBodiesRLPPacket66 is the BlockBodiesRLPPacket over eth/66 +type BlockBodiesRLPPacket66 struct { + RequestId uint64 + BlockBodiesRLPPacket +} + // BlockBody represents the data content of a single block. type BlockBody struct { Transactions []*types.Transaction // Transactions contained within a block @@ -215,24 +250,78 @@ func (p *BlockBodiesPacket) Unpack() ([][]*types.Transaction, [][]*types.Header) // GetNodeDataPacket represents a trie node data query. type GetNodeDataPacket []common.Hash +// GetNodeDataPacket represents a trie node data query over eth/66. +type GetNodeDataPacket66 struct { + RequestId uint64 + GetNodeDataPacket +} + // NodeDataPacket is the network packet for trie node data distribution. type NodeDataPacket [][]byte +// NodeDataPacket is the network packet for trie node data distribution over eth/66. +type NodeDataPacket66 struct { + RequestId uint64 + NodeDataPacket +} + // GetReceiptsPacket represents a block receipts query. type GetReceiptsPacket []common.Hash +// GetReceiptsPacket represents a block receipts query over eth/66. +type GetReceiptsPacket66 struct { + RequestId uint64 + GetReceiptsPacket +} + // ReceiptsPacket is the network packet for block receipts distribution. type ReceiptsPacket [][]*types.Receipt +// ReceiptsPacket is the network packet for block receipts distribution over eth/66. +type ReceiptsPacket66 struct { + RequestId uint64 + ReceiptsPacket +} + +// ReceiptsRLPPacket is used for receipts, when we already have it encoded +type ReceiptsRLPPacket []rlp.RawValue + +// ReceiptsPacket66 is the eth-66 version of ReceiptsRLPPacket +type ReceiptsRLPPacket66 struct { + RequestId uint64 + ReceiptsRLPPacket +} + // NewPooledTransactionHashesPacket represents a transaction announcement packet. type NewPooledTransactionHashesPacket []common.Hash // GetPooledTransactionsPacket represents a transaction query. type GetPooledTransactionsPacket []common.Hash +type GetPooledTransactionsPacket66 struct { + RequestId uint64 + GetPooledTransactionsPacket +} + // PooledTransactionsPacket is the network packet for transaction distribution. type PooledTransactionsPacket []*types.Transaction +// PooledTransactionsPacket is the network packet for transaction distribution over eth/66. +type PooledTransactionsPacket66 struct { + RequestId uint64 + PooledTransactionsPacket +} + +// PooledTransactionsPacket is the network packet for transaction distribution, used +// in the cases we already have them in rlp-encoded form +type PooledTransactionsRLPPacket []rlp.RawValue + +// PooledTransactionsRLPPacket66 is the eth/66 form of PooledTransactionsRLPPacket +type PooledTransactionsRLPPacket66 struct { + RequestId uint64 + PooledTransactionsRLPPacket +} + func (*StatusPacket) Name() string { return "Status" } func (*StatusPacket) Kind() byte { return StatusMsg } diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index 056ea56480..d92f3ea837 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -17,9 +17,12 @@ package eth import ( + "bytes" + "math/big" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" ) @@ -66,3 +69,200 @@ func TestGetBlockHeadersDataEncodeDecode(t *testing.T) { } } } + +// TestEth66EmptyMessages tests encoding of empty eth66 messages +func TestEth66EmptyMessages(t *testing.T) { + // All empty messages encodes to the same format + want := common.FromHex("c4820457c0") + + for i, msg := range []interface{}{ + // Headers + GetBlockHeadersPacket66{1111, nil}, + BlockHeadersPacket66{1111, nil}, + // Bodies + GetBlockBodiesPacket66{1111, nil}, + BlockBodiesPacket66{1111, nil}, + BlockBodiesRLPPacket66{1111, nil}, + // Node data + GetNodeDataPacket66{1111, nil}, + NodeDataPacket66{1111, nil}, + // Receipts + GetReceiptsPacket66{1111, nil}, + ReceiptsPacket66{1111, nil}, + // Transactions + GetPooledTransactionsPacket66{1111, nil}, + PooledTransactionsPacket66{1111, nil}, + PooledTransactionsRLPPacket66{1111, nil}, + + // Headers + BlockHeadersPacket66{1111, BlockHeadersPacket([]*types.Header{})}, + // Bodies + GetBlockBodiesPacket66{1111, GetBlockBodiesPacket([]common.Hash{})}, + BlockBodiesPacket66{1111, BlockBodiesPacket([]*BlockBody{})}, + BlockBodiesRLPPacket66{1111, BlockBodiesRLPPacket([]rlp.RawValue{})}, + // Node data + GetNodeDataPacket66{1111, GetNodeDataPacket([]common.Hash{})}, + NodeDataPacket66{1111, NodeDataPacket([][]byte{})}, + // Receipts + GetReceiptsPacket66{1111, GetReceiptsPacket([]common.Hash{})}, + ReceiptsPacket66{1111, ReceiptsPacket([][]*types.Receipt{})}, + // Transactions + GetPooledTransactionsPacket66{1111, GetPooledTransactionsPacket([]common.Hash{})}, + PooledTransactionsPacket66{1111, PooledTransactionsPacket([]*types.Transaction{})}, + PooledTransactionsRLPPacket66{1111, PooledTransactionsRLPPacket([]rlp.RawValue{})}, + } { + if have, _ := rlp.EncodeToBytes(msg); !bytes.Equal(have, want) { + t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, msg, have, want) + } + } + +} + +// TestEth66Messages tests the encoding of all redefined eth66 messages +func TestEth66Messages(t *testing.T) { + + // Some basic structs used during testing + var ( + header *types.Header + blockBody *BlockBody + blockBodyRlp rlp.RawValue + txs []*types.Transaction + txRlps []rlp.RawValue + hashes []common.Hash + receipts []*types.Receipt + receiptsRlp rlp.RawValue + + err error + ) + header = &types.Header{ + Difficulty: big.NewInt(2222), + Number: big.NewInt(3333), + GasLimit: 4444, + GasUsed: 5555, + Time: 6666, + Extra: []byte{0x77, 0x88}, + } + // Init the transactions, taken from a different test + { + for _, hexrlp := range []string{ + "f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10", + "f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afb", + } { + var tx *types.Transaction + rlpdata := common.FromHex(hexrlp) + if err := rlp.DecodeBytes(rlpdata, &tx); err != nil { + t.Fatal(err) + } + txs = append(txs, tx) + txRlps = append(txRlps, rlpdata) + } + } + // init the block body data, both object and rlp form + blockBody = &BlockBody{ + Transactions: txs, + Uncles: []*types.Header{header}, + } + blockBodyRlp, err = rlp.EncodeToBytes(blockBody) + if err != nil { + t.Fatal(err) + } + + hashes = []common.Hash{ + common.HexToHash("deadc0de"), + common.HexToHash("feedbeef"), + } + byteSlices := [][]byte{ + common.FromHex("deadc0de"), + common.FromHex("feedbeef"), + } + // init the receipts + { + receipts = []*types.Receipt{ + &types.Receipt{ + Status: types.ReceiptStatusFailed, + CumulativeGasUsed: 1, + Logs: []*types.Log{ + { + Address: common.BytesToAddress([]byte{0x11}), + Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, + Data: []byte{0x01, 0x00, 0xff}, + }, + }, + TxHash: hashes[0], + ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), + GasUsed: 111111, + }, + } + rlpData, err := rlp.EncodeToBytes(receipts) + if err != nil { + t.Fatal(err) + } + receiptsRlp = rlpData + } + + for i, tc := range []struct { + message interface{} + want []byte + }{ + { + GetBlockHeadersPacket66{1111, &GetBlockHeadersPacket{HashOrNumber{hashes[0], 0}, 5, 5, false}}, + common.FromHex("e8820457e4a000000000000000000000000000000000000000000000000000000000deadc0de050580"), + }, + { + GetBlockHeadersPacket66{1111, &GetBlockHeadersPacket{HashOrNumber{common.Hash{}, 9999}, 5, 5, false}}, + common.FromHex("ca820457c682270f050580"), + }, + { + BlockHeadersPacket66{1111, BlockHeadersPacket{header}}, + common.FromHex("f90202820457f901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), + }, + { + GetBlockBodiesPacket66{1111, GetBlockBodiesPacket(hashes)}, + common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), + }, + { + BlockBodiesPacket66{1111, BlockBodiesPacket([]*BlockBody{blockBody})}, + common.FromHex("f902dc820457f902d6f902d3f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afbf901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), + }, + { // Identical to non-rlp-shortcut version + BlockBodiesRLPPacket66{1111, BlockBodiesRLPPacket([]rlp.RawValue{blockBodyRlp})}, + common.FromHex("f902dc820457f902d6f902d3f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afbf901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), + }, + { + GetNodeDataPacket66{1111, GetNodeDataPacket(hashes)}, + common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), + }, + { + NodeDataPacket66{1111, NodeDataPacket(byteSlices)}, + common.FromHex("ce820457ca84deadc0de84feedbeef"), + }, + { + GetReceiptsPacket66{1111, GetReceiptsPacket(hashes)}, + common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), + }, + { + ReceiptsPacket66{1111, ReceiptsPacket([][]*types.Receipt{receipts})}, + common.FromHex("f90172820457f9016cf90169f901668001b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ff"), + }, + { + ReceiptsRLPPacket66{1111, ReceiptsRLPPacket([]rlp.RawValue{receiptsRlp})}, + common.FromHex("f90172820457f9016cf90169f901668001b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ff"), + }, + { + GetPooledTransactionsPacket66{1111, GetPooledTransactionsPacket(hashes)}, + common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), + }, + { + PooledTransactionsPacket66{1111, PooledTransactionsPacket(txs)}, + common.FromHex("f8d7820457f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afb"), + }, + { + PooledTransactionsRLPPacket66{1111, PooledTransactionsRLPPacket(txRlps)}, + common.FromHex("f8d7820457f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afb"), + }, + } { + if have, _ := rlp.EncodeToBytes(tc.message); !bytes.Equal(have, tc.want) { + t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, tc.message, have, tc.want) + } + } +} diff --git a/p2p/message.go b/p2p/message.go index 10b55a939c..bd048138c3 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -70,6 +70,10 @@ func (msg Msg) Discard() error { return err } +func (msg Msg) Time() time.Time { + return msg.ReceivedAt +} + type MsgReader interface { ReadMsg() (Msg, error) } From d36276d85e39f7a0071d3f5d948785e008ca1519 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 19 Feb 2021 09:54:46 +0100 Subject: [PATCH 144/709] p2p/dnsdisc: fix hot-spin when all trees are empty (#22313) In the random sync algorithm used by the DNS node iterator, we first pick a random tree and then perform one sync action on that tree. This happens in a loop until any node is found. If no trees contain any nodes, the iterator will enter a hot loop spinning at 100% CPU. The fix is complicated. The iterator now checks if a meaningful sync action can be performed on any tree. If there is nothing to do, it waits for the next root record recheck time to arrive and then tries again. Fixes #22306 --- p2p/dnsdisc/client.go | 82 ++++++++++++++++++++++++++++++++------ p2p/dnsdisc/client_test.go | 47 ++++++++++++++++++++++ p2p/dnsdisc/sync.go | 28 ++++++++++--- 3 files changed, 138 insertions(+), 19 deletions(-) diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index b872784828..d3e8111ab5 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -217,8 +217,11 @@ type randomIterator struct { c *Client mu sync.Mutex - trees map[string]*clientTree // all trees lc linkCache // tracks tree dependencies + trees map[string]*clientTree // all trees + // buffers for syncableTrees + syncableList []*clientTree + disabledList []*clientTree } func (c *Client) newRandomIterator() *randomIterator { @@ -238,10 +241,10 @@ func (it *randomIterator) Node() *enode.Node { // Close closes the iterator. func (it *randomIterator) Close() { + it.cancelFn() + it.mu.Lock() defer it.mu.Unlock() - - it.cancelFn() it.trees = nil } @@ -264,7 +267,7 @@ func (it *randomIterator) addTree(url string) error { // nextNode syncs random tree entries until it finds a node. func (it *randomIterator) nextNode() *enode.Node { for { - ct := it.nextTree() + ct := it.pickTree() if ct == nil { return nil } @@ -282,26 +285,79 @@ func (it *randomIterator) nextNode() *enode.Node { } } -// nextTree returns a random tree. -func (it *randomIterator) nextTree() *clientTree { +// pickTree returns a random tree to sync from. +func (it *randomIterator) pickTree() *clientTree { it.mu.Lock() defer it.mu.Unlock() + // Rebuild the trees map if any links have changed. if it.lc.changed { it.rebuildTrees() it.lc.changed = false } - if len(it.trees) == 0 { - return nil + + for { + canSync, trees := it.syncableTrees() + switch { + case canSync: + // Pick a random tree. + return trees[rand.Intn(len(trees))] + case len(trees) > 0: + // No sync action can be performed on any tree right now. The only meaningful + // thing to do is waiting for any root record to get updated. + if !it.waitForRootUpdates(trees) { + // Iterator was closed while waiting. + return nil + } + default: + // There are no trees left, the iterator was closed. + return nil + } } - limit := rand.Intn(len(it.trees)) +} + +// syncableTrees finds trees on which any meaningful sync action can be performed. +func (it *randomIterator) syncableTrees() (canSync bool, trees []*clientTree) { + // Resize tree lists. + it.syncableList = it.syncableList[:0] + it.disabledList = it.disabledList[:0] + + // Partition them into the two lists. for _, ct := range it.trees { - if limit == 0 { - return ct + if ct.canSyncRandom() { + it.syncableList = append(it.syncableList, ct) + } else { + it.disabledList = append(it.disabledList, ct) } - limit-- } - return nil + if len(it.syncableList) > 0 { + return true, it.syncableList + } + return false, it.disabledList +} + +// waitForRootUpdates waits for the closest scheduled root check time on the given trees. +func (it *randomIterator) waitForRootUpdates(trees []*clientTree) bool { + var minTree *clientTree + var nextCheck mclock.AbsTime + for _, ct := range trees { + check := ct.nextScheduledRootCheck() + if minTree == nil || check < nextCheck { + minTree = ct + nextCheck = check + } + } + + sleep := nextCheck.Sub(it.c.clock.Now()) + it.c.cfg.Logger.Debug("DNS iterator waiting for root updates", "sleep", sleep, "tree", minTree.loc.domain) + timeout := it.c.clock.NewTimer(sleep) + defer timeout.Stop() + select { + case <-timeout.C(): + return true + case <-it.ctx.Done(): + return false // Iterator was closed. + } } // rebuildTrees rebuilds the 'trees' map. diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index 6a6705abf2..741bee4230 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -231,6 +231,53 @@ func TestIteratorRootRecheckOnFail(t *testing.T) { checkIterator(t, it, nodes) } +// This test checks that the iterator works correctly when the tree is initially empty. +func TestIteratorEmptyTree(t *testing.T) { + var ( + clock = new(mclock.Simulated) + nodes = testNodes(nodesSeed1, 1) + resolver = newMapResolver() + c = NewClient(Config{ + Resolver: resolver, + Logger: testlog.Logger(t, log.LvlTrace), + RecheckInterval: 20 * time.Minute, + RateLimit: 500, + }) + ) + c.clock = clock + tree1, url := makeTestTree("n", nil, nil) + tree2, _ := makeTestTree("n", nodes, nil) + resolver.add(tree1.ToTXT("n")) + + // Start the iterator. + node := make(chan *enode.Node) + it, err := c.NewIterator(url) + if err != nil { + t.Fatal(err) + } + go func() { + it.Next() + node <- it.Node() + }() + + // Wait for the client to get stuck in waitForRootUpdates. + clock.WaitForTimers(1) + + // Now update the root. + resolver.add(tree2.ToTXT("n")) + + // Wait for it to pick up the root change. + clock.Run(c.cfg.RecheckInterval) + select { + case n := <-node: + if n.ID() != nodes[0].ID() { + t.Fatalf("wrong node returned") + } + case <-time.After(5 * time.Second): + t.Fatal("it.Next() did not unblock within 5s of real time") + } +} + // updateSomeNodes applies ENR updates to some of the given nodes. func updateSomeNodes(keySeed int64, nodes []*enode.Node) { keys := testKeys(nodesSeed1, len(nodes)) diff --git a/p2p/dnsdisc/sync.go b/p2p/dnsdisc/sync.go index 36f02acba6..073547c90d 100644 --- a/p2p/dnsdisc/sync.go +++ b/p2p/dnsdisc/sync.go @@ -25,9 +25,9 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" ) -const ( - rootRecheckFailCount = 5 // update root if this many leaf requests fail -) +// This is the number of consecutive leaf requests that may fail before +// we consider re-resolving the tree root. +const rootRecheckFailCount = 5 // clientTree is a full tree being synced. type clientTree struct { @@ -89,13 +89,22 @@ func (ct *clientTree) syncRandom(ctx context.Context) (n *enode.Node, err error) ct.gcLinks() // Sync next random entry in ENR tree. Once every node has been visited, we simply - // start over. This is fine because entries are cached. + // start over. This is fine because entries are cached internally by the client LRU + // also by DNS resolvers. if ct.enrs.done() { ct.enrs = newSubtreeSync(ct.c, ct.loc, ct.root.eroot, false) } return ct.syncNextRandomENR(ctx) } +// canSyncRandom checks if any meaningful action can be performed by syncRandom. +func (ct *clientTree) canSyncRandom() bool { + // Note: the check for non-zero leaf count is very important here. + // If we're done syncing all nodes, and no leaves were found, the tree + // is empty and we can't use it for sync. + return ct.rootUpdateDue() || !ct.links.done() || !ct.enrs.done() || ct.enrs.leaves != 0 +} + // gcLinks removes outdated links from the global link cache. GC runs once // when the link sync finishes. func (ct *clientTree) gcLinks() { @@ -184,10 +193,14 @@ func (ct *clientTree) updateRoot(ctx context.Context) error { // rootUpdateDue returns true when a root update is needed. func (ct *clientTree) rootUpdateDue() bool { tooManyFailures := ct.leafFailCount > rootRecheckFailCount - scheduledCheck := ct.c.clock.Now().Sub(ct.lastRootCheck) > ct.c.cfg.RecheckInterval + scheduledCheck := ct.c.clock.Now() >= ct.nextScheduledRootCheck() return ct.root == nil || tooManyFailures || scheduledCheck } +func (ct *clientTree) nextScheduledRootCheck() mclock.AbsTime { + return ct.lastRootCheck.Add(ct.c.cfg.RecheckInterval) +} + // slowdownRootUpdate applies a delay to root resolution if is tried // too frequently. This avoids busy polling when the client is offline. // Returns true if the timeout passed, false if sync was canceled. @@ -218,10 +231,11 @@ type subtreeSync struct { root string missing []string // missing tree node hashes link bool // true if this sync is for the link tree + leaves int // counter of synced leaves } func newSubtreeSync(c *Client, loc *linkEntry, root string, link bool) *subtreeSync { - return &subtreeSync{c, loc, root, []string{root}, link} + return &subtreeSync{c, loc, root, []string{root}, link, 0} } func (ts *subtreeSync) done() bool { @@ -253,10 +267,12 @@ func (ts *subtreeSync) resolveNext(ctx context.Context, hash string) (entry, err if ts.link { return nil, errENRInLinkTree } + ts.leaves++ case *linkEntry: if !ts.link { return nil, errLinkInENRTree } + ts.leaves++ case *branchEntry: ts.missing = append(ts.missing, e.children...) } From c027507e036683f555f63baa4cd02a81696fea6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 19 Feb 2021 14:44:16 +0100 Subject: [PATCH 145/709] les: renamed lespay to vflux (#22347) --- les/api.go | 6 ++-- les/client.go | 16 +++++------ les/clientpool.go | 26 ++++++++--------- les/clientpool_test.go | 28 +++++++++---------- les/peer.go | 20 ++++++------- les/protocol.go | 14 +++++----- les/server.go | 8 +++--- les/server_handler.go | 4 +-- les/serverpool.go | 28 +++++++++---------- les/serverpool_test.go | 8 +++--- les/{lespay => vflux}/client/api.go | 2 +- les/{lespay => vflux}/client/fillset.go | 0 les/{lespay => vflux}/client/fillset_test.go | 0 les/{lespay => vflux}/client/queueiterator.go | 0 .../client/queueiterator_test.go | 0 les/{lespay => vflux}/client/requestbasket.go | 0 .../client/requestbasket_test.go | 0 les/{lespay => vflux}/client/timestats.go | 0 .../client/timestats_test.go | 0 les/{lespay => vflux}/client/valuetracker.go | 0 .../client/valuetracker_test.go | 0 les/{lespay => vflux}/client/wrsiterator.go | 0 .../client/wrsiterator_test.go | 0 les/{lespay => vflux}/server/balance.go | 0 les/{lespay => vflux}/server/balance_test.go | 0 .../server/balance_tracker.go | 0 les/{lespay => vflux}/server/clientdb.go | 0 les/{lespay => vflux}/server/clientdb_test.go | 0 les/{lespay => vflux}/server/prioritypool.go | 0 .../server/prioritypool_test.go | 0 30 files changed, 80 insertions(+), 80 deletions(-) rename les/{lespay => vflux}/client/api.go (98%) rename les/{lespay => vflux}/client/fillset.go (100%) rename les/{lespay => vflux}/client/fillset_test.go (100%) rename les/{lespay => vflux}/client/queueiterator.go (100%) rename les/{lespay => vflux}/client/queueiterator_test.go (100%) rename les/{lespay => vflux}/client/requestbasket.go (100%) rename les/{lespay => vflux}/client/requestbasket_test.go (100%) rename les/{lespay => vflux}/client/timestats.go (100%) rename les/{lespay => vflux}/client/timestats_test.go (100%) rename les/{lespay => vflux}/client/valuetracker.go (100%) rename les/{lespay => vflux}/client/valuetracker_test.go (100%) rename les/{lespay => vflux}/client/wrsiterator.go (100%) rename les/{lespay => vflux}/client/wrsiterator_test.go (100%) rename les/{lespay => vflux}/server/balance.go (100%) rename les/{lespay => vflux}/server/balance_test.go (100%) rename les/{lespay => vflux}/server/balance_tracker.go (100%) rename les/{lespay => vflux}/server/clientdb.go (100%) rename les/{lespay => vflux}/server/clientdb_test.go (100%) rename les/{lespay => vflux}/server/prioritypool.go (100%) rename les/{lespay => vflux}/server/prioritypool_test.go (100%) diff --git a/les/api.go b/les/api.go index 66d133b854..6491c4dcc4 100644 --- a/les/api.go +++ b/les/api.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/mclock" - lps "github.com/ethereum/go-ethereum/les/lespay/server" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/p2p/enode" ) @@ -37,7 +37,7 @@ var ( // PrivateLightServerAPI provides an API to access the LES light server. type PrivateLightServerAPI struct { server *LesServer - defaultPosFactors, defaultNegFactors lps.PriceFactors + defaultPosFactors, defaultNegFactors vfs.PriceFactors } // NewPrivateLightServerAPI creates a new LES light server API. @@ -107,7 +107,7 @@ func (api *PrivateLightServerAPI) clientInfo(c *clientInfo) map[string]interface // setParams either sets the given parameters for a single connected client (if specified) // or the default parameters applicable to clients connected in the future -func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientInfo, posFactors, negFactors *lps.PriceFactors) (updateFactors bool, err error) { +func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientInfo, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { defParams := client == nil for name, value := range params { errValue := func() error { diff --git a/les/client.go b/les/client.go index 1b26e9a9b5..9f0afc96c5 100644 --- a/les/client.go +++ b/les/client.go @@ -36,7 +36,7 @@ import ( "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" - lpc "github.com/ethereum/go-ethereum/les/lespay/client" + vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -58,7 +58,7 @@ type LightEthereum struct { txPool *light.TxPool blockchain *light.LightChain serverPool *serverPool - valueTracker *lpc.ValueTracker + valueTracker *vfc.ValueTracker dialCandidates enode.Iterator pruner *pruner @@ -108,7 +108,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), - valueTracker: lpc.NewValueTracker(lespayDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), + valueTracker: vfc.NewValueTracker(lespayDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), p2pServer: stack.Server(), p2pConfig: &stack.Config().P2P, } @@ -193,18 +193,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { } // vtSubscription implements serverPeerSubscriber -type vtSubscription lpc.ValueTracker +type vtSubscription vfc.ValueTracker // registerPeer implements serverPeerSubscriber func (v *vtSubscription) registerPeer(p *serverPeer) { - vt := (*lpc.ValueTracker)(v) + vt := (*vfc.ValueTracker)(v) p.setValueTracker(vt, vt.Register(p.ID())) p.updateVtParams() } // unregisterPeer implements serverPeerSubscriber func (v *vtSubscription) unregisterPeer(p *serverPeer) { - vt := (*lpc.ValueTracker)(v) + vt := (*vfc.ValueTracker)(v) vt.Unregister(p.ID()) p.setValueTracker(nil, nil) } @@ -263,9 +263,9 @@ func (s *LightEthereum) APIs() []rpc.API { Service: NewPrivateLightAPI(&s.lesCommons), Public: false, }, { - Namespace: "lespay", + Namespace: "vflux", Version: "1.0", - Service: lpc.NewPrivateClientAPI(s.valueTracker), + Service: vfc.NewPrivateClientAPI(s.valueTracker), Public: false, }, }...) diff --git a/les/clientpool.go b/les/clientpool.go index da0db6e622..96c0f0f99e 100644 --- a/les/clientpool.go +++ b/les/clientpool.go @@ -23,8 +23,8 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" - lps "github.com/ethereum/go-ethereum/les/lespay/server" "github.com/ethereum/go-ethereum/les/utils" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -64,17 +64,17 @@ const ( // and negative banalce. Boeth positive balance and negative balance will decrease // exponentially. If the balance is low enough, then the record will be dropped. type clientPool struct { - lps.BalanceTrackerSetup - lps.PriorityPoolSetup + vfs.BalanceTrackerSetup + vfs.PriorityPoolSetup lock sync.Mutex clock mclock.Clock closed bool removePeer func(enode.ID) ns *nodestate.NodeStateMachine - pp *lps.PriorityPool - bt *lps.BalanceTracker + pp *vfs.PriorityPool + bt *vfs.BalanceTracker - defaultPosFactors, defaultNegFactors lps.PriceFactors + defaultPosFactors, defaultNegFactors vfs.PriceFactors posExpTC, negExpTC uint64 minCap uint64 // The minimal capacity value allowed for any client connectedBias time.Duration @@ -101,7 +101,7 @@ type clientInfo struct { peer clientPoolPeer connected, priority bool connectedAt mclock.AbsTime - balance *lps.NodeBalance + balance *vfs.NodeBalance } // newClientPool creates a new client pool @@ -115,8 +115,8 @@ func newClientPool(ns *nodestate.NodeStateMachine, lespayDb ethdb.Database, minC connectedBias: connectedBias, removePeer: removePeer, } - pool.bt = lps.NewBalanceTracker(ns, balanceTrackerSetup, lespayDb, clock, &utils.Expirer{}, &utils.Expirer{}) - pool.pp = lps.NewPriorityPool(ns, priorityPoolSetup, clock, minCap, connectedBias, 4) + pool.bt = vfs.NewBalanceTracker(ns, balanceTrackerSetup, lespayDb, clock, &utils.Expirer{}, &utils.Expirer{}) + pool.pp = vfs.NewPriorityPool(ns, priorityPoolSetup, clock, minCap, connectedBias, 4) // set default expiration constants used by tests // Note: server overwrites this if token sale is active @@ -221,7 +221,7 @@ func (f *clientPool) connect(peer clientPoolPeer) (uint64, error) { } f.ns.SetField(node, clientInfoField, c) f.ns.SetField(node, connAddressField, freeID) - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*lps.NodeBalance); c.balance == nil { + if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance == nil { f.disconnect(peer) return 0, nil } @@ -266,7 +266,7 @@ func (f *clientPool) disconnectNode(node *enode.Node) { } // setDefaultFactors sets the default price factors applied to subsequently connected clients -func (f *clientPool) setDefaultFactors(posFactors, negFactors lps.PriceFactors) { +func (f *clientPool) setDefaultFactors(posFactors, negFactors vfs.PriceFactors) { f.lock.Lock() defer f.lock.Unlock() @@ -305,7 +305,7 @@ func (f *clientPool) setCapacity(node *enode.Node, freeID string, capacity uint6 c = &clientInfo{node: node} f.ns.SetField(node, clientInfoField, c) f.ns.SetField(node, connAddressField, freeID) - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*lps.NodeBalance); c.balance == nil { + if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance == nil { log.Error("BalanceField is missing", "node", node.ID()) return 0, fmt.Errorf("BalanceField of %064x is missing", node.ID()) } @@ -371,7 +371,7 @@ func (f *clientPool) forClients(ids []enode.ID, cb func(client *clientInfo)) { c = &clientInfo{node: node} f.ns.SetField(node, clientInfoField, c) f.ns.SetField(node, connAddressField, "") - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*lps.NodeBalance); c.balance != nil { + if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance != nil { cb(c) } else { log.Error("BalanceField is missing") diff --git a/les/clientpool_test.go b/les/clientpool_test.go index b1c38d374c..5cff010409 100644 --- a/les/clientpool_test.go +++ b/les/clientpool_test.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/rawdb" - lps "github.com/ethereum/go-ethereum/les/lespay/server" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nodestate" @@ -100,7 +100,7 @@ func getBalance(pool *clientPool, p *poolTestPeer) (pos, neg uint64) { if temp { pool.ns.SetField(p.node, connAddressField, p.freeClientId()) } - n, _ := pool.ns.GetField(p.node, pool.BalanceField).(*lps.NodeBalance) + n, _ := pool.ns.GetField(p.node, pool.BalanceField).(*vfs.NodeBalance) pos, neg = n.GetBalance() if temp { pool.ns.SetField(p.node, connAddressField, nil) @@ -138,7 +138,7 @@ func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, rando pool.ns.Start() pool.setLimits(activeLimit, uint64(activeLimit)) - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) // pool should accept new peers up to its connected limit for i := 0; i < activeLimit; i++ { @@ -243,7 +243,7 @@ func TestConnectPaidClient(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) // Add balance for an external client and mark it as paid client addBalance(pool, newPoolTestPeer(0, nil).node.ID(), int64(time.Minute)) @@ -259,7 +259,7 @@ func TestConnectPaidClientToSmallPool(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) // Add balance for an external client and mark it as paid client addBalance(pool, newPoolTestPeer(0, nil).node.ID(), int64(time.Minute)) @@ -278,7 +278,7 @@ func TestConnectPaidClientToFullPool(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { addBalance(pool, newPoolTestPeer(i, nil).node.ID(), int64(time.Second*20)) @@ -309,7 +309,7 @@ func TestPaidClientKickedOut(t *testing.T) { pool.bt.SetExpirationTCs(0, 0) defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { addBalance(pool, newPoolTestPeer(i, kickedCh).node.ID(), 10000000000) // 10 second allowance @@ -339,7 +339,7 @@ func TestConnectFreeClient(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) if cap, _ := pool.connect(newPoolTestPeer(0, nil)); cap == 0 { t.Fatalf("Failed to connect free client") } @@ -356,7 +356,7 @@ func TestConnectFreeClientToFullPool(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { pool.connect(newPoolTestPeer(i, nil)) @@ -386,7 +386,7 @@ func TestFreeClientKickedOut(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { pool.connect(newPoolTestPeer(i, kicked)) @@ -428,7 +428,7 @@ func TestPositiveBalanceCalculation(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) addBalance(pool, newPoolTestPeer(0, kicked).node.ID(), int64(time.Minute*3)) testPriorityConnect(t, pool, newPoolTestPeer(0, kicked), 10, true) @@ -452,7 +452,7 @@ func TestDowngradePriorityClient(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) p := newPoolTestPeer(0, kicked) addBalance(pool, p.node.ID(), int64(time.Minute)) @@ -487,7 +487,7 @@ func TestNegativeBalanceCalculation(t *testing.T) { pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { pool.connect(newPoolTestPeer(i, nil)) @@ -564,7 +564,7 @@ func TestInactiveClient(t *testing.T) { if p2.cap != 0 { t.Fatalf("Failed to deactivate peer #2") } - pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}) + pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}) p4 := newPoolTestPeer(4, nil) addBalance(pool, p4.node.ID(), 1500*int64(time.Second)) // p1: 1000 p2: 500 p3: 2000 p4: 1500 diff --git a/les/peer.go b/les/peer.go index 0e2ed52c12..52ab506368 100644 --- a/les/peer.go +++ b/les/peer.go @@ -32,9 +32,9 @@ import ( "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/les/flowcontrol" - lpc "github.com/ethereum/go-ethereum/les/lespay/client" - lps "github.com/ethereum/go-ethereum/les/lespay/server" "github.com/ethereum/go-ethereum/les/utils" + vfc "github.com/ethereum/go-ethereum/les/vflux/client" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -349,8 +349,8 @@ type serverPeer struct { fcServer *flowcontrol.ServerNode // Client side mirror token bucket. vtLock sync.Mutex - valueTracker *lpc.ValueTracker - nodeValueTracker *lpc.NodeValueTracker + valueTracker *vfc.ValueTracker + nodeValueTracker *vfc.NodeValueTracker sentReqs map[uint64]sentReqEntry // Statistics @@ -676,7 +676,7 @@ func (p *serverPeer) Handshake(genesis common.Hash, forkid forkid.ID, forkFilter // setValueTracker sets the value tracker references for connected servers. Note that the // references should be removed upon disconnection by setValueTracker(nil, nil). -func (p *serverPeer) setValueTracker(vt *lpc.ValueTracker, nvt *lpc.NodeValueTracker) { +func (p *serverPeer) setValueTracker(vt *vfc.ValueTracker, nvt *vfc.NodeValueTracker) { p.vtLock.Lock() p.valueTracker = vt p.nodeValueTracker = nvt @@ -739,17 +739,17 @@ func (p *serverPeer) answeredRequest(id uint64) { return } var ( - vtReqs [2]lpc.ServedRequest + vtReqs [2]vfc.ServedRequest reqCount int ) m := requestMapping[e.reqType] if m.rest == -1 || e.amount <= 1 { reqCount = 1 - vtReqs[0] = lpc.ServedRequest{ReqType: uint32(m.first), Amount: e.amount} + vtReqs[0] = vfc.ServedRequest{ReqType: uint32(m.first), Amount: e.amount} } else { reqCount = 2 - vtReqs[0] = lpc.ServedRequest{ReqType: uint32(m.first), Amount: 1} - vtReqs[1] = lpc.ServedRequest{ReqType: uint32(m.rest), Amount: e.amount - 1} + vtReqs[0] = vfc.ServedRequest{ReqType: uint32(m.first), Amount: 1} + vtReqs[1] = vfc.ServedRequest{ReqType: uint32(m.rest), Amount: e.amount - 1} } dt := time.Duration(mclock.Now() - e.at) vt.Served(nvt, vtReqs[:reqCount], dt) @@ -765,7 +765,7 @@ type clientPeer struct { responseLock sync.Mutex responseCount uint64 // Counter to generate an unique id for request processing. - balance *lps.NodeBalance + balance *vfs.NodeBalance // invalidLock is used for protecting invalidCount. invalidLock sync.RWMutex diff --git a/les/protocol.go b/les/protocol.go index 9eb6ec7471..909d25d375 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - lpc "github.com/ethereum/go-ethereum/les/lespay/client" + vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" ) @@ -89,7 +89,7 @@ type requestInfo struct { refBasketFirst, refBasketRest float64 } -// reqMapping maps an LES request to one or two lespay service vector entries. +// reqMapping maps an LES request to one or two vflux service vector entries. // If rest != -1 and the request type is used with amounts larger than one then the // first one of the multi-request is mapped to first while the rest is mapped to rest. type reqMapping struct { @@ -98,7 +98,7 @@ type reqMapping struct { var ( // requests describes the available LES request types and their initializing amounts - // in the lespay/client.ValueTracker reference basket. Initial values are estimates + // in the vfc.ValueTracker reference basket. Initial values are estimates // based on the same values as the server's default cost estimates (reqAvgTimeCost). requests = map[uint64]requestInfo{ GetBlockHeadersMsg: {"GetBlockHeaders", MaxHeaderFetch, 10, 1000}, @@ -110,25 +110,25 @@ var ( SendTxV2Msg: {"SendTxV2", MaxTxSend, 1, 0}, GetTxStatusMsg: {"GetTxStatus", MaxTxStatus, 10, 0}, } - requestList []lpc.RequestInfo + requestList []vfc.RequestInfo requestMapping map[uint32]reqMapping ) -// init creates a request list and mapping between protocol message codes and lespay +// init creates a request list and mapping between protocol message codes and vflux // service vector indices. func init() { requestMapping = make(map[uint32]reqMapping) for code, req := range requests { cost := reqAvgTimeCost[code] rm := reqMapping{len(requestList), -1} - requestList = append(requestList, lpc.RequestInfo{ + requestList = append(requestList, vfc.RequestInfo{ Name: req.name + ".first", InitAmount: req.refBasketFirst, InitValue: float64(cost.baseCost + cost.reqCost), }) if req.refBasketRest != 0 { rm.rest = len(requestList) - requestList = append(requestList, lpc.RequestInfo{ + requestList = append(requestList, vfc.RequestInfo{ Name: req.name + ".rest", InitAmount: req.refBasketRest, InitValue: float64(cost.reqCost), diff --git a/les/server.go b/les/server.go index 44495eb311..351a53f690 100644 --- a/les/server.go +++ b/les/server.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" - lps "github.com/ethereum/go-ethereum/les/lespay/server" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -43,8 +43,8 @@ var ( clientPeerField = serverSetup.NewField("clientPeer", reflect.TypeOf(&clientPeer{})) clientInfoField = serverSetup.NewField("clientInfo", reflect.TypeOf(&clientInfo{})) connAddressField = serverSetup.NewField("connAddr", reflect.TypeOf("")) - balanceTrackerSetup = lps.NewBalanceTrackerSetup(serverSetup) - priorityPoolSetup = lps.NewPriorityPoolSetup(serverSetup) + balanceTrackerSetup = vfs.NewBalanceTrackerSetup(serverSetup) + priorityPoolSetup = vfs.NewPriorityPoolSetup(serverSetup) ) func init() { @@ -137,7 +137,7 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les } srv.fcManager.SetCapacityLimits(srv.minCapacity, srv.maxCapacity, srv.minCapacity*2) srv.clientPool = newClientPool(ns, srv.chainDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient) - srv.clientPool.setDefaultFactors(lps.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}) + srv.clientPool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}) checkpoint := srv.latestLocalCheckpoint() if !checkpoint.Empty() { diff --git a/les/server_handler.go b/les/server_handler.go index bec4206e2b..fd81e273c9 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - lps "github.com/ethereum/go-ethereum/les/lespay/server" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -169,7 +169,7 @@ func (h *serverHandler) handle(p *clientPeer) error { p.Log().Debug("Light Ethereum peer rejected", "err", errFullClientPool) return errFullClientPool } - p.balance, _ = h.server.ns.GetField(p.Node(), h.server.clientPool.BalanceField).(*lps.NodeBalance) + p.balance, _ = h.server.ns.GetField(p.Node(), h.server.clientPool.BalanceField).(*vfs.NodeBalance) if p.balance == nil { return p2p.DiscRequested } diff --git a/les/serverpool.go b/les/serverpool.go index ac87acf6da..977579988e 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -26,8 +26,8 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" - lpc "github.com/ethereum/go-ethereum/les/lespay/client" "github.com/ethereum/go-ethereum/les/utils" + vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -58,23 +58,23 @@ type serverPool struct { db ethdb.KeyValueStore ns *nodestate.NodeStateMachine - vt *lpc.ValueTracker + vt *vfc.ValueTracker mixer *enode.FairMix mixSources []enode.Iterator dialIterator enode.Iterator validSchemes enr.IdentityScheme trustedURLs []string - fillSet *lpc.FillSet + fillSet *vfc.FillSet queryFails uint32 timeoutLock sync.RWMutex timeout time.Duration - timeWeights lpc.ResponseTimeWeights + timeWeights vfc.ResponseTimeWeights timeoutRefreshed mclock.AbsTime } // nodeHistory keeps track of dial costs which determine node weight together with the -// service value calculated by lpc.ValueTracker. +// service value calculated by vfc.ValueTracker. type nodeHistory struct { dialCost utils.ExpiredValue redialWaitStart, redialWaitEnd int64 // unix time (seconds) @@ -127,11 +127,11 @@ var ( }, ) sfiNodeWeight = serverPoolSetup.NewField("nodeWeight", reflect.TypeOf(uint64(0))) - sfiConnectedStats = serverPoolSetup.NewField("connectedStats", reflect.TypeOf(lpc.ResponseTimeStats{})) + sfiConnectedStats = serverPoolSetup.NewField("connectedStats", reflect.TypeOf(vfc.ResponseTimeStats{})) ) // newServerPool creates a new server pool -func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string) *serverPool { +func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *vfc.ValueTracker, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string) *serverPool { s := &serverPool{ db: db, clock: clock, @@ -143,8 +143,8 @@ func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, m } s.recalTimeout() s.mixer = enode.NewFairMix(mixTimeout) - knownSelector := lpc.NewWrsIterator(s.ns, sfHasValue, sfDisableSelection, sfiNodeWeight) - alwaysConnect := lpc.NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) + knownSelector := vfc.NewWrsIterator(s.ns, sfHasValue, sfDisableSelection, sfiNodeWeight) + alwaysConnect := vfc.NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) s.mixSources = append(s.mixSources, knownSelector) s.mixSources = append(s.mixSources, alwaysConnect) @@ -183,7 +183,7 @@ func (s *serverPool) addSource(source enode.Iterator) { // Nodes that are filtered out and does not appear on the output iterator are put back // into redialWait state. func (s *serverPool) addPreNegFilter(input enode.Iterator, query queryFunc) enode.Iterator { - s.fillSet = lpc.NewFillSet(s.ns, input, sfQueried) + s.fillSet = vfc.NewFillSet(s.ns, input, sfQueried) s.ns.SubscribeState(sfQueried, func(n *enode.Node, oldState, newState nodestate.Flags) { if newState.Equals(sfQueried) { fails := atomic.LoadUint32(&s.queryFails) @@ -221,7 +221,7 @@ func (s *serverPool) addPreNegFilter(input enode.Iterator, query queryFunc) enod }() } }) - return lpc.NewQueueIterator(s.ns, sfCanDial, nodestate.Flags{}, false, func(waiting bool) { + return vfc.NewQueueIterator(s.ns, sfCanDial, nodestate.Flags{}, false, func(waiting bool) { if waiting { s.fillSet.SetTarget(preNegLimit) } else { @@ -330,7 +330,7 @@ func (s *serverPool) recalTimeout() { s.timeoutLock.Lock() if s.timeout != timeout { s.timeout = timeout - s.timeWeights = lpc.TimeoutWeights(s.timeout) + s.timeWeights = vfc.TimeoutWeights(s.timeout) suggestedTimeoutGauge.Update(int64(s.timeout / time.Millisecond)) totalValueGauge.Update(int64(rts.Value(s.timeWeights, s.vt.StatsExpFactor()))) @@ -349,7 +349,7 @@ func (s *serverPool) getTimeout() time.Duration { // getTimeoutAndWeight returns the recommended request timeout as well as the // response time weight which is necessary to calculate service value. -func (s *serverPool) getTimeoutAndWeight() (time.Duration, lpc.ResponseTimeWeights) { +func (s *serverPool) getTimeoutAndWeight() (time.Duration, vfc.ResponseTimeWeights) { s.recalTimeout() s.timeoutLock.RLock() defer s.timeoutLock.RUnlock() @@ -381,7 +381,7 @@ func (s *serverPool) serviceValue(node *enode.Node) (sessionValue, totalValue fl expFactor := s.vt.StatsExpFactor() totalValue = currentStats.Value(timeWeights, expFactor) - if connStats, ok := s.ns.GetField(node, sfiConnectedStats).(lpc.ResponseTimeStats); ok { + if connStats, ok := s.ns.GetField(node, sfiConnectedStats).(vfc.ResponseTimeStats); ok { diff := currentStats diff.SubStats(&connStats) sessionValue = diff.Value(timeWeights, expFactor) diff --git a/les/serverpool_test.go b/les/serverpool_test.go index 3b7ae65d5d..5c8ae56f6c 100644 --- a/les/serverpool_test.go +++ b/les/serverpool_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" - lpc "github.com/ethereum/go-ethereum/les/lespay/client" + vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -55,7 +55,7 @@ type serverPoolTest struct { clock *mclock.Simulated quit chan struct{} preNeg, preNegFail bool - vt *lpc.ValueTracker + vt *vfc.ValueTracker sp *serverPool input enode.Iterator testNodes []spTestNode @@ -144,7 +144,7 @@ func (s *serverPoolTest) start() { } } - s.vt = lpc.NewValueTracker(s.db, s.clock, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)) + s.vt = vfc.NewValueTracker(s.db, s.clock, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)) s.sp = newServerPool(s.db, []byte("serverpool:"), s.vt, 0, testQuery, s.clock, s.trusted) s.sp.addSource(s.input) s.sp.validSchemes = enode.ValidSchemesForTesting @@ -224,7 +224,7 @@ func (s *serverPoolTest) run() { n.peer = &serverPeer{peerCommons: peerCommons{Peer: p2p.NewPeer(id, "", nil)}} s.sp.registerPeer(n.peer) if n.service { - s.vt.Served(s.vt.GetNode(id), []lpc.ServedRequest{{ReqType: 0, Amount: 100}}, 0) + s.vt.Served(s.vt.GetNode(id), []vfc.ServedRequest{{ReqType: 0, Amount: 100}}, 0) } } } diff --git a/les/lespay/client/api.go b/les/vflux/client/api.go similarity index 98% rename from les/lespay/client/api.go rename to les/vflux/client/api.go index 5ad6ffd77e..135273ef96 100644 --- a/les/lespay/client/api.go +++ b/les/vflux/client/api.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" ) -// PrivateClientAPI implements the lespay client side API +// PrivateClientAPI implements the vflux client side API type PrivateClientAPI struct { vt *ValueTracker } diff --git a/les/lespay/client/fillset.go b/les/vflux/client/fillset.go similarity index 100% rename from les/lespay/client/fillset.go rename to les/vflux/client/fillset.go diff --git a/les/lespay/client/fillset_test.go b/les/vflux/client/fillset_test.go similarity index 100% rename from les/lespay/client/fillset_test.go rename to les/vflux/client/fillset_test.go diff --git a/les/lespay/client/queueiterator.go b/les/vflux/client/queueiterator.go similarity index 100% rename from les/lespay/client/queueiterator.go rename to les/vflux/client/queueiterator.go diff --git a/les/lespay/client/queueiterator_test.go b/les/vflux/client/queueiterator_test.go similarity index 100% rename from les/lespay/client/queueiterator_test.go rename to les/vflux/client/queueiterator_test.go diff --git a/les/lespay/client/requestbasket.go b/les/vflux/client/requestbasket.go similarity index 100% rename from les/lespay/client/requestbasket.go rename to les/vflux/client/requestbasket.go diff --git a/les/lespay/client/requestbasket_test.go b/les/vflux/client/requestbasket_test.go similarity index 100% rename from les/lespay/client/requestbasket_test.go rename to les/vflux/client/requestbasket_test.go diff --git a/les/lespay/client/timestats.go b/les/vflux/client/timestats.go similarity index 100% rename from les/lespay/client/timestats.go rename to les/vflux/client/timestats.go diff --git a/les/lespay/client/timestats_test.go b/les/vflux/client/timestats_test.go similarity index 100% rename from les/lespay/client/timestats_test.go rename to les/vflux/client/timestats_test.go diff --git a/les/lespay/client/valuetracker.go b/les/vflux/client/valuetracker.go similarity index 100% rename from les/lespay/client/valuetracker.go rename to les/vflux/client/valuetracker.go diff --git a/les/lespay/client/valuetracker_test.go b/les/vflux/client/valuetracker_test.go similarity index 100% rename from les/lespay/client/valuetracker_test.go rename to les/vflux/client/valuetracker_test.go diff --git a/les/lespay/client/wrsiterator.go b/les/vflux/client/wrsiterator.go similarity index 100% rename from les/lespay/client/wrsiterator.go rename to les/vflux/client/wrsiterator.go diff --git a/les/lespay/client/wrsiterator_test.go b/les/vflux/client/wrsiterator_test.go similarity index 100% rename from les/lespay/client/wrsiterator_test.go rename to les/vflux/client/wrsiterator_test.go diff --git a/les/lespay/server/balance.go b/les/vflux/server/balance.go similarity index 100% rename from les/lespay/server/balance.go rename to les/vflux/server/balance.go diff --git a/les/lespay/server/balance_test.go b/les/vflux/server/balance_test.go similarity index 100% rename from les/lespay/server/balance_test.go rename to les/vflux/server/balance_test.go diff --git a/les/lespay/server/balance_tracker.go b/les/vflux/server/balance_tracker.go similarity index 100% rename from les/lespay/server/balance_tracker.go rename to les/vflux/server/balance_tracker.go diff --git a/les/lespay/server/clientdb.go b/les/vflux/server/clientdb.go similarity index 100% rename from les/lespay/server/clientdb.go rename to les/vflux/server/clientdb.go diff --git a/les/lespay/server/clientdb_test.go b/les/vflux/server/clientdb_test.go similarity index 100% rename from les/lespay/server/clientdb_test.go rename to les/vflux/server/clientdb_test.go diff --git a/les/lespay/server/prioritypool.go b/les/vflux/server/prioritypool.go similarity index 100% rename from les/lespay/server/prioritypool.go rename to les/vflux/server/prioritypool.go diff --git a/les/lespay/server/prioritypool_test.go b/les/vflux/server/prioritypool_test.go similarity index 100% rename from les/lespay/server/prioritypool_test.go rename to les/vflux/server/prioritypool_test.go From ca76db6116b64bb10c83085a70898750668593d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 19 Feb 2021 15:53:05 +0200 Subject: [PATCH 146/709] cmd/utils: disable caching preimages by default --- cmd/utils/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0d7b0e1bf5..d065e02047 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -401,9 +401,9 @@ var ( Name: "cache.noprefetch", Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", } - CachePreimagesFlag = cli.BoolTFlag{ + CachePreimagesFlag = cli.BoolFlag{ Name: "cache.preimages", - Usage: "Enable recording the SHA3/keccak preimages of trie keys (default: true)", + Usage: "Enable recording the SHA3/keccak preimages of trie keys", } // Miner settings MiningEnabledFlag = cli.BoolFlag{ From c5023e1dc56f3ced0e3a24733e533bf962515844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 19 Feb 2021 16:03:17 +0200 Subject: [PATCH 147/709] travis, appveyor, build: bump Go to 1.16 --- .travis.yml | 22 +++++++++++----------- appveyor.yml | 4 ++-- build/checksums.txt | 24 ++++++++++++------------ build/ci.go | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1268c6d657..39a0456c3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: allow_failures: - stage: build os: osx - go: 1.14.x + go: 1.15.x env: - azure-osx - azure-ios @@ -16,7 +16,7 @@ jobs: - stage: lint os: linux dist: xenial - go: 1.15.x + go: 1.16.x env: - lint git: @@ -29,7 +29,7 @@ jobs: if: type = push os: linux dist: xenial - go: 1.15.x + go: 1.16.x env: - ubuntu-ppa - GO111MODULE=on @@ -54,7 +54,7 @@ jobs: os: linux dist: xenial sudo: required - go: 1.15.x + go: 1.16.x env: - azure-linux - GO111MODULE=on @@ -91,7 +91,7 @@ jobs: dist: xenial services: - docker - go: 1.15.x + go: 1.16.x env: - azure-linux-mips - GO111MODULE=on @@ -139,7 +139,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://dl.google.com/go/go1.15.5.linux-amd64.tar.gz | tar -xz + - curl https://dl.google.com/go/go1.16.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go @@ -157,7 +157,7 @@ jobs: - stage: build if: type = push os: osx - go: 1.15.x + go: 1.16.x env: - azure-osx - azure-ios @@ -189,7 +189,7 @@ jobs: os: linux arch: amd64 dist: xenial - go: 1.15.x + go: 1.16.x env: - GO111MODULE=on script: @@ -200,7 +200,7 @@ jobs: os: linux arch: arm64 dist: xenial - go: 1.15.x + go: 1.16.x env: - GO111MODULE=on script: @@ -209,7 +209,7 @@ jobs: - stage: build os: linux dist: xenial - go: 1.14.x + go: 1.15.x env: - GO111MODULE=on script: @@ -220,7 +220,7 @@ jobs: if: type = cron os: linux dist: xenial - go: 1.15.x + go: 1.16.x env: - azure-purge - GO111MODULE=on diff --git a/appveyor.yml b/appveyor.yml index 2bf67d4568..052280be15 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,8 +24,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://dl.google.com/go/go1.15.5.windows-%GETH_ARCH%.zip - - 7z x go1.15.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://dl.google.com/go/go1.16.windows-%GETH_ARCH%.zip + - 7z x go1.16.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/build/checksums.txt b/build/checksums.txt index a7a6a657e9..d5bd4d0cd3 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,17 +1,17 @@ # This file contains sha256 checksums of optional build dependencies. -890bba73c5e2b19ffb1180e385ea225059eb008eb91b694875dd86ea48675817 go1.15.6.src.tar.gz -940a73b45993a3bae5792cf324140dded34af97c548af4864d22fd6d49f3bd9f go1.15.6.darwin-amd64.tar.gz -ad187f02158b9a9013ef03f41d14aa69c402477f178825a3940280814bcbb755 go1.15.6.linux-386.tar.gz -3918e6cc85e7eaaa6f859f1bdbaac772e7a825b0eb423c63d3ae68b21f84b844 go1.15.6.linux-amd64.tar.gz -f87515b9744154ffe31182da9341d0a61eb0795551173d242c8cad209239e492 go1.15.6.linux-arm64.tar.gz -40ba9a57764e374195018ef37c38a5fbac9bbce908eab436370631a84bfc5788 go1.15.6.linux-armv6l.tar.gz -5872eff6746a0a5f304272b27cbe9ce186f468454e95749cce01e903fbfc0e17 go1.15.6.windows-386.zip -b7b3808bb072c2bab73175009187fd5a7f20ffe0a31739937003a14c5c4d9006 go1.15.6.windows-amd64.zip -9d9dd5c217c1392f1b2ed5e03e1c71bf4cf8553884e57a38e68fd37fdcfe31a8 go1.15.6.freebsd-386.tar.gz -609f065d855aed5a0b40ef0245aacbcc0b4b7882dc3b1e75ae50576cf25265ee go1.15.6.freebsd-amd64.tar.gz -d4174fc217e749ac049eacc8827df879689f2246ac230d04991ae7df336f7cb2 go1.15.6.linux-ppc64le.tar.gz -839cc6b67687d8bb7cb044e4a9a2eac0c090765cc8ec55ffe714dfb7cd51cf3a go1.15.6.linux-s390x.tar.gz +7688063d55656105898f323d90a79a39c378d86fe89ae192eb3b7fc46347c95a go1.16.src.tar.gz +6000a9522975d116bf76044967d7e69e04e982e9625330d9a539a8b45395f9a8 go1.16.darwin-amd64.tar.gz +ea435a1ac6d497b03e367fdfb74b33e961d813883468080f6e239b3b03bea6aa go1.16.linux-386.tar.gz +013a489ebb3e24ef3d915abe5b94c3286c070dfe0818d5bca8108f1d6e8440d2 go1.16.linux-amd64.tar.gz +3770f7eb22d05e25fbee8fb53c2a4e897da043eb83c69b9a14f8d98562cd8098 go1.16.linux-arm64.tar.gz +d1d9404b1dbd77afa2bdc70934e10fbfcf7d785c372efc29462bb7d83d0a32fd go1.16.linux-armv6l.tar.gz +481492a17d42193d471b93b7a06da3555331bd833b76336afc87be820c48933f go1.16.windows-386.zip +5cc88fa506b3d5c453c54c3ea218fc8dd05d7362ae1de15bb67986b72089ce93 go1.16.windows-amd64.zip +d7d6c70b05a7c2f68b48aab5ab8cb5116b8444c9ddad131673b152e7cff7c726 go1.16.freebsd-386.tar.gz +40b03216f6945fb6883a50604fc7f409a83f62171607229a9c598e701e684f8a go1.16.freebsd-amd64.tar.gz +27a1aaa988e930b7932ce459c8a63ad5b3333b3a06b016d87ff289f2a11aacd6 go1.16.linux-ppc64le.tar.gz +be4c9e4e2cf058efc4e3eb013a760cb989ddc4362f111950c990d1c63b27ccbe go1.16.linux-s390x.tar.gz d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35 golangci-lint-1.27.0-linux-armv7.tar.gz bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint.exe-1.27.0-windows-amd64.zip diff --git a/build/ci.go b/build/ci.go index 73d8961629..756fba8399 100644 --- a/build/ci.go +++ b/build/ci.go @@ -152,7 +152,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.15.6" + dlgoVersion = "1.16" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From 8647233a8ec2a2410a078013ca12c38fdc229866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 19 Feb 2021 15:53:12 +0100 Subject: [PATCH 148/709] les: fix balance expiration (#22343) * les/lespay/server: fix balance expiration and add test * les: move client balances to a new db * les: rename lespayDb to lesDb --- les/client.go | 8 ++-- les/clientpool.go | 4 +- les/commons.go | 2 +- les/server.go | 8 +++- les/vflux/server/balance_test.go | 73 ++++++++++++++++++++++++++++- les/vflux/server/balance_tracker.go | 6 ++- 6 files changed, 92 insertions(+), 9 deletions(-) diff --git a/les/client.go b/les/client.go index 9f0afc96c5..d08c9feba5 100644 --- a/les/client.go +++ b/les/client.go @@ -81,7 +81,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - lespayDb, err := stack.OpenDatabase("lespay", 0, 0, "eth/db/lespay") + lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/les.client") if err != nil { return nil, err } @@ -99,6 +99,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { chainConfig: chainConfig, iConfig: light.DefaultClientIndexerConfig, chainDb: chainDb, + lesDb: lesDb, closeCh: make(chan struct{}), }, peers: peers, @@ -108,13 +109,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), - valueTracker: vfc.NewValueTracker(lespayDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), + valueTracker: vfc.NewValueTracker(lesDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), p2pServer: stack.Server(), p2pConfig: &stack.Config().P2P, } peers.subscribe((*vtSubscription)(leth.valueTracker)) - leth.serverPool = newServerPool(lespayDb, []byte("serverpool:"), leth.valueTracker, time.Second, nil, &mclock.System{}, config.UltraLightServers) + leth.serverPool = newServerPool(lesDb, []byte("serverpool:"), leth.valueTracker, time.Second, nil, &mclock.System{}, config.UltraLightServers) peers.subscribe(leth.serverPool) leth.dialCandidates = leth.serverPool.dialIterator @@ -331,6 +332,7 @@ func (s *LightEthereum) Stop() error { s.eventMux.Stop() rawdb.PopUncleanShutdownMarker(s.chainDb) s.chainDb.Close() + s.lesDb.Close() s.wg.Wait() log.Info("Light ethereum stopped") return nil diff --git a/les/clientpool.go b/les/clientpool.go index 96c0f0f99e..4e1499bf5d 100644 --- a/les/clientpool.go +++ b/les/clientpool.go @@ -105,7 +105,7 @@ type clientInfo struct { } // newClientPool creates a new client pool -func newClientPool(ns *nodestate.NodeStateMachine, lespayDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID)) *clientPool { +func newClientPool(ns *nodestate.NodeStateMachine, lesDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID)) *clientPool { pool := &clientPool{ ns: ns, BalanceTrackerSetup: balanceTrackerSetup, @@ -115,7 +115,7 @@ func newClientPool(ns *nodestate.NodeStateMachine, lespayDb ethdb.Database, minC connectedBias: connectedBias, removePeer: removePeer, } - pool.bt = vfs.NewBalanceTracker(ns, balanceTrackerSetup, lespayDb, clock, &utils.Expirer{}, &utils.Expirer{}) + pool.bt = vfs.NewBalanceTracker(ns, balanceTrackerSetup, lesDb, clock, &utils.Expirer{}, &utils.Expirer{}) pool.pp = vfs.NewPriorityPool(ns, priorityPoolSetup, clock, minCap, connectedBias, 4) // set default expiration constants used by tests diff --git a/les/commons.go b/les/commons.go index a2fce1dc97..d090fc21fc 100644 --- a/les/commons.go +++ b/les/commons.go @@ -51,7 +51,7 @@ type lesCommons struct { config *ethconfig.Config chainConfig *params.ChainConfig iConfig *light.IndexerConfig - chainDb ethdb.Database + chainDb, lesDb ethdb.Database chainReader chainReader chtIndexer, bloomTrieIndexer *core.ChainIndexer oracle *checkpointoracle.CheckpointOracle diff --git a/les/server.go b/les/server.go index 351a53f690..e34647f290 100644 --- a/les/server.go +++ b/les/server.go @@ -85,6 +85,10 @@ type LesServer struct { } func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*LesServer, error) { + lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/les.server") + if err != nil { + return nil, err + } ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup) // Calculate the number of threads used to service the light client // requests based on the user-specified value. @@ -99,6 +103,7 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les chainConfig: e.BlockChain().Config(), iConfig: light.DefaultServerIndexerConfig, chainDb: e.ChainDb(), + lesDb: lesDb, chainReader: e.BlockChain(), chtIndexer: light.NewChtIndexer(e.ChainDb(), nil, params.CHTFrequency, params.HelperTrieProcessConfirmations, true), bloomTrieIndexer: light.NewBloomTrieIndexer(e.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency, true), @@ -136,7 +141,7 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les srv.maxCapacity = totalRecharge } srv.fcManager.SetCapacityLimits(srv.minCapacity, srv.maxCapacity, srv.minCapacity*2) - srv.clientPool = newClientPool(ns, srv.chainDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient) + srv.clientPool = newClientPool(ns, lesDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient) srv.clientPool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}) checkpoint := srv.latestLocalCheckpoint() @@ -222,6 +227,7 @@ func (s *LesServer) Stop() error { // Note, bloom trie indexer is closed by parent bloombits indexer. s.chtIndexer.Close() + s.lesDb.Close() s.wg.Wait() log.Info("Les server stopped") diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 67e1944373..6c817aa26c 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -17,6 +17,7 @@ package server import ( + "math" "math/rand" "reflect" "testing" @@ -69,7 +70,9 @@ func (b *balanceTestSetup) newNode(capacity uint64) *NodeBalance { node := enode.SignNull(&enr.Record{}, enode.ID{}) b.ns.SetState(node, testFlag, nodestate.Flags{}, 0) b.ns.SetField(node, btTestSetup.connAddressField, "") - b.ns.SetField(node, ppTestSetup.CapacityField, capacity) + if capacity != 0 { + b.ns.SetField(node, ppTestSetup.CapacityField, capacity) + } n, _ := b.ns.GetField(node, btTestSetup.BalanceField).(*NodeBalance) return n } @@ -398,3 +401,71 @@ func TestCallback(t *testing.T) { case <-time.NewTimer(time.Millisecond * 100).C: } } + +func TestBalancePersistence(t *testing.T) { + clock := &mclock.Simulated{} + ns := nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) + db := memorydb.New() + posExp := &utils.Expirer{} + negExp := &utils.Expirer{} + posExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour*2)) // halves every two hours + negExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour)) // halves every hour + bt := NewBalanceTracker(ns, btTestSetup, db, clock, posExp, negExp) + ns.Start() + bts := &balanceTestSetup{ + clock: clock, + ns: ns, + bt: bt, + } + var nb *NodeBalance + exp := func(expPos, expNeg uint64) { + pos, neg := nb.GetBalance() + if pos != expPos { + t.Fatalf("Positive balance incorrect, want %v, got %v", expPos, pos) + } + if neg != expNeg { + t.Fatalf("Positive balance incorrect, want %v, got %v", expPos, pos) + } + } + expTotal := func(expTotal uint64) { + total := bt.TotalTokenAmount() + if total != expTotal { + t.Fatalf("Total token amount incorrect, want %v, got %v", expTotal, total) + } + } + + expTotal(0) + nb = bts.newNode(0) + expTotal(0) + nb.SetBalance(16000000000, 16000000000) + exp(16000000000, 16000000000) + expTotal(16000000000) + clock.Run(time.Hour * 2) + exp(8000000000, 4000000000) + expTotal(8000000000) + bt.Stop() + ns.Stop() + + clock = &mclock.Simulated{} + ns = nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) + posExp = &utils.Expirer{} + negExp = &utils.Expirer{} + posExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour*2)) // halves every two hours + negExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour)) // halves every hour + bt = NewBalanceTracker(ns, btTestSetup, db, clock, posExp, negExp) + ns.Start() + bts = &balanceTestSetup{ + clock: clock, + ns: ns, + bt: bt, + } + expTotal(8000000000) + nb = bts.newNode(0) + exp(8000000000, 4000000000) + expTotal(8000000000) + clock.Run(time.Hour * 2) + exp(4000000000, 1000000000) + expTotal(4000000000) + bt.Stop() + ns.Stop() +} diff --git a/les/vflux/server/balance_tracker.go b/les/vflux/server/balance_tracker.go index edd4b288d9..1708019de4 100644 --- a/les/vflux/server/balance_tracker.go +++ b/les/vflux/server/balance_tracker.go @@ -99,6 +99,10 @@ func NewBalanceTracker(ns *nodestate.NodeStateMachine, setup BalanceTrackerSetup balanceTimer: utils.NewUpdateTimer(clock, time.Second*10), quit: make(chan struct{}), } + posOffset, negOffset := bt.ndb.getExpiration() + posExp.SetLogOffset(clock.Now(), posOffset) + negExp.SetLogOffset(clock.Now(), negOffset) + bt.ndb.forEachBalance(false, func(id enode.ID, balance utils.ExpiredValue) bool { bt.inactive.AddExp(balance) return true @@ -177,7 +181,7 @@ func (bt *BalanceTracker) TotalTokenAmount() uint64 { bt.balanceTimer.Update(func(_ time.Duration) bool { bt.active = utils.ExpiredValue{} bt.ns.ForEach(nodestate.Flags{}, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { - if n, ok := bt.ns.GetField(node, bt.BalanceField).(*NodeBalance); ok { + if n, ok := bt.ns.GetField(node, bt.BalanceField).(*NodeBalance); ok && n.active { pos, _ := n.GetRawBalance() bt.active.AddExp(pos) } From 8f03e3b107c0f7a39de31a9e7deb658431a937ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Sat, 20 Feb 2021 10:40:38 +0100 Subject: [PATCH 149/709] tests/fuzzers/les: add fuzzer for les server handler (#22282) * les: refactored server handler * tests/fuzzers/les: add fuzzer for les server handler * tests, les: update les fuzzer tests: update les fuzzer tests/fuzzer/les: release resources tests/fuzzer/les: pre-initialize all resources * les: refactored server handler and fuzzer Co-authored-by: rjl493456442 --- les/handler_test.go | 34 +- les/peer.go | 4 +- les/protocol.go | 65 ++- les/server_handler.go | 780 +++++--------------------------- les/server_requests.go | 569 +++++++++++++++++++++++ les/test_helper.go | 4 + oss-fuzz.sh | 1 + tests/fuzzers/les/debug/main.go | 41 ++ tests/fuzzers/les/les-fuzzer.go | 407 +++++++++++++++++ 9 files changed, 1218 insertions(+), 687 deletions(-) create mode 100644 les/server_requests.go create mode 100644 tests/fuzzers/les/debug/main.go create mode 100644 tests/fuzzers/les/les-fuzzer.go diff --git a/les/handler_test.go b/les/handler_test.go index 83be312081..e251f4503b 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -65,27 +65,27 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Create a batch of tests for various scenarios limit := uint64(MaxHeaderFetch) tests := []struct { - query *getBlockHeadersData // The query to execute for header retrieval + query *GetBlockHeadersData // The query to execute for header retrieval expect []common.Hash // The hashes of the block whose headers are expected }{ // A single random block should be retrievable by hash and number too { - &getBlockHeadersData{Origin: hashOrNumber{Hash: bc.GetBlockByNumber(limit / 2).Hash()}, Amount: 1}, + &GetBlockHeadersData{Origin: hashOrNumber{Hash: bc.GetBlockByNumber(limit / 2).Hash()}, Amount: 1}, []common.Hash{bc.GetBlockByNumber(limit / 2).Hash()}, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 1}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 1}, []common.Hash{bc.GetBlockByNumber(limit / 2).Hash()}, }, // Multiple headers should be retrievable in both directions { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3}, []common.Hash{ bc.GetBlockByNumber(limit / 2).Hash(), bc.GetBlockByNumber(limit/2 + 1).Hash(), bc.GetBlockByNumber(limit/2 + 2).Hash(), }, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3, Reverse: true}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3, Reverse: true}, []common.Hash{ bc.GetBlockByNumber(limit / 2).Hash(), bc.GetBlockByNumber(limit/2 - 1).Hash(), @@ -94,14 +94,14 @@ func testGetBlockHeaders(t *testing.T, protocol int) { }, // Multiple headers with skip lists should be retrievable { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3}, []common.Hash{ bc.GetBlockByNumber(limit / 2).Hash(), bc.GetBlockByNumber(limit/2 + 4).Hash(), bc.GetBlockByNumber(limit/2 + 8).Hash(), }, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3, Reverse: true}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3, Reverse: true}, []common.Hash{ bc.GetBlockByNumber(limit / 2).Hash(), bc.GetBlockByNumber(limit/2 - 4).Hash(), @@ -110,26 +110,26 @@ func testGetBlockHeaders(t *testing.T, protocol int) { }, // The chain endpoints should be retrievable { - &getBlockHeadersData{Origin: hashOrNumber{Number: 0}, Amount: 1}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: 0}, Amount: 1}, []common.Hash{bc.GetBlockByNumber(0).Hash()}, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64()}, Amount: 1}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64()}, Amount: 1}, []common.Hash{bc.CurrentBlock().Hash()}, }, // Ensure protocol limits are honored //{ - // &getBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() - 1}, Amount: limit + 10, Reverse: true}, + // &GetBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() - 1}, Amount: limit + 10, Reverse: true}, // []common.Hash{}, //}, // Check that requesting more than available is handled gracefully { - &getBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() - 4}, Skip: 3, Amount: 3}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() - 4}, Skip: 3, Amount: 3}, []common.Hash{ bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 4).Hash(), bc.GetBlockByNumber(bc.CurrentBlock().NumberU64()).Hash(), }, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 3, Amount: 3, Reverse: true}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 3, Amount: 3, Reverse: true}, []common.Hash{ bc.GetBlockByNumber(4).Hash(), bc.GetBlockByNumber(0).Hash(), @@ -137,13 +137,13 @@ func testGetBlockHeaders(t *testing.T, protocol int) { }, // Check that requesting more than available is handled gracefully, even if mid skip { - &getBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() - 4}, Skip: 2, Amount: 3}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() - 4}, Skip: 2, Amount: 3}, []common.Hash{ bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 4).Hash(), bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1).Hash(), }, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 2, Amount: 3, Reverse: true}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 2, Amount: 3, Reverse: true}, []common.Hash{ bc.GetBlockByNumber(4).Hash(), bc.GetBlockByNumber(1).Hash(), @@ -151,10 +151,10 @@ func testGetBlockHeaders(t *testing.T, protocol int) { }, // Check that non existing headers aren't returned { - &getBlockHeadersData{Origin: hashOrNumber{Hash: unknown}, Amount: 1}, + &GetBlockHeadersData{Origin: hashOrNumber{Hash: unknown}, Amount: 1}, []common.Hash{}, }, { - &getBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() + 1}, Amount: 1}, + &GetBlockHeadersData{Origin: hashOrNumber{Number: bc.CurrentBlock().NumberU64() + 1}, Amount: 1}, []common.Hash{}, }, } @@ -619,7 +619,7 @@ func TestStopResumeLes3(t *testing.T) { header := server.handler.blockchain.CurrentHeader() req := func() { reqID++ - sendRequest(server.peer.app, GetBlockHeadersMsg, reqID, &getBlockHeadersData{Origin: hashOrNumber{Hash: header.Hash()}, Amount: 1}) + sendRequest(server.peer.app, GetBlockHeadersMsg, reqID, &GetBlockHeadersData{Origin: hashOrNumber{Hash: header.Hash()}, Amount: 1}) } for i := 1; i <= 5; i++ { // send requests while we still have enough buffer and expect a response diff --git a/les/peer.go b/les/peer.go index 52ab506368..479b4034bc 100644 --- a/les/peer.go +++ b/les/peer.go @@ -432,14 +432,14 @@ func (p *serverPeer) sendRequest(msgcode, reqID uint64, data interface{}, amount // specified header query, based on the hash of an origin block. func (p *serverPeer) requestHeadersByHash(reqID uint64, origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - return p.sendRequest(GetBlockHeadersMsg, reqID, &getBlockHeadersData{Origin: hashOrNumber{Hash: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse}, amount) + return p.sendRequest(GetBlockHeadersMsg, reqID, &GetBlockHeadersData{Origin: hashOrNumber{Hash: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse}, amount) } // requestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *serverPeer) requestHeadersByNumber(reqID, origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - return p.sendRequest(GetBlockHeadersMsg, reqID, &getBlockHeadersData{Origin: hashOrNumber{Number: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse}, amount) + return p.sendRequest(GetBlockHeadersMsg, reqID, &GetBlockHeadersData{Origin: hashOrNumber{Number: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse}, amount) } // requestBodies fetches a batch of blocks' bodies corresponding to the hashes diff --git a/les/protocol.go b/les/protocol.go index 909d25d375..07a4452f40 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -24,6 +24,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/p2p/enode" @@ -83,6 +84,62 @@ const ( ResumeMsg = 0x17 ) +// GetBlockHeadersData represents a block header query (the request ID is not included) +type GetBlockHeadersData struct { + Origin hashOrNumber // Block from which to retrieve headers + Amount uint64 // Maximum number of headers to retrieve + Skip uint64 // Blocks to skip between consecutive headers + Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) +} + +// GetBlockHeadersPacket represents a block header request +type GetBlockHeadersPacket struct { + ReqID uint64 + Query GetBlockHeadersData +} + +// GetBlockBodiesPacket represents a block body request +type GetBlockBodiesPacket struct { + ReqID uint64 + Hashes []common.Hash +} + +// GetCodePacket represents a contract code request +type GetCodePacket struct { + ReqID uint64 + Reqs []CodeReq +} + +// GetReceiptsPacket represents a block receipts request +type GetReceiptsPacket struct { + ReqID uint64 + Hashes []common.Hash +} + +// GetProofsPacket represents a proof request +type GetProofsPacket struct { + ReqID uint64 + Reqs []ProofReq +} + +// GetHelperTrieProofsPacket represents a helper trie proof request +type GetHelperTrieProofsPacket struct { + ReqID uint64 + Reqs []HelperTrieReq +} + +// SendTxPacket represents a transaction propagation request +type SendTxPacket struct { + ReqID uint64 + Txs []*types.Transaction +} + +// GetTxStatusPacket represents a transaction status query +type GetTxStatusPacket struct { + ReqID uint64 + Hashes []common.Hash +} + type requestInfo struct { name string maxCount uint64 @@ -229,14 +286,6 @@ type blockInfo struct { Td *big.Int // Total difficulty of one particular block being announced } -// getBlockHeadersData represents a block header query. -type getBlockHeadersData struct { - Origin hashOrNumber // Block from which to retrieve headers - Amount uint64 // Maximum number of headers to retrieve - Skip uint64 // Blocks to skip between consecutive headers - Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) -} - // hashOrNumber is a combined field for specifying an origin block. type hashOrNumber struct { Hash common.Hash // Block hash from which to retrieve headers (excludes Number) diff --git a/les/server_handler.go b/les/server_handler.go index fd81e273c9..b6e8b050b1 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -18,8 +18,6 @@ package les import ( "crypto/ecdsa" - "encoding/binary" - "encoding/json" "errors" "sync" "sync/atomic" @@ -223,648 +221,109 @@ func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error { } defer msg.Discard() - var ( - maxCost uint64 - task *servingTask - ) p.responseCount++ responseCount := p.responseCount - // accept returns an indicator whether the request can be served. - // If so, deduct the max cost from the flow control buffer. - accept := func(reqID, reqCnt, maxCnt uint64) bool { - // Short circuit if the peer is already frozen or the request is invalid. - inSizeCost := h.server.costTracker.realCost(0, msg.Size, 0) - if p.isFrozen() || reqCnt == 0 || reqCnt > maxCnt { - p.fcClient.OneTimeCost(inSizeCost) - return false - } - // Prepaid max cost units before request been serving. - maxCost = p.fcCosts.getMaxCost(msg.Code, reqCnt) - accepted, bufShort, priority := p.fcClient.AcceptRequest(reqID, responseCount, maxCost) - if !accepted { - p.freeze() - p.Log().Error("Request came too early", "remaining", common.PrettyDuration(time.Duration(bufShort*1000000/p.fcParams.MinRecharge))) - p.fcClient.OneTimeCost(inSizeCost) - return false - } - // Create a multi-stage task, estimate the time it takes for the task to - // execute, and cache it in the request service queue. - factor := h.server.costTracker.globalFactor() - if factor < 0.001 { - factor = 1 - p.Log().Error("Invalid global cost factor", "factor", factor) - } - maxTime := uint64(float64(maxCost) / factor) - task = h.server.servingQueue.newTask(p, maxTime, priority) - if task.start() { - return true - } - p.fcClient.RequestProcessed(reqID, responseCount, maxCost, inSizeCost) - return false - } - // sendResponse sends back the response and updates the flow control statistic. - sendResponse := func(reqID, amount uint64, reply *reply, servingTime uint64) { - p.responseLock.Lock() - defer p.responseLock.Unlock() - // Short circuit if the client is already frozen. - if p.isFrozen() { - realCost := h.server.costTracker.realCost(servingTime, msg.Size, 0) - p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) - return - } - // Positive correction buffer value with real cost. - var replySize uint32 - if reply != nil { - replySize = reply.size() - } - var realCost uint64 - if h.server.costTracker.testing { - realCost = maxCost // Assign a fake cost for testing purpose - } else { - realCost = h.server.costTracker.realCost(servingTime, msg.Size, replySize) - if realCost > maxCost { - realCost = maxCost - } - } - bv := p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) - if amount != 0 { - // Feed cost tracker request serving statistic. - h.server.costTracker.updateStats(msg.Code, amount, servingTime, realCost) - // Reduce priority "balance" for the specific peer. - p.balance.RequestServed(realCost) - } - if reply != nil { - p.queueSend(func() { - if err := reply.send(bv); err != nil { - select { - case p.errCh <- err: - default: - } - } - }) - } + req, ok := Les3[msg.Code] + if !ok { + p.Log().Trace("Received invalid message", "code", msg.Code) + clientErrorMeter.Mark(1) + return errResp(ErrInvalidMsgCode, "%v", msg.Code) } - switch msg.Code { - case GetBlockHeadersMsg: - p.Log().Trace("Received block header request") - if metrics.EnabledExpensive { - miscInHeaderPacketsMeter.Mark(1) - miscInHeaderTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Query getBlockHeadersData - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "%v: %v", msg, err) - } - query := req.Query - if accept(req.ReqID, query.Amount, MaxHeaderFetch) { - wg.Add(1) - go func() { - defer wg.Done() - hashMode := query.Origin.Hash != (common.Hash{}) - first := true - maxNonCanonical := uint64(100) - - // Gather headers until the fetch or network limits is reached - var ( - bytes common.StorageSize - headers []*types.Header - unknown bool - ) - for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit { - if !first && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - // Retrieve the next header satisfying the query - var origin *types.Header - if hashMode { - if first { - origin = h.blockchain.GetHeaderByHash(query.Origin.Hash) - if origin != nil { - query.Origin.Number = origin.Number.Uint64() - } - } else { - origin = h.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) - } - } else { - origin = h.blockchain.GetHeaderByNumber(query.Origin.Number) - } - if origin == nil { - break - } - headers = append(headers, origin) - bytes += estHeaderRlpSize - - // Advance to the next header of the query - switch { - case hashMode && query.Reverse: - // Hash based traversal towards the genesis block - ancestor := query.Skip + 1 - if ancestor == 0 { - unknown = true - } else { - query.Origin.Hash, query.Origin.Number = h.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) - unknown = query.Origin.Hash == common.Hash{} - } - case hashMode && !query.Reverse: - // Hash based traversal towards the leaf block - var ( - current = origin.Number.Uint64() - next = current + query.Skip + 1 - ) - if next <= current { - infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") - p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) - unknown = true - } else { - if header := h.blockchain.GetHeaderByNumber(next); header != nil { - nextHash := header.Hash() - expOldHash, _ := h.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) - if expOldHash == query.Origin.Hash { - query.Origin.Hash, query.Origin.Number = nextHash, next - } else { - unknown = true - } - } else { - unknown = true - } - } - case query.Reverse: - // Number based traversal towards the genesis block - if query.Origin.Number >= query.Skip+1 { - query.Origin.Number -= query.Skip + 1 - } else { - unknown = true - } - - case !query.Reverse: - // Number based traversal towards the leaf block - query.Origin.Number += query.Skip + 1 - } - first = false - } - reply := p.replyBlockHeaders(req.ReqID, headers) - sendResponse(req.ReqID, query.Amount, reply, task.done()) - if metrics.EnabledExpensive { - miscOutHeaderPacketsMeter.Mark(1) - miscOutHeaderTrafficMeter.Mark(int64(reply.size())) - miscServingTimeHeaderTimer.Update(time.Duration(task.servingTime)) - } - }() - } - - case GetBlockBodiesMsg: - p.Log().Trace("Received block bodies request") - if metrics.EnabledExpensive { - miscInBodyPacketsMeter.Mark(1) - miscInBodyTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Hashes []common.Hash - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - var ( - bytes int - bodies []rlp.RawValue - ) - reqCnt := len(req.Hashes) - if accept(req.ReqID, uint64(reqCnt), MaxBodyFetch) { - wg.Add(1) - go func() { - defer wg.Done() - for i, hash := range req.Hashes { - if i != 0 && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - if bytes >= softResponseLimit { - break - } - body := h.blockchain.GetBodyRLP(hash) - if body == nil { - p.bumpInvalid() - continue - } - bodies = append(bodies, body) - bytes += len(body) - } - reply := p.replyBlockBodiesRLP(req.ReqID, bodies) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) - if metrics.EnabledExpensive { - miscOutBodyPacketsMeter.Mark(1) - miscOutBodyTrafficMeter.Mark(int64(reply.size())) - miscServingTimeBodyTimer.Update(time.Duration(task.servingTime)) - } - }() - } - - case GetCodeMsg: - p.Log().Trace("Received code request") - if metrics.EnabledExpensive { - miscInCodePacketsMeter.Mark(1) - miscInCodeTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Reqs []CodeReq - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - var ( - bytes int - data [][]byte - ) - reqCnt := len(req.Reqs) - if accept(req.ReqID, uint64(reqCnt), MaxCodeFetch) { - wg.Add(1) - go func() { - defer wg.Done() - for i, request := range req.Reqs { - if i != 0 && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - // Look up the root hash belonging to the request - header := h.blockchain.GetHeaderByHash(request.BHash) - if header == nil { - p.Log().Warn("Failed to retrieve associate header for code", "hash", request.BHash) - p.bumpInvalid() - continue - } - // Refuse to search stale state data in the database since looking for - // a non-exist key is kind of expensive. - local := h.blockchain.CurrentHeader().Number.Uint64() - if !h.server.archiveMode && header.Number.Uint64()+core.TriesInMemory <= local { - p.Log().Debug("Reject stale code request", "number", header.Number.Uint64(), "head", local) - p.bumpInvalid() - continue - } - triedb := h.blockchain.StateCache().TrieDB() - - account, err := h.getAccount(triedb, header.Root, common.BytesToHash(request.AccKey)) - if err != nil { - p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err) - p.bumpInvalid() - continue - } - code, err := h.blockchain.StateCache().ContractCode(common.BytesToHash(request.AccKey), common.BytesToHash(account.CodeHash)) - if err != nil { - p.Log().Warn("Failed to retrieve account code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "codehash", common.BytesToHash(account.CodeHash), "err", err) - continue - } - // Accumulate the code and abort if enough data was retrieved - data = append(data, code) - if bytes += len(code); bytes >= softResponseLimit { - break - } - } - reply := p.replyCode(req.ReqID, data) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) - if metrics.EnabledExpensive { - miscOutCodePacketsMeter.Mark(1) - miscOutCodeTrafficMeter.Mark(int64(reply.size())) - miscServingTimeCodeTimer.Update(time.Duration(task.servingTime)) - } - }() - } - - case GetReceiptsMsg: - p.Log().Trace("Received receipts request") - if metrics.EnabledExpensive { - miscInReceiptPacketsMeter.Mark(1) - miscInReceiptTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Hashes []common.Hash - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - var ( - bytes int - receipts []rlp.RawValue - ) - reqCnt := len(req.Hashes) - if accept(req.ReqID, uint64(reqCnt), MaxReceiptFetch) { - wg.Add(1) - go func() { - defer wg.Done() - for i, hash := range req.Hashes { - if i != 0 && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - if bytes >= softResponseLimit { - break - } - // Retrieve the requested block's receipts, skipping if unknown to us - results := h.blockchain.GetReceiptsByHash(hash) - if results == nil { - if header := h.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - p.bumpInvalid() - continue - } - } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(results); err != nil { - log.Error("Failed to encode receipt", "err", err) - } else { - receipts = append(receipts, encoded) - bytes += len(encoded) - } - } - reply := p.replyReceiptsRLP(req.ReqID, receipts) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) - if metrics.EnabledExpensive { - miscOutReceiptPacketsMeter.Mark(1) - miscOutReceiptTrafficMeter.Mark(int64(reply.size())) - miscServingTimeReceiptTimer.Update(time.Duration(task.servingTime)) - } - }() - } - - case GetProofsV2Msg: - p.Log().Trace("Received les/2 proofs request") - if metrics.EnabledExpensive { - miscInTrieProofPacketsMeter.Mark(1) - miscInTrieProofTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Reqs []ProofReq - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Gather state data until the fetch or network limits is reached - var ( - lastBHash common.Hash - root common.Hash - header *types.Header - ) - reqCnt := len(req.Reqs) - if accept(req.ReqID, uint64(reqCnt), MaxProofsFetch) { - wg.Add(1) - go func() { - defer wg.Done() - nodes := light.NewNodeSet() + p.Log().Trace("Received " + req.Name) - for i, request := range req.Reqs { - if i != 0 && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - // Look up the root hash belonging to the request - if request.BHash != lastBHash { - root, lastBHash = common.Hash{}, request.BHash + serve, reqID, reqCnt, err := req.Handle(msg) + if err != nil { + clientErrorMeter.Mark(1) + return errResp(ErrDecode, "%v: %v", msg, err) + } - if header = h.blockchain.GetHeaderByHash(request.BHash); header == nil { - p.Log().Warn("Failed to retrieve header for proof", "hash", request.BHash) - p.bumpInvalid() - continue - } - // Refuse to search stale state data in the database since looking for - // a non-exist key is kind of expensive. - local := h.blockchain.CurrentHeader().Number.Uint64() - if !h.server.archiveMode && header.Number.Uint64()+core.TriesInMemory <= local { - p.Log().Debug("Reject stale trie request", "number", header.Number.Uint64(), "head", local) - p.bumpInvalid() - continue - } - root = header.Root - } - // If a header lookup failed (non existent), ignore subsequent requests for the same header - if root == (common.Hash{}) { - p.bumpInvalid() - continue - } - // Open the account or storage trie for the request - statedb := h.blockchain.StateCache() + if metrics.EnabledExpensive { + req.InPacketsMeter.Mark(1) + req.InTrafficMeter.Mark(int64(msg.Size)) + } - var trie state.Trie - switch len(request.AccKey) { - case 0: - // No account key specified, open an account trie - trie, err = statedb.OpenTrie(root) - if trie == nil || err != nil { - p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "root", root, "err", err) - continue - } - default: - // Account key specified, open a storage trie - account, err := h.getAccount(statedb.TrieDB(), root, common.BytesToHash(request.AccKey)) - if err != nil { - p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err) - p.bumpInvalid() - continue - } - trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root) - if trie == nil || err != nil { - p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "root", account.Root, "err", err) - continue - } - } - // Prove the user's request from the account or stroage trie - if err := trie.Prove(request.Key, request.FromLevel, nodes); err != nil { - p.Log().Warn("Failed to prove state request", "block", header.Number, "hash", header.Hash(), "err", err) - continue - } - if nodes.DataSize() >= softResponseLimit { - break - } - } - reply := p.replyProofsV2(req.ReqID, nodes.NodeList()) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) - if metrics.EnabledExpensive { - miscOutTrieProofPacketsMeter.Mark(1) - miscOutTrieProofTrafficMeter.Mark(int64(reply.size())) - miscServingTimeTrieProofTimer.Update(time.Duration(task.servingTime)) - } - }() - } + // Short circuit if the peer is already frozen or the request is invalid. + inSizeCost := h.server.costTracker.realCost(0, msg.Size, 0) + if p.isFrozen() || reqCnt == 0 || reqCnt > req.MaxCount { + p.fcClient.OneTimeCost(inSizeCost) + return nil + } + // Prepaid max cost units before request been serving. + maxCost := p.fcCosts.getMaxCost(msg.Code, reqCnt) + accepted, bufShort, priority := p.fcClient.AcceptRequest(reqID, responseCount, maxCost) + if !accepted { + p.freeze() + p.Log().Error("Request came too early", "remaining", common.PrettyDuration(time.Duration(bufShort*1000000/p.fcParams.MinRecharge))) + p.fcClient.OneTimeCost(inSizeCost) + return nil + } + // Create a multi-stage task, estimate the time it takes for the task to + // execute, and cache it in the request service queue. + factor := h.server.costTracker.globalFactor() + if factor < 0.001 { + factor = 1 + p.Log().Error("Invalid global cost factor", "factor", factor) + } + maxTime := uint64(float64(maxCost) / factor) + task := h.server.servingQueue.newTask(p, maxTime, priority) + if task.start() { + wg.Add(1) + go func() { + defer wg.Done() + reply := serve(h, p, task.waitOrStop) + if reply != nil { + task.done() + } - case GetHelperTrieProofsMsg: - p.Log().Trace("Received helper trie proof request") - if metrics.EnabledExpensive { - miscInHelperTriePacketsMeter.Mark(1) - miscInHelperTrieTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Reqs []HelperTrieReq - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Gather state data until the fetch or network limits is reached - var ( - auxBytes int - auxData [][]byte - ) - reqCnt := len(req.Reqs) - if accept(req.ReqID, uint64(reqCnt), MaxHelperTrieProofsFetch) { - wg.Add(1) - go func() { - defer wg.Done() - var ( - lastIdx uint64 - lastType uint - root common.Hash - auxTrie *trie.Trie - ) - nodes := light.NewNodeSet() - for i, request := range req.Reqs { - if i != 0 && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - if auxTrie == nil || request.Type != lastType || request.TrieIdx != lastIdx { - auxTrie, lastType, lastIdx = nil, request.Type, request.TrieIdx + p.responseLock.Lock() + defer p.responseLock.Unlock() - var prefix string - if root, prefix = h.getHelperTrie(request.Type, request.TrieIdx); root != (common.Hash{}) { - auxTrie, _ = trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) - } - } - if auxTrie == nil { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - // TODO(rjl493456442) short circuit if the proving is failed. - // The original client side code has a dirty hack to retrieve - // the headers with no valid proof. Keep the compatibility for - // legacy les protocol and drop this hack when the les2/3 are - // not supported. - err := auxTrie.Prove(request.Key, request.FromLevel, nodes) - if p.version >= lpv4 && err != nil { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return - } - if request.AuxReq == htAuxHeader { - data := h.getAuxiliaryHeaders(request) - auxData = append(auxData, data) - auxBytes += len(data) - } - if nodes.DataSize()+auxBytes >= softResponseLimit { - break - } - } - reply := p.replyHelperTrieProofs(req.ReqID, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) - if metrics.EnabledExpensive { - miscOutHelperTriePacketsMeter.Mark(1) - miscOutHelperTrieTrafficMeter.Mark(int64(reply.size())) - miscServingTimeHelperTrieTimer.Update(time.Duration(task.servingTime)) + // Short circuit if the client is already frozen. + if p.isFrozen() { + realCost := h.server.costTracker.realCost(task.servingTime, msg.Size, 0) + p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) + return + } + // Positive correction buffer value with real cost. + var replySize uint32 + if reply != nil { + replySize = reply.size() + } + var realCost uint64 + if h.server.costTracker.testing { + realCost = maxCost // Assign a fake cost for testing purpose + } else { + realCost = h.server.costTracker.realCost(task.servingTime, msg.Size, replySize) + if realCost > maxCost { + realCost = maxCost } - }() - } - - case SendTxV2Msg: - p.Log().Trace("Received new transactions") - if metrics.EnabledExpensive { - miscInTxsPacketsMeter.Mark(1) - miscInTxsTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Txs []*types.Transaction - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - reqCnt := len(req.Txs) - if accept(req.ReqID, uint64(reqCnt), MaxTxSend) { - wg.Add(1) - go func() { - defer wg.Done() - stats := make([]light.TxStatus, len(req.Txs)) - for i, tx := range req.Txs { - if i != 0 && !task.waitOrStop() { - return - } - hash := tx.Hash() - stats[i] = h.txStatus(hash) - if stats[i].Status == core.TxStatusUnknown { - addFn := h.txpool.AddRemotes - // Add txs synchronously for testing purpose - if h.addTxsSync { - addFn = h.txpool.AddRemotesSync - } - if errs := addFn([]*types.Transaction{tx}); errs[0] != nil { - stats[i].Error = errs[0].Error() - continue + } + bv := p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) + if reply != nil { + // Feed cost tracker request serving statistic. + h.server.costTracker.updateStats(msg.Code, reqCnt, task.servingTime, realCost) + // Reduce priority "balance" for the specific peer. + p.balance.RequestServed(realCost) + p.queueSend(func() { + if err := reply.send(bv); err != nil { + select { + case p.errCh <- err: + default: } - stats[i] = h.txStatus(hash) - } - } - reply := p.replyTxStatus(req.ReqID, stats) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) - if metrics.EnabledExpensive { - miscOutTxsPacketsMeter.Mark(1) - miscOutTxsTrafficMeter.Mark(int64(reply.size())) - miscServingTimeTxTimer.Update(time.Duration(task.servingTime)) - } - }() - } - - case GetTxStatusMsg: - p.Log().Trace("Received transaction status query request") - if metrics.EnabledExpensive { - miscInTxStatusPacketsMeter.Mark(1) - miscInTxStatusTrafficMeter.Mark(int64(msg.Size)) - } - var req struct { - ReqID uint64 - Hashes []common.Hash - } - if err := msg.Decode(&req); err != nil { - clientErrorMeter.Mark(1) - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - reqCnt := len(req.Hashes) - if accept(req.ReqID, uint64(reqCnt), MaxTxStatus) { - wg.Add(1) - go func() { - defer wg.Done() - stats := make([]light.TxStatus, len(req.Hashes)) - for i, hash := range req.Hashes { - if i != 0 && !task.waitOrStop() { - sendResponse(req.ReqID, 0, nil, task.servingTime) - return } - stats[i] = h.txStatus(hash) - } - reply := p.replyTxStatus(req.ReqID, stats) - sendResponse(req.ReqID, uint64(reqCnt), reply, task.done()) + }) if metrics.EnabledExpensive { - miscOutTxStatusPacketsMeter.Mark(1) - miscOutTxStatusTrafficMeter.Mark(int64(reply.size())) - miscServingTimeTxStatusTimer.Update(time.Duration(task.servingTime)) + req.OutPacketsMeter.Mark(1) + req.OutTrafficMeter.Mark(int64(replySize)) + req.ServingTimeMeter.Update(time.Duration(task.servingTime)) } - }() - } - - default: - p.Log().Trace("Received invalid message", "code", msg.Code) - clientErrorMeter.Mark(1) - return errResp(ErrInvalidMsgCode, "%v", msg.Code) + } + }() + } else { + p.fcClient.RequestProcessed(reqID, responseCount, maxCost, inSizeCost) } + // If the client has made too much invalid request(e.g. request a non-existent data), // reject them to prevent SPAM attack. if p.getInvalid() > maxRequestErrors { @@ -874,8 +333,28 @@ func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error { return nil } +// BlockChain implements serverBackend +func (h *serverHandler) BlockChain() *core.BlockChain { + return h.blockchain +} + +// TxPool implements serverBackend +func (h *serverHandler) TxPool() *core.TxPool { + return h.txpool +} + +// ArchiveMode implements serverBackend +func (h *serverHandler) ArchiveMode() bool { + return h.server.archiveMode +} + +// AddTxsSync implements serverBackend +func (h *serverHandler) AddTxsSync() bool { + return h.addTxsSync +} + // getAccount retrieves an account from the state based on root. -func (h *serverHandler) getAccount(triedb *trie.Database, root, hash common.Hash) (state.Account, error) { +func getAccount(triedb *trie.Database, root, hash common.Hash) (state.Account, error) { trie, err := trie.New(root, triedb) if err != nil { return state.Account{}, err @@ -892,43 +371,24 @@ func (h *serverHandler) getAccount(triedb *trie.Database, root, hash common.Hash } // getHelperTrie returns the post-processed trie root for the given trie ID and section index -func (h *serverHandler) getHelperTrie(typ uint, index uint64) (common.Hash, string) { +func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { + var ( + root common.Hash + prefix string + ) switch typ { case htCanonical: sectionHead := rawdb.ReadCanonicalHash(h.chainDb, (index+1)*h.server.iConfig.ChtSize-1) - return light.GetChtRoot(h.chainDb, index, sectionHead), light.ChtTablePrefix + root, prefix = light.GetChtRoot(h.chainDb, index, sectionHead), light.ChtTablePrefix case htBloomBits: sectionHead := rawdb.ReadCanonicalHash(h.chainDb, (index+1)*h.server.iConfig.BloomTrieSize-1) - return light.GetBloomTrieRoot(h.chainDb, index, sectionHead), light.BloomTrieTablePrefix - } - return common.Hash{}, "" -} - -// getAuxiliaryHeaders returns requested auxiliary headers for the CHT request. -func (h *serverHandler) getAuxiliaryHeaders(req HelperTrieReq) []byte { - if req.Type == htCanonical && req.AuxReq == htAuxHeader && len(req.Key) == 8 { - blockNum := binary.BigEndian.Uint64(req.Key) - hash := rawdb.ReadCanonicalHash(h.chainDb, blockNum) - return rawdb.ReadHeaderRLP(h.chainDb, hash, blockNum) + root, prefix = light.GetBloomTrieRoot(h.chainDb, index, sectionHead), light.BloomTrieTablePrefix } - return nil -} - -// txStatus returns the status of a specified transaction. -func (h *serverHandler) txStatus(hash common.Hash) light.TxStatus { - var stat light.TxStatus - // Looking the transaction in txpool first. - stat.Status = h.txpool.Status([]common.Hash{hash})[0] - - // If the transaction is unknown to the pool, try looking it up locally. - if stat.Status == core.TxStatusUnknown { - lookup := h.blockchain.GetTransactionLookup(hash) - if lookup != nil { - stat.Status = core.TxStatusIncluded - stat.Lookup = lookup - } + if root == (common.Hash{}) { + return nil } - return stat + trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + return trie } // broadcastLoop broadcasts new block information to all connected light diff --git a/les/server_requests.go b/les/server_requests.go new file mode 100644 index 0000000000..d4af8006f0 --- /dev/null +++ b/les/server_requests.go @@ -0,0 +1,569 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "encoding/binary" + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// serverBackend defines the backend functions needed for serving LES requests +type serverBackend interface { + ArchiveMode() bool + AddTxsSync() bool + BlockChain() *core.BlockChain + TxPool() *core.TxPool + GetHelperTrie(typ uint, index uint64) *trie.Trie +} + +// Decoder is implemented by the messages passed to the handler functions +type Decoder interface { + Decode(val interface{}) error +} + +// RequestType is a static struct that describes an LES request type and references +// its handler function. +type RequestType struct { + Name string + MaxCount uint64 + InPacketsMeter, InTrafficMeter, OutPacketsMeter, OutTrafficMeter metrics.Meter + ServingTimeMeter metrics.Timer + Handle func(msg Decoder) (serve serveRequestFn, reqID, amount uint64, err error) +} + +// serveRequestFn is returned by the request handler functions after decoding the request. +// This function does the actual request serving using the supplied backend. waitOrStop is +// called between serving individual request items and may block if the serving process +// needs to be throttled. If it returns false then the process is terminated. +// The reply is not sent by this function yet. The flow control feedback value is supplied +// by the protocol handler when calling the send function of the returned reply struct. +type serveRequestFn func(backend serverBackend, peer *clientPeer, waitOrStop func() bool) *reply + +// Les3 contains the request types supported by les/2 and les/3 +var Les3 = map[uint64]RequestType{ + GetBlockHeadersMsg: RequestType{ + Name: "block header request", + MaxCount: MaxHeaderFetch, + InPacketsMeter: miscInHeaderPacketsMeter, + InTrafficMeter: miscInHeaderTrafficMeter, + OutPacketsMeter: miscOutHeaderPacketsMeter, + OutTrafficMeter: miscOutHeaderTrafficMeter, + ServingTimeMeter: miscServingTimeHeaderTimer, + Handle: handleGetBlockHeaders, + }, + GetBlockBodiesMsg: RequestType{ + Name: "block bodies request", + MaxCount: MaxBodyFetch, + InPacketsMeter: miscInBodyPacketsMeter, + InTrafficMeter: miscInBodyTrafficMeter, + OutPacketsMeter: miscOutBodyPacketsMeter, + OutTrafficMeter: miscOutBodyTrafficMeter, + ServingTimeMeter: miscServingTimeBodyTimer, + Handle: handleGetBlockBodies, + }, + GetCodeMsg: RequestType{ + Name: "code request", + MaxCount: MaxCodeFetch, + InPacketsMeter: miscInCodePacketsMeter, + InTrafficMeter: miscInCodeTrafficMeter, + OutPacketsMeter: miscOutCodePacketsMeter, + OutTrafficMeter: miscOutCodeTrafficMeter, + ServingTimeMeter: miscServingTimeCodeTimer, + Handle: handleGetCode, + }, + GetReceiptsMsg: RequestType{ + Name: "receipts request", + MaxCount: MaxReceiptFetch, + InPacketsMeter: miscInReceiptPacketsMeter, + InTrafficMeter: miscInReceiptTrafficMeter, + OutPacketsMeter: miscOutReceiptPacketsMeter, + OutTrafficMeter: miscOutReceiptTrafficMeter, + ServingTimeMeter: miscServingTimeReceiptTimer, + Handle: handleGetReceipts, + }, + GetProofsV2Msg: RequestType{ + Name: "les/2 proofs request", + MaxCount: MaxProofsFetch, + InPacketsMeter: miscInTrieProofPacketsMeter, + InTrafficMeter: miscInTrieProofTrafficMeter, + OutPacketsMeter: miscOutTrieProofPacketsMeter, + OutTrafficMeter: miscOutTrieProofTrafficMeter, + ServingTimeMeter: miscServingTimeTrieProofTimer, + Handle: handleGetProofs, + }, + GetHelperTrieProofsMsg: RequestType{ + Name: "helper trie proof request", + MaxCount: MaxHelperTrieProofsFetch, + InPacketsMeter: miscInHelperTriePacketsMeter, + InTrafficMeter: miscInHelperTrieTrafficMeter, + OutPacketsMeter: miscOutHelperTriePacketsMeter, + OutTrafficMeter: miscOutHelperTrieTrafficMeter, + ServingTimeMeter: miscServingTimeHelperTrieTimer, + Handle: handleGetHelperTrieProofs, + }, + SendTxV2Msg: RequestType{ + Name: "new transactions", + MaxCount: MaxTxSend, + InPacketsMeter: miscInTxsPacketsMeter, + InTrafficMeter: miscInTxsTrafficMeter, + OutPacketsMeter: miscOutTxsPacketsMeter, + OutTrafficMeter: miscOutTxsTrafficMeter, + ServingTimeMeter: miscServingTimeTxTimer, + Handle: handleSendTx, + }, + GetTxStatusMsg: RequestType{ + Name: "transaction status query request", + MaxCount: MaxTxStatus, + InPacketsMeter: miscInTxStatusPacketsMeter, + InTrafficMeter: miscInTxStatusTrafficMeter, + OutPacketsMeter: miscOutTxStatusPacketsMeter, + OutTrafficMeter: miscOutTxStatusTrafficMeter, + ServingTimeMeter: miscServingTimeTxStatusTimer, + Handle: handleGetTxStatus, + }, +} + +// handleGetBlockHeaders handles a block header request +func handleGetBlockHeaders(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetBlockHeadersPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + // Gather headers until the fetch or network limits is reached + var ( + bc = backend.BlockChain() + hashMode = r.Query.Origin.Hash != (common.Hash{}) + first = true + maxNonCanonical = uint64(100) + bytes common.StorageSize + headers []*types.Header + unknown bool + ) + for !unknown && len(headers) < int(r.Query.Amount) && bytes < softResponseLimit { + if !first && !waitOrStop() { + return nil + } + // Retrieve the next header satisfying the r + var origin *types.Header + if hashMode { + if first { + origin = bc.GetHeaderByHash(r.Query.Origin.Hash) + if origin != nil { + r.Query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = bc.GetHeader(r.Query.Origin.Hash, r.Query.Origin.Number) + } + } else { + origin = bc.GetHeaderByNumber(r.Query.Origin.Number) + } + if origin == nil { + break + } + headers = append(headers, origin) + bytes += estHeaderRlpSize + + // Advance to the next header of the r + switch { + case hashMode && r.Query.Reverse: + // Hash based traversal towards the genesis block + ancestor := r.Query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + r.Query.Origin.Hash, r.Query.Origin.Number = bc.GetAncestor(r.Query.Origin.Hash, r.Query.Origin.Number, ancestor, &maxNonCanonical) + unknown = r.Query.Origin.Hash == common.Hash{} + } + case hashMode && !r.Query.Reverse: + // Hash based traversal towards the leaf block + var ( + current = origin.Number.Uint64() + next = current + r.Query.Skip + 1 + ) + if next <= current { + infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") + p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", r.Query.Skip, "next", next, "attacker", infos) + unknown = true + } else { + if header := bc.GetHeaderByNumber(next); header != nil { + nextHash := header.Hash() + expOldHash, _ := bc.GetAncestor(nextHash, next, r.Query.Skip+1, &maxNonCanonical) + if expOldHash == r.Query.Origin.Hash { + r.Query.Origin.Hash, r.Query.Origin.Number = nextHash, next + } else { + unknown = true + } + } else { + unknown = true + } + } + case r.Query.Reverse: + // Number based traversal towards the genesis block + if r.Query.Origin.Number >= r.Query.Skip+1 { + r.Query.Origin.Number -= r.Query.Skip + 1 + } else { + unknown = true + } + + case !r.Query.Reverse: + // Number based traversal towards the leaf block + r.Query.Origin.Number += r.Query.Skip + 1 + } + first = false + } + return p.replyBlockHeaders(r.ReqID, headers) + }, r.ReqID, r.Query.Amount, nil +} + +// handleGetBlockBodies handles a block body request +func handleGetBlockBodies(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetBlockBodiesPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + var ( + bytes int + bodies []rlp.RawValue + ) + bc := backend.BlockChain() + for i, hash := range r.Hashes { + if i != 0 && !waitOrStop() { + return nil + } + if bytes >= softResponseLimit { + break + } + body := bc.GetBodyRLP(hash) + if body == nil { + p.bumpInvalid() + continue + } + bodies = append(bodies, body) + bytes += len(body) + } + return p.replyBlockBodiesRLP(r.ReqID, bodies) + }, r.ReqID, uint64(len(r.Hashes)), nil +} + +// handleGetCode handles a contract code request +func handleGetCode(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetCodePacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + var ( + bytes int + data [][]byte + ) + bc := backend.BlockChain() + for i, request := range r.Reqs { + if i != 0 && !waitOrStop() { + return nil + } + // Look up the root hash belonging to the request + header := bc.GetHeaderByHash(request.BHash) + if header == nil { + p.Log().Warn("Failed to retrieve associate header for code", "hash", request.BHash) + p.bumpInvalid() + continue + } + // Refuse to search stale state data in the database since looking for + // a non-exist key is kind of expensive. + local := bc.CurrentHeader().Number.Uint64() + if !backend.ArchiveMode() && header.Number.Uint64()+core.TriesInMemory <= local { + p.Log().Debug("Reject stale code request", "number", header.Number.Uint64(), "head", local) + p.bumpInvalid() + continue + } + triedb := bc.StateCache().TrieDB() + + account, err := getAccount(triedb, header.Root, common.BytesToHash(request.AccKey)) + if err != nil { + p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err) + p.bumpInvalid() + continue + } + code, err := bc.StateCache().ContractCode(common.BytesToHash(request.AccKey), common.BytesToHash(account.CodeHash)) + if err != nil { + p.Log().Warn("Failed to retrieve account code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "codehash", common.BytesToHash(account.CodeHash), "err", err) + continue + } + // Accumulate the code and abort if enough data was retrieved + data = append(data, code) + if bytes += len(code); bytes >= softResponseLimit { + break + } + } + return p.replyCode(r.ReqID, data) + }, r.ReqID, uint64(len(r.Reqs)), nil +} + +// handleGetReceipts handles a block receipts request +func handleGetReceipts(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetReceiptsPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + var ( + bytes int + receipts []rlp.RawValue + ) + bc := backend.BlockChain() + for i, hash := range r.Hashes { + if i != 0 && !waitOrStop() { + return nil + } + if bytes >= softResponseLimit { + break + } + // Retrieve the requested block's receipts, skipping if unknown to us + results := bc.GetReceiptsByHash(hash) + if results == nil { + if header := bc.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + p.bumpInvalid() + continue + } + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(results); err != nil { + log.Error("Failed to encode receipt", "err", err) + } else { + receipts = append(receipts, encoded) + bytes += len(encoded) + } + } + return p.replyReceiptsRLP(r.ReqID, receipts) + }, r.ReqID, uint64(len(r.Hashes)), nil +} + +// handleGetProofs handles a proof request +func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetProofsPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + var ( + lastBHash common.Hash + root common.Hash + header *types.Header + err error + ) + bc := backend.BlockChain() + nodes := light.NewNodeSet() + + for i, request := range r.Reqs { + if i != 0 && !waitOrStop() { + return nil + } + // Look up the root hash belonging to the request + if request.BHash != lastBHash { + root, lastBHash = common.Hash{}, request.BHash + + if header = bc.GetHeaderByHash(request.BHash); header == nil { + p.Log().Warn("Failed to retrieve header for proof", "hash", request.BHash) + p.bumpInvalid() + continue + } + // Refuse to search stale state data in the database since looking for + // a non-exist key is kind of expensive. + local := bc.CurrentHeader().Number.Uint64() + if !backend.ArchiveMode() && header.Number.Uint64()+core.TriesInMemory <= local { + p.Log().Debug("Reject stale trie request", "number", header.Number.Uint64(), "head", local) + p.bumpInvalid() + continue + } + root = header.Root + } + // If a header lookup failed (non existent), ignore subsequent requests for the same header + if root == (common.Hash{}) { + p.bumpInvalid() + continue + } + // Open the account or storage trie for the request + statedb := bc.StateCache() + + var trie state.Trie + switch len(request.AccKey) { + case 0: + // No account key specified, open an account trie + trie, err = statedb.OpenTrie(root) + if trie == nil || err != nil { + p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "root", root, "err", err) + continue + } + default: + // Account key specified, open a storage trie + account, err := getAccount(statedb.TrieDB(), root, common.BytesToHash(request.AccKey)) + if err != nil { + p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err) + p.bumpInvalid() + continue + } + trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root) + if trie == nil || err != nil { + p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "root", account.Root, "err", err) + continue + } + } + // Prove the user's request from the account or stroage trie + if err := trie.Prove(request.Key, request.FromLevel, nodes); err != nil { + p.Log().Warn("Failed to prove state request", "block", header.Number, "hash", header.Hash(), "err", err) + continue + } + if nodes.DataSize() >= softResponseLimit { + break + } + } + return p.replyProofsV2(r.ReqID, nodes.NodeList()) + }, r.ReqID, uint64(len(r.Reqs)), nil +} + +// handleGetHelperTrieProofs handles a helper trie proof request +func handleGetHelperTrieProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetHelperTrieProofsPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + var ( + lastIdx uint64 + lastType uint + auxTrie *trie.Trie + auxBytes int + auxData [][]byte + ) + bc := backend.BlockChain() + nodes := light.NewNodeSet() + for i, request := range r.Reqs { + if i != 0 && !waitOrStop() { + return nil + } + if auxTrie == nil || request.Type != lastType || request.TrieIdx != lastIdx { + lastType, lastIdx = request.Type, request.TrieIdx + auxTrie = backend.GetHelperTrie(request.Type, request.TrieIdx) + } + if auxTrie == nil { + return nil + } + // TODO(rjl493456442) short circuit if the proving is failed. + // The original client side code has a dirty hack to retrieve + // the headers with no valid proof. Keep the compatibility for + // legacy les protocol and drop this hack when the les2/3 are + // not supported. + err := auxTrie.Prove(request.Key, request.FromLevel, nodes) + if p.version >= lpv4 && err != nil { + return nil + } + if request.Type == htCanonical && request.AuxReq == htAuxHeader && len(request.Key) == 8 { + header := bc.GetHeaderByNumber(binary.BigEndian.Uint64(request.Key)) + data, err := rlp.EncodeToBytes(header) + if err != nil { + log.Error("Failed to encode header", "err", err) + return nil + } + auxData = append(auxData, data) + auxBytes += len(data) + } + if nodes.DataSize()+auxBytes >= softResponseLimit { + break + } + } + return p.replyHelperTrieProofs(r.ReqID, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) + }, r.ReqID, uint64(len(r.Reqs)), nil +} + +// handleSendTx handles a transaction propagation request +func handleSendTx(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r SendTxPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + amount := uint64(len(r.Txs)) + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + stats := make([]light.TxStatus, len(r.Txs)) + for i, tx := range r.Txs { + if i != 0 && !waitOrStop() { + return nil + } + hash := tx.Hash() + stats[i] = txStatus(backend, hash) + if stats[i].Status == core.TxStatusUnknown { + addFn := backend.TxPool().AddRemotes + // Add txs synchronously for testing purpose + if backend.AddTxsSync() { + addFn = backend.TxPool().AddRemotesSync + } + if errs := addFn([]*types.Transaction{tx}); errs[0] != nil { + stats[i].Error = errs[0].Error() + continue + } + stats[i] = txStatus(backend, hash) + } + } + return p.replyTxStatus(r.ReqID, stats) + }, r.ReqID, amount, nil +} + +// handleGetTxStatus handles a transaction status query +func handleGetTxStatus(msg Decoder) (serveRequestFn, uint64, uint64, error) { + var r GetTxStatusPacket + if err := msg.Decode(&r); err != nil { + return nil, 0, 0, err + } + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { + stats := make([]light.TxStatus, len(r.Hashes)) + for i, hash := range r.Hashes { + if i != 0 && !waitOrStop() { + return nil + } + stats[i] = txStatus(backend, hash) + } + return p.replyTxStatus(r.ReqID, stats) + }, r.ReqID, uint64(len(r.Hashes)), nil +} + +// txStatus returns the status of a specified transaction. +func txStatus(b serverBackend, hash common.Hash) light.TxStatus { + var stat light.TxStatus + // Looking the transaction in txpool first. + stat.Status = b.TxPool().Status([]common.Hash{hash})[0] + + // If the transaction is unknown to the pool, try looking it up locally. + if stat.Status == core.TxStatusUnknown { + lookup := b.BlockChain().GetTransactionLookup(hash) + if lookup != nil { + stat.Status = core.TxStatusIncluded + stat.Lookup = lookup + } + } + return stat +} diff --git a/les/test_helper.go b/les/test_helper.go index 1afc9be7d8..6f93c8a48f 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -594,3 +594,7 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, callback indexer } return s, c, teardown } + +func NewFuzzerPeer(version int) *clientPeer { + return newClientPeer(version, 0, p2p.NewPeer(enode.ID{}, "", nil), nil) +} diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 860ce0952f..ac93a5a467 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -101,6 +101,7 @@ compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty compile_fuzzer tests/fuzzers/abi Fuzz fuzzAbi +compile_fuzzer tests/fuzzers/les Fuzz fuzzLes compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul diff --git a/tests/fuzzers/les/debug/main.go b/tests/fuzzers/les/debug/main.go new file mode 100644 index 0000000000..09e087d4c8 --- /dev/null +++ b/tests/fuzzers/les/debug/main.go @@ -0,0 +1,41 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/tests/fuzzers/les" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: debug \n") + fmt.Fprintf(os.Stderr, "Example\n") + fmt.Fprintf(os.Stderr, " $ debug ../crashers/4bbef6857c733a87ecf6fd8b9e7238f65eb9862a\n") + os.Exit(1) + } + crasher := os.Args[1] + data, err := ioutil.ReadFile(crasher) + if err != nil { + fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) + os.Exit(1) + } + les.Fuzz(data) +} diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go new file mode 100644 index 0000000000..9e896c2c1b --- /dev/null +++ b/tests/fuzzers/les/les-fuzzer.go @@ -0,0 +1,407 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "bytes" + "encoding/binary" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + l "github.com/ethereum/go-ethereum/les" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + bankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + bankAddr = crypto.PubkeyToAddress(bankKey.PublicKey) + bankFunds = new(big.Int).Mul(big.NewInt(100), big.NewInt(params.Ether)) + + testChainLen = 256 + testContractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") + + chain *core.BlockChain + addrHashes []common.Hash + txHashes []common.Hash + + chtTrie *trie.Trie + bloomTrie *trie.Trie + chtKeys [][]byte + bloomKeys [][]byte +) + +func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { + db := rawdb.NewMemoryDatabase() + gspec := core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{bankAddr: {Balance: bankFunds}}, + GasLimit: 100000000, + } + genesis := gspec.MustCommit(db) + signer := types.HomesteadSigner{} + blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, testChainLen, + func(i int, gen *core.BlockGen) { + var ( + tx *types.Transaction + addr common.Address + ) + nonce := uint64(i) + if i%4 == 0 { + tx, _ = types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, bankKey) + addr = crypto.CreateAddress(bankAddr, nonce) + } else { + addr = common.BigToAddress(big.NewInt(int64(i))) + tx, _ = types.SignTx(types.NewTransaction(nonce, addr, big.NewInt(10000), params.TxGas, big.NewInt(params.GWei), nil), signer, bankKey) + } + gen.AddTx(tx) + addrHashes = append(addrHashes, crypto.Keccak256Hash(addr[:])) + txHashes = append(txHashes, tx.Hash()) + }) + bc, _ = core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + if _, err := bc.InsertChain(blocks); err != nil { + panic(err) + } + return +} + +func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { + chtTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + bloomTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + for i := 0; i < testChainLen; i++ { + // The element in CHT is -> + key := make([]byte, 8) + binary.BigEndian.PutUint64(key, uint64(i+1)) + chtTrie.Update(key, []byte{0x1, 0xf}) + chtKeys = append(chtKeys, key) + + // The element in Bloom trie is <2 byte bit index> + -> bloom + key2 := make([]byte, 10) + binary.BigEndian.PutUint64(key2[2:], uint64(i+1)) + bloomTrie.Update(key2, []byte{0x2, 0xe}) + bloomKeys = append(bloomKeys, key2) + } + return +} + +func init() { + chain, addrHashes, txHashes = makechain() + chtTrie, bloomTrie, chtKeys, bloomKeys = makeTries() +} + +type fuzzer struct { + chain *core.BlockChain + pool *core.TxPool + + chainLen int + addr, txs []common.Hash + nonce uint64 + + chtKeys [][]byte + bloomKeys [][]byte + chtTrie *trie.Trie + bloomTrie *trie.Trie + + input io.Reader + exhausted bool +} + +func newFuzzer(input []byte) *fuzzer { + return &fuzzer{ + chain: chain, + chainLen: testChainLen, + addr: addrHashes, + txs: txHashes, + chtTrie: chtTrie, + bloomTrie: bloomTrie, + chtKeys: chtKeys, + bloomKeys: bloomKeys, + nonce: uint64(len(txHashes)), + pool: core.NewTxPool(core.DefaultTxPoolConfig, params.TestChainConfig, chain), + input: bytes.NewReader(input), + } +} + +func (f *fuzzer) read(size int) []byte { + out := make([]byte, size) + if _, err := f.input.Read(out); err != nil { + f.exhausted = true + } + return out +} + +func (f *fuzzer) randomByte() byte { + d := f.read(1) + return d[0] +} + +func (f *fuzzer) randomBool() bool { + d := f.read(1) + return d[0]&1 == 1 +} + +func (f *fuzzer) randomInt(max int) int { + if max == 0 { + return 0 + } + if max <= 256 { + return int(f.randomByte()) % max + } + var a uint16 + if err := binary.Read(f.input, binary.LittleEndian, &a); err != nil { + f.exhausted = true + } + return int(a % uint16(max)) +} + +func (f *fuzzer) randomX(max int) uint64 { + var a uint16 + if err := binary.Read(f.input, binary.LittleEndian, &a); err != nil { + f.exhausted = true + } + if a < 0x8000 { + return uint64(a%uint16(max+1)) - 1 + } + return (uint64(1)<<(a%64+1) - 1) & (uint64(a) * 343897772345826595) +} + +func (f *fuzzer) randomBlockHash() common.Hash { + h := f.chain.GetCanonicalHash(uint64(f.randomInt(3 * f.chainLen))) + if h != (common.Hash{}) { + return h + } + return common.BytesToHash(f.read(common.HashLength)) +} + +func (f *fuzzer) randomAddrHash() []byte { + i := f.randomInt(3 * len(f.addr)) + if i < len(f.addr) { + return f.addr[i].Bytes() + } + return f.read(common.HashLength) +} + +func (f *fuzzer) randomCHTTrieKey() []byte { + i := f.randomInt(3 * len(f.chtKeys)) + if i < len(f.chtKeys) { + return f.chtKeys[i] + } + return f.read(8) +} + +func (f *fuzzer) randomBloomTrieKey() []byte { + i := f.randomInt(3 * len(f.bloomKeys)) + if i < len(f.bloomKeys) { + return f.bloomKeys[i] + } + return f.read(10) +} + +func (f *fuzzer) randomTxHash() common.Hash { + i := f.randomInt(3 * len(f.txs)) + if i < len(f.txs) { + return f.txs[i] + } + return common.BytesToHash(f.read(common.HashLength)) +} + +func (f *fuzzer) BlockChain() *core.BlockChain { + return f.chain +} + +func (f *fuzzer) TxPool() *core.TxPool { + return f.pool +} + +func (f *fuzzer) ArchiveMode() bool { + return false +} + +func (f *fuzzer) AddTxsSync() bool { + return false +} + +func (f *fuzzer) GetHelperTrie(typ uint, index uint64) *trie.Trie { + if typ == 0 { + return f.chtTrie + } else if typ == 1 { + return f.bloomTrie + } + return nil +} + +type dummyMsg struct { + data []byte +} + +func (d dummyMsg) Decode(val interface{}) error { + return rlp.DecodeBytes(d.data, val) +} + +func (f *fuzzer) doFuzz(msgCode uint64, packet interface{}) { + version := f.randomInt(3) + 2 // [LES2, LES3, LES4] + peer := l.NewFuzzerPeer(version) + enc, err := rlp.EncodeToBytes(packet) + if err != nil { + panic(err) + } + fn, _, _, err := l.Les3[msgCode].Handle(dummyMsg{enc}) + if err != nil { + panic(err) + } + fn(f, peer, func() bool { return true }) + +} + +func Fuzz(input []byte) int { + // We expect some large inputs + if len(input) < 100 { + return -1 + } + f := newFuzzer(input) + if f.exhausted { + return -1 + } + for !f.exhausted { + switch f.randomInt(8) { + case 0: + req := &l.GetBlockHeadersPacket{ + Query: l.GetBlockHeadersData{ + Amount: f.randomX(l.MaxHeaderFetch + 1), + Skip: f.randomX(10), + Reverse: f.randomBool(), + }, + } + if f.randomBool() { + req.Query.Origin.Hash = f.randomBlockHash() + } else { + req.Query.Origin.Number = uint64(f.randomInt(f.chainLen * 2)) + } + f.doFuzz(l.GetBlockHeadersMsg, req) + + case 1: + req := &l.GetBlockBodiesPacket{Hashes: make([]common.Hash, f.randomInt(l.MaxBodyFetch+1))} + for i := range req.Hashes { + req.Hashes[i] = f.randomBlockHash() + } + f.doFuzz(l.GetBlockBodiesMsg, req) + + case 2: + req := &l.GetCodePacket{Reqs: make([]l.CodeReq, f.randomInt(l.MaxCodeFetch+1))} + for i := range req.Reqs { + req.Reqs[i] = l.CodeReq{ + BHash: f.randomBlockHash(), + AccKey: f.randomAddrHash(), + } + } + f.doFuzz(l.GetCodeMsg, req) + + case 3: + req := &l.GetReceiptsPacket{Hashes: make([]common.Hash, f.randomInt(l.MaxReceiptFetch+1))} + for i := range req.Hashes { + req.Hashes[i] = f.randomBlockHash() + } + f.doFuzz(l.GetReceiptsMsg, req) + + case 4: + req := &l.GetProofsPacket{Reqs: make([]l.ProofReq, f.randomInt(l.MaxProofsFetch+1))} + for i := range req.Reqs { + if f.randomBool() { + req.Reqs[i] = l.ProofReq{ + BHash: f.randomBlockHash(), + AccKey: f.randomAddrHash(), + Key: f.randomAddrHash(), + FromLevel: uint(f.randomX(3)), + } + } else { + req.Reqs[i] = l.ProofReq{ + BHash: f.randomBlockHash(), + Key: f.randomAddrHash(), + FromLevel: uint(f.randomX(3)), + } + } + } + f.doFuzz(l.GetProofsV2Msg, req) + + case 5: + req := &l.GetHelperTrieProofsPacket{Reqs: make([]l.HelperTrieReq, f.randomInt(l.MaxHelperTrieProofsFetch+1))} + for i := range req.Reqs { + switch f.randomInt(3) { + case 0: + // Canonical hash trie + req.Reqs[i] = l.HelperTrieReq{ + Type: 0, + TrieIdx: f.randomX(3), + Key: f.randomCHTTrieKey(), + FromLevel: uint(f.randomX(3)), + AuxReq: uint(2), + } + case 1: + // Bloom trie + req.Reqs[i] = l.HelperTrieReq{ + Type: 1, + TrieIdx: f.randomX(3), + Key: f.randomBloomTrieKey(), + FromLevel: uint(f.randomX(3)), + AuxReq: 0, + } + default: + // Random trie + req.Reqs[i] = l.HelperTrieReq{ + Type: 2, + TrieIdx: f.randomX(3), + Key: f.randomCHTTrieKey(), + FromLevel: uint(f.randomX(3)), + AuxReq: 0, + } + } + } + f.doFuzz(l.GetHelperTrieProofsMsg, req) + + case 6: + req := &l.SendTxPacket{Txs: make([]*types.Transaction, f.randomInt(l.MaxTxSend+1))} + signer := types.HomesteadSigner{} + for i := range req.Txs { + var nonce uint64 + if f.randomBool() { + nonce = uint64(f.randomByte()) + } else { + nonce = f.nonce + f.nonce += 1 + } + req.Txs[i], _ = types.SignTx(types.NewTransaction(nonce, common.Address{}, big.NewInt(10000), params.TxGas, big.NewInt(1000000000*int64(f.randomByte())), nil), signer, bankKey) + } + f.doFuzz(l.SendTxV2Msg, req) + + case 7: + req := &l.GetTxStatusPacket{Hashes: make([]common.Hash, f.randomInt(l.MaxTxStatus+1))} + for i := range req.Hashes { + req.Hashes[i] = f.randomTxHash() + } + f.doFuzz(l.GetTxStatusMsg, req) + } + } + return 0 +} From 3ecfdccd9a0065365a00e7c8b60de7ee2df4e40b Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 22 Feb 2021 21:33:11 +0800 Subject: [PATCH 150/709] les: clean up server handler (#22357) --- les/server_handler.go | 185 +++++++++++++++++++++++------------------ les/server_requests.go | 16 ++-- 2 files changed, 114 insertions(+), 87 deletions(-) diff --git a/les/server_handler.go b/les/server_handler.go index b6e8b050b1..7651d03cab 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -204,6 +204,90 @@ func (h *serverHandler) handle(p *clientPeer) error { } } +// beforeHandle will do a series of prechecks before handling message. +func (h *serverHandler) beforeHandle(p *clientPeer, reqID, responseCount uint64, msg p2p.Msg, reqCnt uint64, maxCount uint64) (*servingTask, uint64) { + // Ensure that the request sent by client peer is valid + inSizeCost := h.server.costTracker.realCost(0, msg.Size, 0) + if reqCnt == 0 || reqCnt > maxCount { + p.fcClient.OneTimeCost(inSizeCost) + return nil, 0 + } + // Ensure that the client peer complies with the flow control + // rules agreed by both sides. + if p.isFrozen() { + p.fcClient.OneTimeCost(inSizeCost) + return nil, 0 + } + maxCost := p.fcCosts.getMaxCost(msg.Code, reqCnt) + accepted, bufShort, priority := p.fcClient.AcceptRequest(reqID, responseCount, maxCost) + if !accepted { + p.freeze() + p.Log().Error("Request came too early", "remaining", common.PrettyDuration(time.Duration(bufShort*1000000/p.fcParams.MinRecharge))) + p.fcClient.OneTimeCost(inSizeCost) + return nil, 0 + } + // Create a multi-stage task, estimate the time it takes for the task to + // execute, and cache it in the request service queue. + factor := h.server.costTracker.globalFactor() + if factor < 0.001 { + factor = 1 + p.Log().Error("Invalid global cost factor", "factor", factor) + } + maxTime := uint64(float64(maxCost) / factor) + task := h.server.servingQueue.newTask(p, maxTime, priority) + if !task.start() { + p.fcClient.RequestProcessed(reqID, responseCount, maxCost, inSizeCost) + return nil, 0 + } + return task, maxCost +} + +// Afterhandle will perform a series of operations after message handling, +// such as updating flow control data, sending reply, etc. +func (h *serverHandler) afterHandle(p *clientPeer, reqID, responseCount uint64, msg p2p.Msg, maxCost uint64, reqCnt uint64, task *servingTask, reply *reply) { + if reply != nil { + task.done() + } + p.responseLock.Lock() + defer p.responseLock.Unlock() + + // Short circuit if the client is already frozen. + if p.isFrozen() { + realCost := h.server.costTracker.realCost(task.servingTime, msg.Size, 0) + p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) + return + } + // Positive correction buffer value with real cost. + var replySize uint32 + if reply != nil { + replySize = reply.size() + } + var realCost uint64 + if h.server.costTracker.testing { + realCost = maxCost // Assign a fake cost for testing purpose + } else { + realCost = h.server.costTracker.realCost(task.servingTime, msg.Size, replySize) + if realCost > maxCost { + realCost = maxCost + } + } + bv := p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) + if reply != nil { + // Feed cost tracker request serving statistic. + h.server.costTracker.updateStats(msg.Code, reqCnt, task.servingTime, realCost) + // Reduce priority "balance" for the specific peer. + p.balance.RequestServed(realCost) + p.queueSend(func() { + if err := reply.send(bv); err != nil { + select { + case p.errCh <- err: + default: + } + } + }) + } +} + // handleMsg is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error { @@ -221,9 +305,8 @@ func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error { } defer msg.Discard() - p.responseCount++ - responseCount := p.responseCount - + // Lookup the request handler table, ensure it's supported + // message type by the protocol. req, ok := Les3[msg.Code] if !ok { p.Log().Trace("Received invalid message", "code", msg.Code) @@ -232,98 +315,42 @@ func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error { } p.Log().Trace("Received " + req.Name) + // Decode the p2p message, resolve the concrete handler for it. serve, reqID, reqCnt, err := req.Handle(msg) if err != nil { clientErrorMeter.Mark(1) return errResp(ErrDecode, "%v: %v", msg, err) } - if metrics.EnabledExpensive { req.InPacketsMeter.Mark(1) req.InTrafficMeter.Mark(int64(msg.Size)) } + p.responseCount++ + responseCount := p.responseCount - // Short circuit if the peer is already frozen or the request is invalid. - inSizeCost := h.server.costTracker.realCost(0, msg.Size, 0) - if p.isFrozen() || reqCnt == 0 || reqCnt > req.MaxCount { - p.fcClient.OneTimeCost(inSizeCost) - return nil - } - // Prepaid max cost units before request been serving. - maxCost := p.fcCosts.getMaxCost(msg.Code, reqCnt) - accepted, bufShort, priority := p.fcClient.AcceptRequest(reqID, responseCount, maxCost) - if !accepted { - p.freeze() - p.Log().Error("Request came too early", "remaining", common.PrettyDuration(time.Duration(bufShort*1000000/p.fcParams.MinRecharge))) - p.fcClient.OneTimeCost(inSizeCost) + // First check this client message complies all rules before + // handling it and return a processor if all checks are passed. + task, maxCost := h.beforeHandle(p, reqID, responseCount, msg, reqCnt, req.MaxCount) + if task == nil { return nil } - // Create a multi-stage task, estimate the time it takes for the task to - // execute, and cache it in the request service queue. - factor := h.server.costTracker.globalFactor() - if factor < 0.001 { - factor = 1 - p.Log().Error("Invalid global cost factor", "factor", factor) - } - maxTime := uint64(float64(maxCost) / factor) - task := h.server.servingQueue.newTask(p, maxTime, priority) - if task.start() { - wg.Add(1) - go func() { - defer wg.Done() - reply := serve(h, p, task.waitOrStop) - if reply != nil { - task.done() - } + wg.Add(1) + go func() { + defer wg.Done() - p.responseLock.Lock() - defer p.responseLock.Unlock() + reply := serve(h, p, task.waitOrStop) + h.afterHandle(p, reqID, responseCount, msg, maxCost, reqCnt, task, reply) - // Short circuit if the client is already frozen. - if p.isFrozen() { - realCost := h.server.costTracker.realCost(task.servingTime, msg.Size, 0) - p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) - return - } - // Positive correction buffer value with real cost. - var replySize uint32 + if metrics.EnabledExpensive { + size := uint32(0) if reply != nil { - replySize = reply.size() - } - var realCost uint64 - if h.server.costTracker.testing { - realCost = maxCost // Assign a fake cost for testing purpose - } else { - realCost = h.server.costTracker.realCost(task.servingTime, msg.Size, replySize) - if realCost > maxCost { - realCost = maxCost - } + size = reply.size() } - bv := p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) - if reply != nil { - // Feed cost tracker request serving statistic. - h.server.costTracker.updateStats(msg.Code, reqCnt, task.servingTime, realCost) - // Reduce priority "balance" for the specific peer. - p.balance.RequestServed(realCost) - p.queueSend(func() { - if err := reply.send(bv); err != nil { - select { - case p.errCh <- err: - default: - } - } - }) - if metrics.EnabledExpensive { - req.OutPacketsMeter.Mark(1) - req.OutTrafficMeter.Mark(int64(replySize)) - req.ServingTimeMeter.Update(time.Duration(task.servingTime)) - } - } - }() - } else { - p.fcClient.RequestProcessed(reqID, responseCount, maxCost, inSizeCost) - } - + req.OutPacketsMeter.Mark(1) + req.OutTrafficMeter.Mark(int64(size)) + req.ServingTimeMeter.Update(time.Duration(task.servingTime)) + } + }() // If the client has made too much invalid request(e.g. request a non-existent data), // reject them to prevent SPAM attack. if p.getInvalid() > maxRequestErrors { diff --git a/les/server_requests.go b/les/server_requests.go index d4af8006f0..07f30b1b73 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -65,7 +65,7 @@ type serveRequestFn func(backend serverBackend, peer *clientPeer, waitOrStop fun // Les3 contains the request types supported by les/2 and les/3 var Les3 = map[uint64]RequestType{ - GetBlockHeadersMsg: RequestType{ + GetBlockHeadersMsg: { Name: "block header request", MaxCount: MaxHeaderFetch, InPacketsMeter: miscInHeaderPacketsMeter, @@ -75,7 +75,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeHeaderTimer, Handle: handleGetBlockHeaders, }, - GetBlockBodiesMsg: RequestType{ + GetBlockBodiesMsg: { Name: "block bodies request", MaxCount: MaxBodyFetch, InPacketsMeter: miscInBodyPacketsMeter, @@ -85,7 +85,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeBodyTimer, Handle: handleGetBlockBodies, }, - GetCodeMsg: RequestType{ + GetCodeMsg: { Name: "code request", MaxCount: MaxCodeFetch, InPacketsMeter: miscInCodePacketsMeter, @@ -95,7 +95,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeCodeTimer, Handle: handleGetCode, }, - GetReceiptsMsg: RequestType{ + GetReceiptsMsg: { Name: "receipts request", MaxCount: MaxReceiptFetch, InPacketsMeter: miscInReceiptPacketsMeter, @@ -105,7 +105,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeReceiptTimer, Handle: handleGetReceipts, }, - GetProofsV2Msg: RequestType{ + GetProofsV2Msg: { Name: "les/2 proofs request", MaxCount: MaxProofsFetch, InPacketsMeter: miscInTrieProofPacketsMeter, @@ -115,7 +115,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeTrieProofTimer, Handle: handleGetProofs, }, - GetHelperTrieProofsMsg: RequestType{ + GetHelperTrieProofsMsg: { Name: "helper trie proof request", MaxCount: MaxHelperTrieProofsFetch, InPacketsMeter: miscInHelperTriePacketsMeter, @@ -125,7 +125,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeHelperTrieTimer, Handle: handleGetHelperTrieProofs, }, - SendTxV2Msg: RequestType{ + SendTxV2Msg: { Name: "new transactions", MaxCount: MaxTxSend, InPacketsMeter: miscInTxsPacketsMeter, @@ -135,7 +135,7 @@ var Les3 = map[uint64]RequestType{ ServingTimeMeter: miscServingTimeTxTimer, Handle: handleSendTx, }, - GetTxStatusMsg: RequestType{ + GetTxStatusMsg: { Name: "transaction status query request", MaxCount: MaxTxStatus, InPacketsMeter: miscInTxStatusPacketsMeter, From c4a2b682ff3ea2465417671de76c4d1e9a29fef8 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 23 Feb 2021 11:27:32 +0100 Subject: [PATCH 151/709] cmd/geth: add db commands stats, compact, put, get, delete (#22014) This PR introduces: - db.put to put a value into the database - db.get to read a value from the database - db.delete to delete a value from the database - db.stats to check compaction info from the database - db.compact to trigger a db compaction It also moves inspectdb to db.inspect. --- cmd/geth/chaincmd.go | 129 +------------- cmd/geth/dbcmd.go | 341 ++++++++++++++++++++++++++++++++++++++ cmd/geth/main.go | 3 +- cmd/utils/flags.go | 8 +- core/rawdb/database.go | 12 +- ethdb/leveldb/leveldb.go | 56 +++++-- internal/flags/helpers.go | 4 +- 7 files changed, 401 insertions(+), 152 deletions(-) create mode 100644 cmd/geth/dbcmd.go diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 9eec30f813..ba4289ddc3 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "runtime" "strconv" "sync/atomic" @@ -28,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -170,18 +168,6 @@ The export-preimages command export hash preimages to an RLP encoded stream`, Category: "BLOCKCHAIN COMMANDS", Description: ` The first argument must be the directory containing the blockchain to download from`, - } - removedbCommand = cli.Command{ - Action: utils.MigrateFlags(removeDB), - Name: "removedb", - Usage: "Remove blockchain and state databases", - ArgsUsage: " ", - Flags: []cli.Flag{ - utils.DataDirFlag, - }, - Category: "BLOCKCHAIN COMMANDS", - Description: ` -Remove blockchain and state databases`, } dumpCommand = cli.Command{ Action: utils.MigrateFlags(dump), @@ -202,25 +188,6 @@ Remove blockchain and state databases`, The arguments are interpreted as block numbers or hashes. Use "ethereum dump 0" to dump the genesis block.`, } - inspectCommand = cli.Command{ - Action: utils.MigrateFlags(inspect), - Name: "inspect", - Usage: "Inspect the storage size for each type of data in the database", - ArgsUsage: " ", - Flags: []cli.Flag{ - utils.DataDirFlag, - utils.AncientFlag, - utils.CacheFlag, - utils.MainnetFlag, - utils.RopstenFlag, - utils.RinkebyFlag, - utils.GoerliFlag, - utils.YoloV3Flag, - utils.LegacyTestnetFlag, - utils.SyncModeFlag, - }, - Category: "BLOCKCHAIN COMMANDS", - } ) // initGenesis will initialise the given JSON format genesis file and writes it as @@ -323,17 +290,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Import done in %v.\n\n", time.Since(start)) // Output pre-compaction stats mostly to see the import trashing - stats, err := db.Stat("leveldb.stats") - if err != nil { - utils.Fatalf("Failed to read database stats: %v", err) - } - fmt.Println(stats) - - ioStats, err := db.Stat("leveldb.iostats") - if err != nil { - utils.Fatalf("Failed to read database iostats: %v", err) - } - fmt.Println(ioStats) + showLeveldbStats(db) // Print the memory statistics used by the importing mem := new(runtime.MemStats) @@ -351,22 +308,12 @@ func importChain(ctx *cli.Context) error { // Compact the entire database to more accurately measure disk io and print the stats start = time.Now() fmt.Println("Compacting entire database...") - if err = db.Compact(nil, nil); err != nil { + if err := db.Compact(nil, nil); err != nil { utils.Fatalf("Compaction failed: %v", err) } fmt.Printf("Compaction done in %v.\n\n", time.Since(start)) - stats, err = db.Stat("leveldb.stats") - if err != nil { - utils.Fatalf("Failed to read database stats: %v", err) - } - fmt.Println(stats) - - ioStats, err = db.Stat("leveldb.iostats") - if err != nil { - utils.Fatalf("Failed to read database iostats: %v", err) - } - fmt.Println(ioStats) + showLeveldbStats(db) return importErr } @@ -499,66 +446,6 @@ func copyDb(ctx *cli.Context) error { return nil } -func removeDB(ctx *cli.Context) error { - stack, config := makeConfigNode(ctx) - - // Remove the full node state database - path := stack.ResolvePath("chaindata") - if common.FileExist(path) { - confirmAndRemoveDB(path, "full node state database") - } else { - log.Info("Full node state database missing", "path", path) - } - // Remove the full node ancient database - path = config.Eth.DatabaseFreezer - switch { - case path == "": - path = filepath.Join(stack.ResolvePath("chaindata"), "ancient") - case !filepath.IsAbs(path): - path = config.Node.ResolvePath(path) - } - if common.FileExist(path) { - confirmAndRemoveDB(path, "full node ancient database") - } else { - log.Info("Full node ancient database missing", "path", path) - } - // Remove the light node database - path = stack.ResolvePath("lightchaindata") - if common.FileExist(path) { - confirmAndRemoveDB(path, "light node database") - } else { - log.Info("Light node database missing", "path", path) - } - return nil -} - -// confirmAndRemoveDB prompts the user for a last confirmation and removes the -// folder if accepted. -func confirmAndRemoveDB(database string, kind string) { - confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database)) - switch { - case err != nil: - utils.Fatalf("%v", err) - case !confirm: - log.Info("Database deletion skipped", "path", database) - default: - start := time.Now() - filepath.Walk(database, func(path string, info os.FileInfo, err error) error { - // If we're at the top level folder, recurse into - if path == database { - return nil - } - // Delete all the files, but not subfolders - if !info.IsDir() { - os.Remove(path) - return nil - } - return filepath.SkipDir - }) - log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start))) - } -} - func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -598,16 +485,6 @@ func dump(ctx *cli.Context) error { return nil } -func inspect(ctx *cli.Context) error { - node, _ := makeConfigNode(ctx) - defer node.Close() - - _, chainDb := utils.MakeChain(ctx, node, true) - defer chainDb.Close() - - return rawdb.InspectDatabase(chainDb) -} - // hashish returns true for strings that look like hashes. func hashish(x string) bool { _, err := strconv.Atoi(x) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go new file mode 100644 index 0000000000..48478f613e --- /dev/null +++ b/cmd/geth/dbcmd.go @@ -0,0 +1,341 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/console/prompt" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/log" + "github.com/syndtr/goleveldb/leveldb/opt" + "gopkg.in/urfave/cli.v1" +) + +var ( + removedbCommand = cli.Command{ + Action: utils.MigrateFlags(removeDB), + Name: "removedb", + Usage: "Remove blockchain and state databases", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + }, + Category: "DATABASE COMMANDS", + Description: ` +Remove blockchain and state databases`, + } + dbCommand = cli.Command{ + Name: "db", + Usage: "Low level database operations", + ArgsUsage: "", + Category: "DATABASE COMMANDS", + Subcommands: []cli.Command{ + dbInspectCmd, + dbStatCmd, + dbCompactCmd, + dbGetCmd, + dbDeleteCmd, + dbPutCmd, + }, + } + dbInspectCmd = cli.Command{ + Action: utils.MigrateFlags(inspect), + Name: "inspect", + ArgsUsage: " ", + + Usage: "Inspect the storage size for each type of data in the database", + Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, + } + dbStatCmd = cli.Command{ + Action: dbStats, + Name: "stats", + Usage: "Print leveldb statistics", + } + dbCompactCmd = cli.Command{ + Action: dbCompact, + Name: "compact", + Usage: "Compact leveldb database. WARNING: May take a very long time", + Description: `This command performs a database compaction. +WARNING: This operation may take a very long time to finish, and may cause database +corruption if it is aborted during execution'!`, + } + dbGetCmd = cli.Command{ + Action: dbGet, + Name: "get", + Usage: "Show the value of a database key", + ArgsUsage: "", + Description: "This command looks up the specified database key from the database.", + } + dbDeleteCmd = cli.Command{ + Action: dbDelete, + Name: "delete", + Usage: "Delete a database key (WARNING: may corrupt your database)", + ArgsUsage: "", + Description: `This command deletes the specified database key from the database. +WARNING: This is a low-level operation which may cause database corruption!`, + } + dbPutCmd = cli.Command{ + Action: dbPut, + Name: "put", + Usage: "Set the value of a database key (WARNING: may corrupt your database)", + ArgsUsage: " ", + Description: `This command sets a given database key to the given value. +WARNING: This is a low-level operation which may cause database corruption!`, + } +) + +func removeDB(ctx *cli.Context) error { + stack, config := makeConfigNode(ctx) + + // Remove the full node state database + path := stack.ResolvePath("chaindata") + if common.FileExist(path) { + confirmAndRemoveDB(path, "full node state database") + } else { + log.Info("Full node state database missing", "path", path) + } + // Remove the full node ancient database + path = config.Eth.DatabaseFreezer + switch { + case path == "": + path = filepath.Join(stack.ResolvePath("chaindata"), "ancient") + case !filepath.IsAbs(path): + path = config.Node.ResolvePath(path) + } + if common.FileExist(path) { + confirmAndRemoveDB(path, "full node ancient database") + } else { + log.Info("Full node ancient database missing", "path", path) + } + // Remove the light node database + path = stack.ResolvePath("lightchaindata") + if common.FileExist(path) { + confirmAndRemoveDB(path, "light node database") + } else { + log.Info("Light node database missing", "path", path) + } + return nil +} + +// confirmAndRemoveDB prompts the user for a last confirmation and removes the +// folder if accepted. +func confirmAndRemoveDB(database string, kind string) { + confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database)) + switch { + case err != nil: + utils.Fatalf("%v", err) + case !confirm: + log.Info("Database deletion skipped", "path", database) + default: + start := time.Now() + filepath.Walk(database, func(path string, info os.FileInfo, err error) error { + // If we're at the top level folder, recurse into + if path == database { + return nil + } + // Delete all the files, but not subfolders + if !info.IsDir() { + os.Remove(path) + return nil + } + return filepath.SkipDir + }) + log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start))) + } +} + +func inspect(ctx *cli.Context) error { + var ( + prefix []byte + start []byte + ) + if ctx.NArg() > 2 { + return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage) + } + if ctx.NArg() >= 1 { + if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { + return fmt.Errorf("failed to hex-decode 'prefix': %v", err) + } else { + prefix = d + } + } + if ctx.NArg() >= 2 { + if d, err := hexutil.Decode(ctx.Args().Get(1)); err != nil { + return fmt.Errorf("failed to hex-decode 'start': %v", err) + } else { + start = d + } + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + _, chainDb := utils.MakeChain(ctx, stack, true) + defer chainDb.Close() + + return rawdb.InspectDatabase(chainDb, prefix, start) +} + +func showLeveldbStats(db ethdb.Stater) { + if stats, err := db.Stat("leveldb.stats"); err != nil { + log.Warn("Failed to read database stats", "error", err) + } else { + fmt.Println(stats) + } + if ioStats, err := db.Stat("leveldb.iostats"); err != nil { + log.Warn("Failed to read database iostats", "error", err) + } else { + fmt.Println(ioStats) + } +} + +func dbStats(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + path := stack.ResolvePath("chaindata") + db, err := leveldb.NewCustom(path, "", func(options *opt.Options) { + options.ReadOnly = true + }) + if err != nil { + return err + } + showLeveldbStats(db) + err = db.Close() + if err != nil { + log.Info("Close err", "error", err) + } + return nil +} + +func dbCompact(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + path := stack.ResolvePath("chaindata") + cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100 + db, err := leveldb.NewCustom(path, "", func(options *opt.Options) { + options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles() + options.BlockCacheCapacity = cache / 2 * opt.MiB + options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally + }) + if err != nil { + return err + } + showLeveldbStats(db) + log.Info("Triggering compaction") + err = db.Compact(nil, nil) + if err != nil { + log.Info("Compact err", "error", err) + } + showLeveldbStats(db) + log.Info("Closing db") + err = db.Close() + if err != nil { + log.Info("Close err", "error", err) + } + log.Info("Exiting") + return err +} + +// dbGet shows the value of a given database key +func dbGet(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + path := stack.ResolvePath("chaindata") + db, err := leveldb.NewCustom(path, "", func(options *opt.Options) { + options.ReadOnly = true + }) + if err != nil { + return err + } + defer db.Close() + key, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the key", "error", err) + return err + } + data, err := db.Get(key) + if err != nil { + log.Info("Get operation failed", "error", err) + return err + } + fmt.Printf("key %#x:\n\t%#x\n", key, data) + return nil +} + +// dbDelete deletes a key from the database +func dbDelete(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + db := utils.MakeChainDatabase(ctx, stack) + defer db.Close() + key, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the key", "error", err) + return err + } + if err = db.Delete(key); err != nil { + log.Info("Delete operation returned an error", "error", err) + return err + } + return nil +} + +// dbPut overwrite a value in the database +func dbPut(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + db := utils.MakeChainDatabase(ctx, stack) + defer db.Close() + var ( + key []byte + value []byte + data []byte + err error + ) + key, err = hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the key", "error", err) + return err + } + value, err = hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + data, err = db.Get(key) + if err == nil { + fmt.Printf("Previous value:\n%#x\n", data) + } + return db.Put(key, value) +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0236ffb7d3..4fa24cc1c2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -238,7 +238,6 @@ func init() { removedbCommand, dumpCommand, dumpGenesisCommand, - inspectCommand, // See accountcmd.go: accountCommand, walletCommand, @@ -254,6 +253,8 @@ func init() { licenseCommand, // See config.go dumpConfigCommand, + // see dbcmd.go + dbCommand, // See cmd/utils/flags_legacy.go utils.ShowDeprecated, // See snapshot.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0d7b0e1bf5..ba643efcc3 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1073,9 +1073,9 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) { } } -// makeDatabaseHandles raises out the number of allowed file handles per process +// MakeDatabaseHandles raises out the number of allowed file handles per process // for Geth and returns half of the allowance to assign to the database. -func makeDatabaseHandles() int { +func MakeDatabaseHandles() int { limit, err := fdlimit.Maximum() if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) @@ -1546,7 +1546,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 } - cfg.DatabaseHandles = makeDatabaseHandles() + cfg.DatabaseHandles = MakeDatabaseHandles() if ctx.GlobalIsSet(AncientFlag.Name) { cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name) } @@ -1821,7 +1821,7 @@ func SplitTagsFlag(tagsFlag string) map[string]string { func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { var ( cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 - handles = makeDatabaseHandles() + handles = MakeDatabaseHandles() err error chainDb ethdb.Database diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 1f8c3f4542..91171ef92c 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -270,8 +270,8 @@ func (s *stat) Count() string { // InspectDatabase traverses the entire database and checks the size // of all different categories of data. -func InspectDatabase(db ethdb.Database) error { - it := db.NewIterator(nil, nil) +func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { + it := db.NewIterator(keyPrefix, keyStart) defer it.Release() var ( @@ -307,8 +307,9 @@ func InspectDatabase(db ethdb.Database) error { bloomTrieNodes stat // Meta- and unaccounted data - metadata stat - unaccounted stat + metadata stat + unaccounted stat + shutdownInfo stat // Totals total common.StorageSize @@ -359,6 +360,8 @@ func InspectDatabase(db ethdb.Database) error { bytes.HasPrefix(key, []byte("bltIndex-")) || bytes.HasPrefix(key, []byte("bltRoot-")): // Bloomtrie sub bloomTrieNodes.Add(size) + case bytes.Equal(key, uncleanShutdownKey): + shutdownInfo.Add(size) default: var accounted bool for _, meta := range [][]byte{ @@ -413,6 +416,7 @@ func InspectDatabase(db ethdb.Database) error { {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, + {"Key-Value store", "Shutdown metadata", shutdownInfo.Size(), shutdownInfo.Count()}, {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()}, {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()}, {"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()}, diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 80380db325..70ac7a91ac 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -84,24 +84,36 @@ type Database struct { // New returns a wrapped LevelDB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. func New(file string, cache int, handles int, namespace string) (*Database, error) { - // Ensure we have some minimal caching and file guarantees - if cache < minCache { - cache = minCache - } - if handles < minHandles { - handles = minHandles - } + return NewCustom(file, namespace, func(options *opt.Options) { + // Ensure we have some minimal caching and file guarantees + if cache < minCache { + cache = minCache + } + if handles < minHandles { + handles = minHandles + } + // Set default options + options.OpenFilesCacheCapacity = handles + options.BlockCacheCapacity = cache / 2 * opt.MiB + options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally + }) +} + +// NewCustom returns a wrapped LevelDB object. The namespace is the prefix that the +// metrics reporting should use for surfacing internal stats. +// The customize function allows the caller to modify the leveldb options. +func NewCustom(file string, namespace string, customize func(options *opt.Options)) (*Database, error) { + options := configureOptions(customize) logger := log.New("database", file) - logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) + usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2 + logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()} + if options.ReadOnly { + logCtx = append(logCtx, "readonly", "true") + } + logger.Info("Allocated cache and file handles", logCtx...) // Open the db and recover any potential corruptions - db, err := leveldb.OpenFile(file, &opt.Options{ - OpenFilesCacheCapacity: handles, - BlockCacheCapacity: cache / 2 * opt.MiB, - WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally - Filter: filter.NewBloomFilter(10), - DisableSeeksCompaction: true, - }) + db, err := leveldb.OpenFile(file, options) if _, corrupted := err.(*errors.ErrCorrupted); corrupted { db, err = leveldb.RecoverFile(file, nil) } @@ -133,6 +145,20 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro return ldb, nil } +// configureOptions sets some default options, then runs the provided setter. +func configureOptions(customizeFn func(*opt.Options)) *opt.Options { + // Set default options + options := &opt.Options{ + Filter: filter.NewBloomFilter(10), + DisableSeeksCompaction: true, + } + // Allow caller to make custom modifications to the options + if customizeFn != nil { + customizeFn(options) + } + return options +} + // Close stops the metrics collection, flushes any pending data to disk and closes // all io accesses to the underlying key-value store. func (db *Database) Close() error { diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index eb5f5547b1..43bbcf0201 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -25,7 +25,7 @@ import ( ) var ( - CommandHelpTemplate = `{{.cmd.Name}}{{if .cmd.Subcommands}} command{{end}}{{if .cmd.Flags}} [command options]{{end}} [arguments...] + CommandHelpTemplate = `{{.cmd.Name}}{{if .cmd.Subcommands}} command{{end}}{{if .cmd.Flags}} [command options]{{end}} {{.cmd.ArgsUsage}} {{if .cmd.Description}}{{.cmd.Description}} {{end}}{{if .cmd.Subcommands}} SUBCOMMANDS: @@ -36,7 +36,7 @@ SUBCOMMANDS: {{end}} {{end}}{{end}}` - OriginCommandHelpTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...] + OriginCommandHelpTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} {{.ArgsUsage}} {{if .Description}}{{.Description}} {{end}}{{if .Subcommands}} SUBCOMMANDS: From 142fbcfd6f4fad825e2ce2684f9d5a487ffb3f84 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 23 Feb 2021 13:09:19 +0100 Subject: [PATCH 152/709] internal/ethapi: reject non-replay-protected txs over RPC (#22339) This PR prevents users from submitting transactions without EIP-155 enabled. This behaviour can be overridden by specifying the flag --rpc.allow-unprotected-txs=true. --- cmd/geth/main.go | 1 + cmd/utils/flags.go | 7 +++++++ eth/api_backend.go | 11 ++++++++--- eth/backend.go | 5 ++++- internal/ethapi/api.go | 4 ++++ internal/ethapi/backend.go | 5 +++-- les/api_backend.go | 11 ++++++++--- les/client.go | 2 +- node/config.go | 3 +++ 9 files changed, 39 insertions(+), 10 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4fa24cc1c2..b0316c5409 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -199,6 +199,7 @@ var ( utils.InsecureUnlockAllowedFlag, utils.RPCGlobalGasCapFlag, utils.RPCGlobalTxFeeCapFlag, + utils.AllowUnprotectedTxs, } whisperFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ba643efcc3..070b3a1de1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -593,6 +593,10 @@ var ( Name: "preload", Usage: "Comma separated list of JavaScript files to preload into the console", } + AllowUnprotectedTxs = cli.BoolFlag{ + Name: "rpc.allow-unprotected-txs", + Usage: "Allow for unprotected (non EIP155 signed) transactions to be submitted via RPC", + } // Network Settings MaxPeersFlag = cli.IntFlag{ @@ -966,6 +970,9 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(HTTPPathPrefixFlag.Name) { cfg.HTTPPathPrefix = ctx.GlobalString(HTTPPathPrefixFlag.Name) } + if ctx.GlobalIsSet(AllowUnprotectedTxs.Name) { + cfg.AllowUnprotectedTxs = ctx.GlobalBool(AllowUnprotectedTxs.Name) + } } // setGraphQL creates the GraphQL listener interface string from the set diff --git a/eth/api_backend.go b/eth/api_backend.go index 17de83a28f..2569972e52 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -41,9 +41,10 @@ import ( // EthAPIBackend implements ethapi.Backend for full nodes type EthAPIBackend struct { - extRPCEnabled bool - eth *Ethereum - gpo *gasprice.Oracle + extRPCEnabled bool + allowUnprotectedTxs bool + eth *Ethereum + gpo *gasprice.Oracle } // ChainConfig returns the active chain configuration. @@ -292,6 +293,10 @@ func (b *EthAPIBackend) ExtRPCEnabled() bool { return b.extRPCEnabled } +func (b *EthAPIBackend) UnprotectedAllowed() bool { + return b.allowUnprotectedTxs +} + func (b *EthAPIBackend) RPCGasCap() uint64 { return b.eth.config.RPCGasCap } diff --git a/eth/backend.go b/eth/backend.go index 4170ecc94e..044422763b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -222,7 +222,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) - eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), eth, nil} + eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} + if eth.APIBackend.allowUnprotectedTxs { + log.Info("Unprotected transactions allowed") + } gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.Miner.GasPrice diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index d3c007b9bf..52b4f5f506 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1555,6 +1555,10 @@ func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (c if err := checkTxFee(tx.GasPrice(), tx.Gas(), b.RPCTxFeeCap()); err != nil { return common.Hash{}, err } + if !b.UnprotectedAllowed() && !tx.Protected() { + // Ensure only eip155 signed transactions are submitted if EIP155Required is set. + return common.Hash{}, errors.New("only replay-protected (EIP-155) transactions allowed over RPC") + } if err := b.SendTx(ctx, tx); err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index f0a4c0493c..ebb088fef5 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -45,8 +45,9 @@ type Backend interface { ChainDb() ethdb.Database AccountManager() *accounts.Manager ExtRPCEnabled() bool - RPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection - RPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs + RPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection + RPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs + UnprotectedAllowed() bool // allows only for EIP155 transactions. // Blockchain API SetHead(number uint64) diff --git a/les/api_backend.go b/les/api_backend.go index 0839614901..f5d2354b60 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -40,9 +40,10 @@ import ( ) type LesApiBackend struct { - extRPCEnabled bool - eth *LightEthereum - gpo *gasprice.Oracle + extRPCEnabled bool + allowUnprotectedTxs bool + eth *LightEthereum + gpo *gasprice.Oracle } func (b *LesApiBackend) ChainConfig() *params.ChainConfig { @@ -263,6 +264,10 @@ func (b *LesApiBackend) ExtRPCEnabled() bool { return b.extRPCEnabled } +func (b *LesApiBackend) UnprotectedAllowed() bool { + return b.allowUnprotectedTxs +} + func (b *LesApiBackend) RPCGasCap() uint64 { return b.eth.config.RPCGasCap } diff --git a/les/client.go b/les/client.go index d08c9feba5..faaf6095e8 100644 --- a/les/client.go +++ b/les/client.go @@ -157,7 +157,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { rawdb.WriteChainConfig(chainDb, genesisHash, chainConfig) } - leth.ApiBackend = &LesApiBackend{stack.Config().ExtRPCEnabled(), leth, nil} + leth.ApiBackend = &LesApiBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, leth, nil} gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.Miner.GasPrice diff --git a/node/config.go b/node/config.go index 447a69505c..ef1da15d70 100644 --- a/node/config.go +++ b/node/config.go @@ -191,6 +191,9 @@ type Config struct { staticNodesWarning bool trustedNodesWarning bool oldGethResourceWarning bool + + // AllowUnprotectedTxs allows non EIP-155 protected transactions to be send over RPC. + AllowUnprotectedTxs bool `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into From 2d1a0e9b03f636babe8785dc833960a5d11e4403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 23 Feb 2021 18:12:25 +0200 Subject: [PATCH 153/709] accounts/abi/bind: fix up Go mod files for Go 1.16 --- accounts/abi/bind/bind_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 400545fd23..d0958cb62f 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1846,11 +1846,16 @@ func TestGolangBindings(t *testing.T) { t.Fatalf("failed to convert binding test to modules: %v\n%s", err, out) } pwd, _ := os.Getwd() - replacer := exec.Command(gocmd, "mod", "edit", "-replace", "github.com/ethereum/go-ethereum="+filepath.Join(pwd, "..", "..", "..")) // Repo root + replacer := exec.Command(gocmd, "mod", "edit", "-x", "-require", "github.com/ethereum/go-ethereum@v0.0.0", "-replace", "github.com/ethereum/go-ethereum="+filepath.Join(pwd, "..", "..", "..")) // Repo root replacer.Dir = pkg if out, err := replacer.CombinedOutput(); err != nil { t.Fatalf("failed to replace binding test dependency to current source tree: %v\n%s", err, out) } + tidier := exec.Command(gocmd, "mod", "tidy") + tidier.Dir = pkg + if out, err := tidier.CombinedOutput(); err != nil { + t.Fatalf("failed to tidy Go module file: %v\n%s", err, out) + } // Test the entire package and report any failures cmd := exec.Command(gocmd, "test", "-v", "-count", "1") cmd.Dir = pkg From 2743fb042945add8dfe4ca782310e123318c7d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 23 Feb 2021 18:28:24 +0200 Subject: [PATCH 154/709] Dockerfile: bump to Go 1.16 base images --- Dockerfile | 2 +- Dockerfile.alltools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d86b776611..6e0dea11b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build Geth in a stock Go builder container -FROM golang:1.15-alpine as builder +FROM golang:1.16-alpine as builder RUN apk add --no-cache make gcc musl-dev linux-headers git diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 715213c5de..483afad8c3 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -1,5 +1,5 @@ # Build Geth in a stock Go builder container -FROM golang:1.15-alpine as builder +FROM golang:1.16-alpine as builder RUN apk add --no-cache make gcc musl-dev linux-headers git From c9aa2670499a874a28c44424c29268889b18d027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 23 Feb 2021 19:57:39 +0200 Subject: [PATCH 155/709] travis: bump Android NDK version --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 39a0456c3b..2e85e5297d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -145,9 +145,9 @@ jobs: - export GOPATH=$HOME/go script: # Build the Android archive and upload it to Maven Central and Azure - - curl https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip -o android-ndk-r19b.zip - - unzip -q android-ndk-r19b.zip && rm android-ndk-r19b.zip - - mv android-ndk-r19b $ANDROID_HOME/ndk-bundle + - curl https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip -o android-ndk-r21e.zip + - unzip -q android-ndk-r21e.zip && rm android-ndk-r21e.zip + - mv android-ndk-r21e $ANDROID_HOME/ndk-bundle - mkdir -p $GOPATH/src/github.com/ethereum - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum From 70afe15f680250e69b459d8d9539f594b5fb7491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 23 Feb 2021 20:31:09 +0200 Subject: [PATCH 156/709] travis: bump builders to Bionic --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e85e5297d..122dee3b36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ jobs: # This builder only tests code linters on latest version of Go - stage: lint os: linux - dist: xenial + dist: bionic go: 1.16.x env: - lint @@ -28,7 +28,7 @@ jobs: - stage: build if: type = push os: linux - dist: xenial + dist: bionic go: 1.16.x env: - ubuntu-ppa @@ -52,7 +52,7 @@ jobs: - stage: build if: type = push os: linux - dist: xenial + dist: bionic sudo: required go: 1.16.x env: @@ -88,7 +88,7 @@ jobs: - stage: build if: type = push os: linux - dist: xenial + dist: bionic services: - docker go: 1.16.x @@ -118,7 +118,7 @@ jobs: - stage: build if: type = push os: linux - dist: xenial + dist: bionic addons: apt: packages: @@ -188,7 +188,7 @@ jobs: - stage: build os: linux arch: amd64 - dist: xenial + dist: bionic go: 1.16.x env: - GO111MODULE=on @@ -199,7 +199,7 @@ jobs: if: type = pull_request os: linux arch: arm64 - dist: xenial + dist: bionic go: 1.16.x env: - GO111MODULE=on @@ -208,7 +208,7 @@ jobs: - stage: build os: linux - dist: xenial + dist: bionic go: 1.15.x env: - GO111MODULE=on @@ -219,7 +219,7 @@ jobs: - stage: build if: type = cron os: linux - dist: xenial + dist: bionic go: 1.16.x env: - azure-purge From f54dc4ab3db0592cf81b3b7ca2ed7a5136ea38a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 24 Feb 2021 11:36:08 +0200 Subject: [PATCH 157/709] travis: manually install Android since Travis is stale (#22373) --- .travis.yml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 122dee3b36..7406f31fe7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -122,16 +122,7 @@ jobs: addons: apt: packages: - - oracle-java8-installer - - oracle-java8-set-default - language: android - android: - components: - - platform-tools - - tools - - android-15 - - android-19 - - android-24 + - openjdk-8-jdk env: - azure-android - maven-android @@ -139,16 +130,24 @@ jobs: git: submodules: false # avoid cloning ethereum/tests before_install: + # Install Android and it's dependencies manually, Travis is stale + - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + - curl https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -o android.zip + - unzip -q android.zip -d $HOME/sdk && rm android.zip + - mv $HOME/sdk/cmdline-tools $HOME/sdk/latest && mkdir $HOME/sdk/cmdline-tools && mv $HOME/sdk/latest $HOME/sdk/cmdline-tools + - export PATH=$PATH:$HOME/sdk/cmdline-tools/latest/bin + - export ANDROID_HOME=$HOME/sdk + + - yes | sdkmanager --licenses >/dev/null + - sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle" + + # Install Go to allow building with - curl https://dl.google.com/go/go1.16.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go script: # Build the Android archive and upload it to Maven Central and Azure - - curl https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip -o android-ndk-r21e.zip - - unzip -q android-ndk-r21e.zip && rm android-ndk-r21e.zip - - mv android-ndk-r21e $ANDROID_HOME/ndk-bundle - - mkdir -p $GOPATH/src/github.com/ethereum - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -signify SIGNIFY_KEY -deploy https://oss.sonatype.org -upload gethstore/builds From 8e547eecd592fe3306e39a4fea703dc1307b8651 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 24 Feb 2021 14:07:58 +0100 Subject: [PATCH 158/709] cmd/utils: remove deprecated command line flags (#22263) This removes support for all deprecated flags except --rpc*. --- README.md | 8 +-- cmd/clef/main.go | 18 ------ cmd/geth/chaincmd.go | 1 - cmd/geth/consolecmd.go | 2 +- cmd/geth/consolecmd_test.go | 4 +- cmd/geth/main.go | 34 +--------- cmd/geth/run_test.go | 2 +- cmd/geth/snapshot.go | 4 -- cmd/geth/usage.go | 14 +--- cmd/puppeth/module_node.go | 2 +- cmd/utils/flags.go | 116 ++++++++------------------------- cmd/utils/flags_legacy.go | 126 +++--------------------------------- internal/debug/flags.go | 55 +--------------- 13 files changed, 53 insertions(+), 333 deletions(-) diff --git a/README.md b/README.md index 57d431db1e..4a083d117a 100644 --- a/README.md +++ b/README.md @@ -314,13 +314,13 @@ ones either). To start a `geth` instance for mining, run it with all your usual by: ```shell -$ geth --mine --miner.threads=1 --etherbase=0x0000000000000000000000000000000000000000 +$ geth --mine --miner.threads=1 --miner.etherbase=0x0000000000000000000000000000000000000000 ``` Which will start mining blocks and transactions on a single CPU thread, crediting all -proceedings to the account specified by `--etherbase`. You can further tune the mining -by changing the default gas limit blocks converge to (`--targetgaslimit`) and the price -transactions are accepted at (`--gasprice`). +proceedings to the account specified by `--miner.etherbase`. You can further tune the mining +by changing the default gas limit blocks converge to (`--miner.targetgaslimit`) and the price +transactions are accepted at (`--miner.gasprice`). ## Contribution diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 090edd4507..dbbb410fb6 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -107,11 +107,6 @@ var ( Usage: "HTTP-RPC server listening port", Value: node.DefaultHTTPPort + 5, } - legacyRPCPortFlag = cli.IntFlag{ - Name: "rpcport", - Usage: "HTTP-RPC server listening port (Deprecated, please use --http.port).", - Value: node.DefaultHTTPPort + 5, - } signerSecretFlag = cli.StringFlag{ Name: "signersecret", Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash", @@ -250,12 +245,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ acceptFlag, }, }, - { - Name: "ALIASED (deprecated)", - Flags: []cli.Flag{ - legacyRPCPortFlag, - }, - }, } func init() { @@ -283,7 +272,6 @@ func init() { testFlag, advancedMode, acceptFlag, - legacyRPCPortFlag, } app.Action = signer app.Commands = []cli.Command{initCommand, @@ -677,12 +665,6 @@ func signer(c *cli.Context) error { // set port port := c.Int(rpcPortFlag.Name) - if c.GlobalIsSet(legacyRPCPortFlag.Name) { - if !c.GlobalIsSet(rpcPortFlag.Name) { - port = c.Int(legacyRPCPortFlag.Name) - } - log.Warn("The flag --rpcport is deprecated and will be removed in the future, please use --http.port") - } // start http server httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.HTTPListenAddrFlag.Name), port) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ba4289ddc3..ff9581fd88 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -163,7 +163,6 @@ The export-preimages command export hash preimages to an RLP encoded stream`, utils.TxLookupLimitFlag, utils.GoerliFlag, utils.YoloV3Flag, - utils.LegacyTestnetFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index cc88b9eec4..e26481003d 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -121,7 +121,7 @@ func remoteConsole(ctx *cli.Context) error { path = ctx.GlobalString(utils.DataDirFlag.Name) } if path != "" { - if ctx.GlobalBool(utils.LegacyTestnetFlag.Name) || ctx.GlobalBool(utils.RopstenFlag.Name) { + if ctx.GlobalBool(utils.RopstenFlag.Name) { // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. legacyPath := filepath.Join(path, "testnet") diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 93f9e3d0d9..c3f41b187c 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -53,7 +53,7 @@ func TestConsoleWelcome(t *testing.T) { coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182" // Start a geth console, make sure it's cleaned up and terminate the console - geth := runMinimalGeth(t, "--etherbase", coinbase, "console") + geth := runMinimalGeth(t, "--miner.etherbase", coinbase, "console") // Gather all the infos the welcome message needs to contain geth.SetTemplateFunc("goos", func() string { return runtime.GOOS }) @@ -100,7 +100,7 @@ func TestAttachWelcome(t *testing.T) { p := trulyRandInt(1024, 65533) // Yeah, sometimes this will fail, sorry :P httpPort = strconv.Itoa(p) wsPort = strconv.Itoa(p + 1) - geth := runMinimalGeth(t, "--etherbase", "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182", + geth := runMinimalGeth(t, "--miner.etherbase", "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182", "--ipcpath", ipc, "--http", "--http.port", httpPort, "--ws", "--ws.port", wsPort) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index b0316c5409..d48bfdd42f 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -61,8 +61,6 @@ var ( utils.UnlockedAccountFlag, utils.PasswordFileFlag, utils.BootnodesFlag, - utils.LegacyBootnodesV4Flag, - utils.LegacyBootnodesV5Flag, utils.DataDirFlag, utils.AncientFlag, utils.MinFreeDiskSpaceFlag, @@ -96,11 +94,9 @@ var ( utils.SnapshotFlag, utils.TxLookupLimitFlag, utils.LightServeFlag, - utils.LegacyLightServFlag, utils.LightIngressFlag, utils.LightEgressFlag, utils.LightMaxPeersFlag, - utils.LegacyLightPeersFlag, utils.LightNoPruneFlag, utils.LightKDFFlag, utils.UltraLightServersFlag, @@ -122,17 +118,12 @@ var ( utils.MaxPendingPeersFlag, utils.MiningEnabledFlag, utils.MinerThreadsFlag, - utils.LegacyMinerThreadsFlag, utils.MinerNotifyFlag, utils.MinerGasTargetFlag, - utils.LegacyMinerGasTargetFlag, utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, - utils.LegacyMinerGasPriceFlag, utils.MinerEtherbaseFlag, - utils.LegacyMinerEtherbaseFlag, utils.MinerExtraDataFlag, - utils.LegacyMinerExtraDataFlag, utils.MinerRecommitIntervalFlag, utils.MinerNoVerfiyFlag, utils.NATFlag, @@ -145,7 +136,6 @@ var ( utils.MainnetFlag, utils.DeveloperFlag, utils.DeveloperPeriodFlag, - utils.LegacyTestnetFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, @@ -158,9 +148,7 @@ var ( utils.FakePoWFlag, utils.NoCompactionFlag, utils.GpoBlocksFlag, - utils.LegacyGpoBlocksFlag, utils.GpoPercentileFlag, - utils.LegacyGpoPercentileFlag, utils.GpoMaxGasPriceFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, @@ -178,22 +166,18 @@ var ( utils.LegacyRPCPortFlag, utils.LegacyRPCCORSDomainFlag, utils.LegacyRPCVirtualHostsFlag, + utils.LegacyRPCApiFlag, utils.GraphQLEnabledFlag, utils.GraphQLCORSDomainFlag, utils.GraphQLVirtualHostsFlag, utils.HTTPApiFlag, utils.HTTPPathPrefixFlag, - utils.LegacyRPCApiFlag, utils.WSEnabledFlag, utils.WSListenAddrFlag, - utils.LegacyWSListenAddrFlag, utils.WSPortFlag, - utils.LegacyWSPortFlag, utils.WSApiFlag, - utils.LegacyWSApiFlag, utils.WSAllowedOriginsFlag, utils.WSPathPrefixFlag, - utils.LegacyWSAllowedOriginsFlag, utils.IPCDisabledFlag, utils.IPCPathFlag, utils.InsecureUnlockAllowedFlag, @@ -267,7 +251,6 @@ func init() { app.Flags = append(app.Flags, rpcFlags...) app.Flags = append(app.Flags, consoleFlags...) app.Flags = append(app.Flags, debug.Flags...) - app.Flags = append(app.Flags, debug.DeprecatedFlags...) app.Flags = append(app.Flags, whisperFlags...) app.Flags = append(app.Flags, metricsFlags...) @@ -293,11 +276,6 @@ func main() { func prepare(ctx *cli.Context) { // If we're running a known preset, log it for convenience. switch { - case ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name): - log.Info("Starting Geth on Ropsten testnet...") - log.Warn("The --testnet flag is ambiguous! Please specify one of --goerli, --rinkeby, or --ropsten.") - log.Warn("The generic --testnet flag is deprecated and will be removed in the future!") - case ctx.GlobalIsSet(utils.RopstenFlag.Name): log.Info("Starting Geth on Ropsten testnet...") @@ -316,7 +294,7 @@ func prepare(ctx *cli.Context) { // If we're a full node on mainnet without --cache specified, bump default cache allowance if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) { // Make sure we're not on any supported preconfigured testnet either - if !ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name) && !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) { + if !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) { // Nope, we're really on mainnet. Bump that cache up! log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096) ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096)) @@ -461,19 +439,11 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend) { if !ok { utils.Fatalf("Ethereum service not running: %v", err) } - // Set the gas price to the limits from the CLI and start mining gasprice := utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) - if ctx.GlobalIsSet(utils.LegacyMinerGasPriceFlag.Name) && !ctx.GlobalIsSet(utils.MinerGasPriceFlag.Name) { - gasprice = utils.GlobalBig(ctx, utils.LegacyMinerGasPriceFlag.Name) - } ethBackend.TxPool().SetGasPrice(gasprice) // start mining threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name) - if ctx.GlobalIsSet(utils.LegacyMinerThreadsFlag.Name) && !ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { - threads = ctx.GlobalInt(utils.LegacyMinerThreadsFlag.Name) - log.Warn("The flag --minerthreads is deprecated and will be removed in the future, please use --miner.threads") - } if err := ethBackend.StartMining(threads); err != nil { utils.Fatalf("Failed to start mining: %v", err) } diff --git a/cmd/geth/run_test.go b/cmd/geth/run_test.go index 79b892c59b..527c38a657 100644 --- a/cmd/geth/run_test.go +++ b/cmd/geth/run_test.go @@ -75,7 +75,7 @@ func runGeth(t *testing.T, args ...string) *testgeth { if i < len(args)-1 { tt.Datadir = args[i+1] } - case "--etherbase": + case "--miner.etherbase": if i < len(args)-1 { tt.Etherbase = args[i+1] } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 6805a42585..b59c530c27 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -60,7 +60,6 @@ var ( utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.LegacyTestnetFlag, utils.CacheTrieJournalFlag, utils.BloomFilterSizeFlag, }, @@ -90,7 +89,6 @@ the trie clean cache with default directory will be deleted. utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.LegacyTestnetFlag, }, Description: ` geth snapshot verify-state @@ -110,7 +108,6 @@ In other words, this command does the snapshot to trie conversion. utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.LegacyTestnetFlag, }, Description: ` geth snapshot traverse-state @@ -132,7 +129,6 @@ It's also usable without snapshot enabled. utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.LegacyTestnetFlag, }, Description: ` geth snapshot traverse-rawstate diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 55e934d31b..cae388c1d3 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -161,8 +161,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ Name: "NETWORKING", Flags: []cli.Flag{ utils.BootnodesFlag, - utils.LegacyBootnodesV4Flag, - utils.LegacyBootnodesV5Flag, utils.DNSDiscoveryFlag, utils.ListenPortFlag, utils.MaxPeersFlag, @@ -223,7 +221,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ }, { Name: "ALIASED (deprecated)", - Flags: append([]cli.Flag{ + Flags: []cli.Flag{ utils.NoUSBFlag, utils.LegacyRPCEnabledFlag, utils.LegacyRPCListenAddrFlag, @@ -231,15 +229,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.LegacyRPCCORSDomainFlag, utils.LegacyRPCVirtualHostsFlag, utils.LegacyRPCApiFlag, - utils.LegacyWSListenAddrFlag, - utils.LegacyWSPortFlag, - utils.LegacyWSAllowedOriginsFlag, - utils.LegacyWSApiFlag, - utils.LegacyGpoBlocksFlag, - utils.LegacyGpoPercentileFlag, - utils.LegacyGraphQLListenAddrFlag, - utils.LegacyGraphQLPortFlag, - }, debug.DeprecatedFlags...), + }, }, { Name: "MISC", diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 5d9ef46523..3ea96870d4 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -94,7 +94,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n lightFlag := "" if config.peersLight > 0 { - lightFlag = fmt.Sprintf("--lightpeers=%d --lightserv=50", config.peersLight) + lightFlag = fmt.Sprintf("--light.maxpeers=%d --light.serve=50", config.peersLight) } dockerfile := new(bytes.Buffer) template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0555acfc75..817041c589 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -765,13 +765,9 @@ var ( // then a subdirectory of the specified datadir will be used. func MakeDataDir(ctx *cli.Context) string { if path := ctx.GlobalString(DataDirFlag.Name); path != "" { - if ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name) { + if ctx.GlobalBool(RopstenFlag.Name) { // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. - legacyPath := filepath.Join(path, "testnet") - if _, err := os.Stat(legacyPath); !os.IsNotExist(err) { - return legacyPath - } return filepath.Join(path, "ropsten") } if ctx.GlobalBool(RinkebyFlag.Name) { @@ -827,13 +823,9 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls := params.MainnetBootnodes switch { - case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(LegacyBootnodesV4Flag.Name): - if ctx.GlobalIsSet(LegacyBootnodesV4Flag.Name) { - urls = SplitAndTrim(ctx.GlobalString(LegacyBootnodesV4Flag.Name)) - } else { - urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) - } - case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name): + case ctx.GlobalIsSet(BootnodesFlag.Name): + urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) + case ctx.GlobalBool(RopstenFlag.Name): urls = params.RopstenBootnodes case ctx.GlobalBool(RinkebyFlag.Name): urls = params.RinkebyBootnodes @@ -863,12 +855,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { urls := params.V5Bootnodes switch { - case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(LegacyBootnodesV5Flag.Name): - if ctx.GlobalIsSet(LegacyBootnodesV5Flag.Name) { - urls = SplitAndTrim(ctx.GlobalString(LegacyBootnodesV5Flag.Name)) - } else { - urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) - } + case ctx.GlobalIsSet(BootnodesFlag.Name): + urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } @@ -921,11 +909,11 @@ func SplitAndTrim(input string) (ret []string) { // command line flags, returning empty if the HTTP endpoint is disabled. func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalBool(LegacyRPCEnabledFlag.Name) && cfg.HTTPHost == "" { - log.Warn("The flag --rpc is deprecated and will be removed in the future, please use --http") + log.Warn("The flag --rpc is deprecated and will be removed June 2021, please use --http") cfg.HTTPHost = "127.0.0.1" if ctx.GlobalIsSet(LegacyRPCListenAddrFlag.Name) { cfg.HTTPHost = ctx.GlobalString(LegacyRPCListenAddrFlag.Name) - log.Warn("The flag --rpcaddr is deprecated and will be removed in the future, please use --http.addr") + log.Warn("The flag --rpcaddr is deprecated and will be removed June 2021, please use --http.addr") } } if ctx.GlobalBool(HTTPEnabledFlag.Name) && cfg.HTTPHost == "" { @@ -937,7 +925,7 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LegacyRPCPortFlag.Name) { cfg.HTTPPort = ctx.GlobalInt(LegacyRPCPortFlag.Name) - log.Warn("The flag --rpcport is deprecated and will be removed in the future, please use --http.port") + log.Warn("The flag --rpcport is deprecated and will be removed June 2021, please use --http.port") } if ctx.GlobalIsSet(HTTPPortFlag.Name) { cfg.HTTPPort = ctx.GlobalInt(HTTPPortFlag.Name) @@ -945,7 +933,7 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LegacyRPCCORSDomainFlag.Name) { cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(LegacyRPCCORSDomainFlag.Name)) - log.Warn("The flag --rpccorsdomain is deprecated and will be removed in the future, please use --http.corsdomain") + log.Warn("The flag --rpccorsdomain is deprecated and will be removed June 2021, please use --http.corsdomain") } if ctx.GlobalIsSet(HTTPCORSDomainFlag.Name) { cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(HTTPCORSDomainFlag.Name)) @@ -953,7 +941,7 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LegacyRPCApiFlag.Name) { cfg.HTTPModules = SplitAndTrim(ctx.GlobalString(LegacyRPCApiFlag.Name)) - log.Warn("The flag --rpcapi is deprecated and will be removed in the future, please use --http.api") + log.Warn("The flag --rpcapi is deprecated and will be removed June 2021, please use --http.api") } if ctx.GlobalIsSet(HTTPApiFlag.Name) { cfg.HTTPModules = SplitAndTrim(ctx.GlobalString(HTTPApiFlag.Name)) @@ -961,7 +949,7 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LegacyRPCVirtualHostsFlag.Name) { cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(LegacyRPCVirtualHostsFlag.Name)) - log.Warn("The flag --rpcvhosts is deprecated and will be removed in the future, please use --http.vhosts") + log.Warn("The flag --rpcvhosts is deprecated and will be removed June 2021, please use --http.vhosts") } if ctx.GlobalIsSet(HTTPVirtualHostsFlag.Name) { cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(HTTPVirtualHostsFlag.Name)) @@ -991,34 +979,18 @@ func setGraphQL(ctx *cli.Context, cfg *node.Config) { func setWS(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalBool(WSEnabledFlag.Name) && cfg.WSHost == "" { cfg.WSHost = "127.0.0.1" - if ctx.GlobalIsSet(LegacyWSListenAddrFlag.Name) { - cfg.WSHost = ctx.GlobalString(LegacyWSListenAddrFlag.Name) - log.Warn("The flag --wsaddr is deprecated and will be removed in the future, please use --ws.addr") - } if ctx.GlobalIsSet(WSListenAddrFlag.Name) { cfg.WSHost = ctx.GlobalString(WSListenAddrFlag.Name) } } - if ctx.GlobalIsSet(LegacyWSPortFlag.Name) { - cfg.WSPort = ctx.GlobalInt(LegacyWSPortFlag.Name) - log.Warn("The flag --wsport is deprecated and will be removed in the future, please use --ws.port") - } if ctx.GlobalIsSet(WSPortFlag.Name) { cfg.WSPort = ctx.GlobalInt(WSPortFlag.Name) } - if ctx.GlobalIsSet(LegacyWSAllowedOriginsFlag.Name) { - cfg.WSOrigins = SplitAndTrim(ctx.GlobalString(LegacyWSAllowedOriginsFlag.Name)) - log.Warn("The flag --wsorigins is deprecated and will be removed in the future, please use --ws.origins") - } if ctx.GlobalIsSet(WSAllowedOriginsFlag.Name) { cfg.WSOrigins = SplitAndTrim(ctx.GlobalString(WSAllowedOriginsFlag.Name)) } - if ctx.GlobalIsSet(LegacyWSApiFlag.Name) { - cfg.WSModules = SplitAndTrim(ctx.GlobalString(LegacyWSApiFlag.Name)) - log.Warn("The flag --wsapi is deprecated and will be removed in the future, please use --ws.api") - } if ctx.GlobalIsSet(WSApiFlag.Name) { cfg.WSModules = SplitAndTrim(ctx.GlobalString(WSApiFlag.Name)) } @@ -1042,10 +1014,6 @@ func setIPC(ctx *cli.Context, cfg *node.Config) { // setLes configures the les server and ultra light client settings from the command line flags. func setLes(ctx *cli.Context, cfg *ethconfig.Config) { - if ctx.GlobalIsSet(LegacyLightServFlag.Name) { - cfg.LightServ = ctx.GlobalInt(LegacyLightServFlag.Name) - log.Warn("The flag --lightserv is deprecated and will be removed in the future, please use --light.serve") - } if ctx.GlobalIsSet(LightServeFlag.Name) { cfg.LightServ = ctx.GlobalInt(LightServeFlag.Name) } @@ -1055,10 +1023,6 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) { if ctx.GlobalIsSet(LightEgressFlag.Name) { cfg.LightEgress = ctx.GlobalInt(LightEgressFlag.Name) } - if ctx.GlobalIsSet(LegacyLightPeersFlag.Name) { - cfg.LightPeers = ctx.GlobalInt(LegacyLightPeersFlag.Name) - log.Warn("The flag --lightpeers is deprecated and will be removed in the future, please use --light.maxpeers") - } if ctx.GlobalIsSet(LightMaxPeersFlag.Name) { cfg.LightPeers = ctx.GlobalInt(LightMaxPeersFlag.Name) } @@ -1122,13 +1086,8 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase either from the directly specified // command line flags or from the keystore if CLI indexed. func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config) { - // Extract the current etherbase, new flag overriding legacy one + // Extract the current etherbase var etherbase string - if ctx.GlobalIsSet(LegacyMinerEtherbaseFlag.Name) { - etherbase = ctx.GlobalString(LegacyMinerEtherbaseFlag.Name) - log.Warn("The flag --etherbase is deprecated and will be removed in the future, please use --miner.etherbase") - - } if ctx.GlobalIsSet(MinerEtherbaseFlag.Name) { etherbase = ctx.GlobalString(MinerEtherbaseFlag.Name) } @@ -1172,27 +1131,24 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setBootstrapNodesV5(ctx, cfg) lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light" - lightServer := (ctx.GlobalInt(LegacyLightServFlag.Name) != 0 || ctx.GlobalInt(LightServeFlag.Name) != 0) + lightServer := (ctx.GlobalInt(LightServeFlag.Name) != 0) - lightPeers := ctx.GlobalInt(LegacyLightPeersFlag.Name) - if ctx.GlobalIsSet(LightMaxPeersFlag.Name) { - lightPeers = ctx.GlobalInt(LightMaxPeersFlag.Name) - } - if lightClient && !ctx.GlobalIsSet(LegacyLightPeersFlag.Name) && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { + lightPeers := ctx.GlobalInt(LightMaxPeersFlag.Name) + if lightClient && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { // dynamic default - for clients we use 1/10th of the default for servers lightPeers /= 10 } if ctx.GlobalIsSet(MaxPeersFlag.Name) { cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name) - if lightServer && !ctx.GlobalIsSet(LegacyLightPeersFlag.Name) && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { + if lightServer && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { cfg.MaxPeers += lightPeers } } else { if lightServer { cfg.MaxPeers += lightPeers } - if lightClient && (ctx.GlobalIsSet(LegacyLightPeersFlag.Name) || ctx.GlobalIsSet(LightMaxPeersFlag.Name)) && cfg.MaxPeers < lightPeers { + if lightClient && ctx.GlobalIsSet(LightMaxPeersFlag.Name) && cfg.MaxPeers < lightPeers { cfg.MaxPeers = lightPeers } } @@ -1297,7 +1253,7 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = ctx.GlobalString(DataDirFlag.Name) case ctx.GlobalBool(DeveloperFlag.Name): cfg.DataDir = "" // unless explicitly requested, use memory databases - case (ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name)) && cfg.DataDir == node.DefaultDataDir(): + case ctx.GlobalBool(RopstenFlag.Name) && cfg.DataDir == node.DefaultDataDir(): // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. legacyPath := filepath.Join(node.DefaultDataDir(), "testnet") @@ -1307,6 +1263,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { } else { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "ropsten") } + + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "ropsten") case ctx.GlobalBool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): @@ -1323,17 +1281,9 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) { cfg.Blocks = ethconfig.LightClientGPO.Blocks cfg.Percentile = ethconfig.LightClientGPO.Percentile } - if ctx.GlobalIsSet(LegacyGpoBlocksFlag.Name) { - cfg.Blocks = ctx.GlobalInt(LegacyGpoBlocksFlag.Name) - log.Warn("The flag --gpoblocks is deprecated and will be removed in the future, please use --gpo.blocks") - } if ctx.GlobalIsSet(GpoBlocksFlag.Name) { cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name) } - if ctx.GlobalIsSet(LegacyGpoPercentileFlag.Name) { - cfg.Percentile = ctx.GlobalInt(LegacyGpoPercentileFlag.Name) - log.Warn("The flag --gpopercentile is deprecated and will be removed in the future, please use --gpo.percentile") - } if ctx.GlobalIsSet(GpoPercentileFlag.Name) { cfg.Percentile = ctx.GlobalInt(GpoPercentileFlag.Name) } @@ -1416,27 +1366,15 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") } - if ctx.GlobalIsSet(LegacyMinerExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(LegacyMinerExtraDataFlag.Name)) - log.Warn("The flag --extradata is deprecated and will be removed in the future, please use --miner.extradata") - } if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } - if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { - cfg.GasFloor = ctx.GlobalUint64(LegacyMinerGasTargetFlag.Name) - log.Warn("The flag --targetgaslimit is deprecated and will be removed in the future, please use --miner.gastarget") - } if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { cfg.GasFloor = ctx.GlobalUint64(MinerGasTargetFlag.Name) } if ctx.GlobalIsSet(MinerGasLimitFlag.Name) { cfg.GasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name) } - if ctx.GlobalIsSet(LegacyMinerGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, LegacyMinerGasPriceFlag.Name) - log.Warn("The flag --gasprice is deprecated and will be removed in the future, please use --miner.gasprice") - } if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) } @@ -1525,11 +1463,11 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, MainnetFlag, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV3Flag) - CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light") + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV3Flag) + CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag) - if (ctx.GlobalIsSet(LegacyLightServFlag.Name) || ctx.GlobalIsSet(LightServeFlag.Name)) && ctx.GlobalIsSet(TxLookupLimitFlag.Name) { + if ctx.GlobalIsSet(LightServeFlag.Name) && ctx.GlobalIsSet(TxLookupLimitFlag.Name) { log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") } var ks *keystore.KeyStore @@ -1644,7 +1582,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = core.DefaultGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) - case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name): + case ctx.GlobalBool(RopstenFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 3 } @@ -1710,7 +1648,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } chaindb.Close() } - if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(LegacyMinerGasPriceFlag.Name) { + if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) { cfg.Miner.GasPrice = big.NewInt(1) } default: @@ -1849,7 +1787,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { func MakeGenesis(ctx *cli.Context) *core.Genesis { var genesis *core.Genesis switch { - case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name): + case ctx.GlobalBool(RopstenFlag.Name): genesis = core.DefaultRopstenGenesisBlock() case ctx.GlobalBool(RinkebyFlag.Name): genesis = core.DefaultRinkebyGenesisBlock() diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index ff45ab9094..fb5fde6576 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -20,7 +20,6 @@ import ( "fmt" "strings" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" "gopkg.in/urfave/cli.v1" ) @@ -34,143 +33,39 @@ var ShowDeprecated = cli.Command{ Description: "Show flags that have been deprecated and will soon be removed", } -var DeprecatedFlags = []cli.Flag{ - LegacyTestnetFlag, - LegacyLightServFlag, - LegacyLightPeersFlag, - LegacyMinerThreadsFlag, - LegacyMinerGasTargetFlag, - LegacyMinerGasPriceFlag, - LegacyMinerEtherbaseFlag, - LegacyMinerExtraDataFlag, -} +var DeprecatedFlags = []cli.Flag{} var ( - // (Deprecated April 2018) - LegacyMinerThreadsFlag = cli.IntFlag{ - Name: "minerthreads", - Usage: "Number of CPU threads to use for mining (deprecated, use --miner.threads)", - Value: 0, - } - LegacyMinerGasTargetFlag = cli.Uint64Flag{ - Name: "targetgaslimit", - Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)", - Value: ethconfig.Defaults.Miner.GasFloor, - } - LegacyMinerGasPriceFlag = BigFlag{ - Name: "gasprice", - Usage: "Minimum gas price for mining a transaction (deprecated, use --miner.gasprice)", - Value: ethconfig.Defaults.Miner.GasPrice, - } - LegacyMinerEtherbaseFlag = cli.StringFlag{ - Name: "etherbase", - Usage: "Public address for block mining rewards (default = first account, deprecated, use --miner.etherbase)", - Value: "0", - } - LegacyMinerExtraDataFlag = cli.StringFlag{ - Name: "extradata", - Usage: "Block extra data set by the miner (default = client version, deprecated, use --miner.extradata)", - } - - // (Deprecated June 2019) - LegacyLightServFlag = cli.IntFlag{ - Name: "lightserv", - Usage: "Maximum percentage of time allowed for serving LES requests (deprecated, use --light.serve)", - Value: ethconfig.Defaults.LightServ, - } - LegacyLightPeersFlag = cli.IntFlag{ - Name: "lightpeers", - Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated, use --light.maxpeers)", - Value: ethconfig.Defaults.LightPeers, - } - - // (Deprecated April 2020) - LegacyTestnetFlag = cli.BoolFlag{ // TODO(q9f): Remove after Ropsten is discontinued. - Name: "testnet", - Usage: "Pre-configured test network (Deprecated: Please choose one of --goerli, --rinkeby, or --ropsten.)", - } - // (Deprecated May 2020, shown in aliased flags section) LegacyRPCEnabledFlag = cli.BoolFlag{ Name: "rpc", - Usage: "Enable the HTTP-RPC server (deprecated, use --http)", + Usage: "Enable the HTTP-RPC server (deprecated and will be removed June 2021, use --http)", } LegacyRPCListenAddrFlag = cli.StringFlag{ Name: "rpcaddr", - Usage: "HTTP-RPC server listening interface (deprecated, use --http.addr)", + Usage: "HTTP-RPC server listening interface (deprecated and will be removed June 2021, use --http.addr)", Value: node.DefaultHTTPHost, } LegacyRPCPortFlag = cli.IntFlag{ Name: "rpcport", - Usage: "HTTP-RPC server listening port (deprecated, use --http.port)", + Usage: "HTTP-RPC server listening port (deprecated and will be removed June 2021, use --http.port)", Value: node.DefaultHTTPPort, } LegacyRPCCORSDomainFlag = cli.StringFlag{ Name: "rpccorsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced) (deprecated, use --http.corsdomain)", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced) (deprecated and will be removed June 2021, use --http.corsdomain)", Value: "", } LegacyRPCVirtualHostsFlag = cli.StringFlag{ Name: "rpcvhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (deprecated, use --http.vhosts)", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (deprecated and will be removed June 2021, use --http.vhosts)", Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), } LegacyRPCApiFlag = cli.StringFlag{ Name: "rpcapi", - Usage: "API's offered over the HTTP-RPC interface (deprecated, use --http.api)", - Value: "", - } - LegacyWSListenAddrFlag = cli.StringFlag{ - Name: "wsaddr", - Usage: "WS-RPC server listening interface (deprecated, use --ws.addr)", - Value: node.DefaultWSHost, - } - LegacyWSPortFlag = cli.IntFlag{ - Name: "wsport", - Usage: "WS-RPC server listening port (deprecated, use --ws.port)", - Value: node.DefaultWSPort, - } - LegacyWSApiFlag = cli.StringFlag{ - Name: "wsapi", - Usage: "API's offered over the WS-RPC interface (deprecated, use --ws.api)", + Usage: "API's offered over the HTTP-RPC interface (deprecated and will be removed June 2021, use --http.api)", Value: "", } - LegacyWSAllowedOriginsFlag = cli.StringFlag{ - Name: "wsorigins", - Usage: "Origins from which to accept websockets requests (deprecated, use --ws.origins)", - Value: "", - } - LegacyGpoBlocksFlag = cli.IntFlag{ - Name: "gpoblocks", - Usage: "Number of recent blocks to check for gas prices (deprecated, use --gpo.blocks)", - Value: ethconfig.Defaults.GPO.Blocks, - } - LegacyGpoPercentileFlag = cli.IntFlag{ - Name: "gpopercentile", - Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices (deprecated, use --gpo.percentile)", - Value: ethconfig.Defaults.GPO.Percentile, - } - LegacyBootnodesV4Flag = cli.StringFlag{ - Name: "bootnodesv4", - Usage: "Comma separated enode URLs for P2P v4 discovery bootstrap (light server, full nodes) (deprecated, use --bootnodes)", - Value: "", - } - LegacyBootnodesV5Flag = cli.StringFlag{ - Name: "bootnodesv5", - Usage: "Comma separated enode URLs for P2P v5 discovery bootstrap (light server, light nodes) (deprecated, use --bootnodes)", - Value: "", - } - - // (Deprecated July 2020, shown in aliased flags section) - LegacyGraphQLListenAddrFlag = cli.StringFlag{ - Name: "graphql.addr", - Usage: "GraphQL server listening interface (deprecated, graphql can only be enabled on the HTTP-RPC server endpoint, use --graphql)", - } - LegacyGraphQLPortFlag = cli.IntFlag{ - Name: "graphql.port", - Usage: "GraphQL server listening port (deprecated, graphql can only be enabled on the HTTP-RPC server endpoint, use --graphql)", - Value: node.DefaultHTTPPort, - } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. @@ -179,8 +74,7 @@ func showDeprecated(*cli.Context) { fmt.Println("The following flags are deprecated and will be removed in the future!") fmt.Println("--------------------------------------------------------------------") fmt.Println() - - for _, flag := range DeprecatedFlags { - fmt.Println(flag.String()) - } + // TODO remove when there are newly deprecated flags + fmt.Println("no deprecated flags to show at this time") + fmt.Println() } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index fc4ea9f518..2c92b19de6 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -90,30 +90,6 @@ var ( Name: "trace", Usage: "Write execution trace to the given file", } - // (Deprecated April 2020) - legacyPprofPortFlag = cli.IntFlag{ - Name: "pprofport", - Usage: "pprof HTTP server listening port (deprecated, use --pprof.port)", - Value: 6060, - } - legacyPprofAddrFlag = cli.StringFlag{ - Name: "pprofaddr", - Usage: "pprof HTTP server listening interface (deprecated, use --pprof.addr)", - Value: "127.0.0.1", - } - legacyMemprofilerateFlag = cli.IntFlag{ - Name: "memprofilerate", - Usage: "Turn on memory profiling with the given rate (deprecated, use --pprof.memprofilerate)", - Value: runtime.MemProfileRate, - } - legacyBlockprofilerateFlag = cli.IntFlag{ - Name: "blockprofilerate", - Usage: "Turn on block profiling with the given rate (deprecated, use --pprof.blockprofilerate)", - } - legacyCpuprofileFlag = cli.StringFlag{ - Name: "cpuprofile", - Usage: "Write CPU profile to the given file (deprecated, use --pprof.cpuprofile)", - } ) // Flags holds all command-line flags required for debugging. @@ -123,12 +99,9 @@ var Flags = []cli.Flag{ blockprofilerateFlag, cpuprofileFlag, traceFlag, } -var DeprecatedFlags = []cli.Flag{ - legacyPprofPortFlag, legacyPprofAddrFlag, legacyMemprofilerateFlag, - legacyBlockprofilerateFlag, legacyCpuprofileFlag, -} - -var glogger *log.GlogHandler +var ( + glogger *log.GlogHandler +) func init() { glogger = log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) @@ -159,16 +132,8 @@ func Setup(ctx *cli.Context) error { log.Root().SetHandler(glogger) // profiling, tracing - if ctx.GlobalIsSet(legacyMemprofilerateFlag.Name) { - runtime.MemProfileRate = ctx.GlobalInt(legacyMemprofilerateFlag.Name) - log.Warn("The flag --memprofilerate is deprecated and will be removed in the future, please use --pprof.memprofilerate") - } runtime.MemProfileRate = ctx.GlobalInt(memprofilerateFlag.Name) - if ctx.GlobalIsSet(legacyBlockprofilerateFlag.Name) { - Handler.SetBlockProfileRate(ctx.GlobalInt(legacyBlockprofilerateFlag.Name)) - log.Warn("The flag --blockprofilerate is deprecated and will be removed in the future, please use --pprof.blockprofilerate") - } Handler.SetBlockProfileRate(ctx.GlobalInt(blockprofilerateFlag.Name)) if traceFile := ctx.GlobalString(traceFlag.Name); traceFile != "" { @@ -182,26 +147,12 @@ func Setup(ctx *cli.Context) error { return err } } - if cpuFile := ctx.GlobalString(legacyCpuprofileFlag.Name); cpuFile != "" { - log.Warn("The flag --cpuprofile is deprecated and will be removed in the future, please use --pprof.cpuprofile") - if err := Handler.StartCPUProfile(cpuFile); err != nil { - return err - } - } // pprof server if ctx.GlobalBool(pprofFlag.Name) { listenHost := ctx.GlobalString(pprofAddrFlag.Name) - if ctx.GlobalIsSet(legacyPprofAddrFlag.Name) && !ctx.GlobalIsSet(pprofAddrFlag.Name) { - listenHost = ctx.GlobalString(legacyPprofAddrFlag.Name) - log.Warn("The flag --pprofaddr is deprecated and will be removed in the future, please use --pprof.addr") - } port := ctx.GlobalInt(pprofPortFlag.Name) - if ctx.GlobalIsSet(legacyPprofPortFlag.Name) && !ctx.GlobalIsSet(pprofPortFlag.Name) { - port = ctx.GlobalInt(legacyPprofPortFlag.Name) - log.Warn("The flag --pprofport is deprecated and will be removed in the future, please use --pprof.port") - } address := fmt.Sprintf("%s:%d", listenHost, port) // This context value ("metrics.addr") represents the utils.MetricsHTTPFlag.Name. From b2b5c82acaa89387960805d53359629e854814bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 25 Feb 2021 12:56:18 +0200 Subject: [PATCH 159/709] eth/protocols/snap: lower abortion and resumption logs to debug --- eth/protocols/snap/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index c31e4a5dae..1cfdef15bd 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -1595,7 +1595,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { // is interrupted and resumed later. However, *do* update the // previous root hash. if subtasks, ok := res.task.SubTasks[res.hashes[i]]; ok { - log.Error("Resuming large storage retrieval", "account", res.hashes[i], "root", account.Root) + log.Debug("Resuming large storage retrieval", "account", res.hashes[i], "root", account.Root) for _, subtask := range subtasks { subtask.root = account.Root } @@ -1614,7 +1614,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { // now we have to live with that. for hash := range res.task.SubTasks { if _, ok := resumed[hash]; !ok { - log.Error("Aborting suspended storage retrieval", "account", hash) + log.Debug("Aborting suspended storage retrieval", "account", hash) delete(res.task.SubTasks, hash) } } From 378e961d857e02a1ce032727da08dfebf2d96cac Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 25 Feb 2021 20:55:07 +0800 Subject: [PATCH 160/709] cmd, eth, les: enable serving light clients when non-synced (#22250) This PR adds a more CLI flag, so that the les-server can serve light clients even the local node is not synced yet. This functionality is needed in some testing environments(e.g. hive). After launching the les server, no more blocks will be imported so the node is always marked as "non-synced". --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 +++++++ eth/ethconfig/config.go | 1 + eth/ethconfig/gen_config.go | 6 ++++++ les/server.go | 6 +++++- 6 files changed, 21 insertions(+), 1 deletion(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d48bfdd42f..b4c622ce2e 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -102,6 +102,7 @@ var ( utils.UltraLightServersFlag, utils.UltraLightFractionFlag, utils.UltraLightOnlyAnnounceFlag, + utils.LightNoSyncServeFlag, utils.WhitelistFlag, utils.BloomFilterSizeFlag, utils.CacheFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index cae388c1d3..24215f55a8 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -68,6 +68,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.UltraLightFractionFlag, utils.UltraLightOnlyAnnounceFlag, utils.LightNoPruneFlag, + utils.LightNoSyncServeFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 817041c589..9fff183a14 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -269,6 +269,10 @@ var ( Name: "light.nopruning", Usage: "Disable ancient light chain data pruning", } + LightNoSyncServeFlag = cli.BoolFlag{ + Name: "light.nosyncserve", + Usage: "Enables serving light clients before syncing", + } // Ethash settings EthashCacheDirFlag = DirectoryFlag{ Name: "ethash.cachedir", @@ -1042,6 +1046,9 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) { if ctx.GlobalIsSet(LightNoPruneFlag.Name) { cfg.LightNoPrune = ctx.GlobalBool(LightNoPruneFlag.Name) } + if ctx.GlobalIsSet(LightNoSyncServeFlag.Name) { + cfg.LightNoSyncServe = ctx.GlobalBool(LightNoSyncServeFlag.Name) + } } // MakeDatabaseHandles raises out the number of allowed file handles per process diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index e192e4d333..841dc5e9e1 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -140,6 +140,7 @@ type Config struct { LightEgress int `toml:",omitempty"` // Outgoing bandwidth limit for light servers LightPeers int `toml:",omitempty"` // Maximum number of LES client peers LightNoPrune bool `toml:",omitempty"` // Whether to disable light chain pruning + LightNoSyncServe bool `toml:",omitempty"` // Whether to serve light clients before syncing SyncFromCheckpoint bool `toml:",omitempty"` // Whether to sync the header chain from the configured checkpoint // Ultra Light client options diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 5814b81b09..ca93b2ad00 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -31,6 +31,7 @@ func (c Config) MarshalTOML() (interface{}, error) { LightEgress int `toml:",omitempty"` LightPeers int `toml:",omitempty"` LightNoPrune bool `toml:",omitempty"` + LightNoSyncServe bool `toml:",omitempty"` SyncFromCheckpoint bool `toml:",omitempty"` UltraLightServers []string `toml:",omitempty"` UltraLightFraction int `toml:",omitempty"` @@ -74,6 +75,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.LightEgress = c.LightEgress enc.LightPeers = c.LightPeers enc.LightNoPrune = c.LightNoPrune + enc.LightNoSyncServe = c.LightNoSyncServe enc.SyncFromCheckpoint = c.SyncFromCheckpoint enc.UltraLightServers = c.UltraLightServers enc.UltraLightFraction = c.UltraLightFraction @@ -121,6 +123,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { LightEgress *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` LightNoPrune *bool `toml:",omitempty"` + LightNoSyncServe *bool `toml:",omitempty"` SyncFromCheckpoint *bool `toml:",omitempty"` UltraLightServers []string `toml:",omitempty"` UltraLightFraction *int `toml:",omitempty"` @@ -195,6 +198,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightNoPrune != nil { c.LightNoPrune = *dec.LightNoPrune } + if dec.LightNoSyncServe != nil { + c.LightNoSyncServe = *dec.LightNoSyncServe + } if dec.SyncFromCheckpoint != nil { c.SyncFromCheckpoint = *dec.SyncFromCheckpoint } diff --git a/les/server.go b/les/server.go index e34647f290..359784cf7d 100644 --- a/les/server.go +++ b/les/server.go @@ -118,7 +118,11 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les threadsIdle: threads, p2pSrv: node.Server(), } - srv.handler = newServerHandler(srv, e.BlockChain(), e.ChainDb(), e.TxPool(), e.Synced) + issync := e.Synced + if config.LightNoSyncServe { + issync = func() bool { return true } + } + srv.handler = newServerHandler(srv, e.BlockChain(), e.ChainDb(), e.TxPool(), issync) srv.costTracker, srv.minCapacity = newCostTracker(e.ChainDb(), config) srv.oracle = srv.setupOracle(node, e.BlockChain().Genesis().Hash(), config) From 7a3c890009535bc3b87b01d9af19566e654be9da Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 25 Feb 2021 21:24:04 +0800 Subject: [PATCH 161/709] les, light: improve txstatus retrieval (#22349) Transaction unindexing will be enabled by default as of 1.10, which causes tx status retrieval will be broken without this PR. This PR introduces a retry mechanism in TxStatus retrieval. --- les/client.go | 2 +- les/fetcher_test.go | 35 ++++- les/handler_test.go | 218 +++++++++++++++++++++-------- les/odr.go | 101 +++++++++++++- les/odr_requests.go | 5 +- les/odr_test.go | 192 ++++++++++++++++++++++++- les/peer.go | 12 +- les/pruner_test.go | 29 ++-- les/request_test.go | 9 +- les/sync_test.go | 56 ++++++-- les/test_helper.go | 334 ++++++++++++++++++++++++++------------------ les/ulc_test.go | 15 +- light/odr.go | 1 + light/odr_util.go | 9 +- 14 files changed, 776 insertions(+), 242 deletions(-) diff --git a/les/client.go b/les/client.go index faaf6095e8..053118df5a 100644 --- a/les/client.go +++ b/les/client.go @@ -122,7 +122,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.getTimeout) leth.relay = newLesTxRelay(peers, leth.retriever) - leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.retriever) + leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.peers, leth.retriever) leth.chtIndexer = light.NewChtIndexer(chainDb, leth.odr, params.CHTFrequency, params.HelperTrieConfirmations, config.LightNoPrune) leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, leth.odr, params.BloomBitsBlocksClient, params.BloomTrieFrequency, config.LightNoPrune) leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) diff --git a/les/fetcher_test.go b/les/fetcher_test.go index a9e6e6835e..d3a74d25c2 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -66,7 +66,12 @@ func TestSequentialAnnouncementsLes2(t *testing.T) { testSequentialAnnouncements func TestSequentialAnnouncementsLes3(t *testing.T) { testSequentialAnnouncements(t, 3) } func testSequentialAnnouncements(t *testing.T, protocol int) { - s, c, teardown := newClientServerEnv(t, 4, protocol, nil, nil, 0, false, false, true) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + nopruning: true, + } + s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() // Create connected peer pair. @@ -101,7 +106,12 @@ func TestGappedAnnouncementsLes2(t *testing.T) { testGappedAnnouncements(t, 2) } func TestGappedAnnouncementsLes3(t *testing.T) { testGappedAnnouncements(t, 3) } func testGappedAnnouncements(t *testing.T, protocol int) { - s, c, teardown := newClientServerEnv(t, 4, protocol, nil, nil, 0, false, false, true) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + nopruning: true, + } + s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() // Create connected peer pair. @@ -183,7 +193,13 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { ids = append(ids, n.String()) } } - _, c, teardown := newClientServerEnv(t, 0, protocol, nil, ids, 60, false, false, true) + netconfig := testnetConfig{ + protocol: protocol, + nopruning: true, + ulcServers: ids, + ulcFraction: 60, + } + _, c, teardown := newClientServerEnv(t, netconfig) defer teardown() defer func() { for i := 0; i < len(teardowns); i++ { @@ -233,8 +249,17 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { check([]uint64{10}, 10, func() { <-newHead }) // Sync the whole chain. } -func TestInvalidAnnounces(t *testing.T) { - s, c, teardown := newClientServerEnv(t, 4, lpv3, nil, nil, 0, false, false, true) +func TestInvalidAnnouncesLES2(t *testing.T) { testInvalidAnnounces(t, lpv2) } +func TestInvalidAnnouncesLES3(t *testing.T) { testInvalidAnnounces(t, lpv3) } +func TestInvalidAnnouncesLES4(t *testing.T) { testInvalidAnnounces(t, lpv4) } + +func testInvalidAnnounces(t *testing.T, protocol int) { + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + nopruning: true, + } + s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() // Create connected peer pair. diff --git a/les/handler_test.go b/les/handler_test.go index e251f4503b..d1dbee6bdf 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -52,9 +52,16 @@ func TestGetBlockHeadersLes3(t *testing.T) { testGetBlockHeaders(t, 3) } func TestGetBlockHeadersLes4(t *testing.T) { testGetBlockHeaders(t, 4) } func testGetBlockHeaders(t *testing.T, protocol int) { - server, tearDown := newServerEnv(t, downloader.MaxHeaderFetch+15, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: downloader.MaxHeaderFetch + 15, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() bc := server.handler.blockchain // Create a "random" unknown hash for testing @@ -169,8 +176,8 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Send the hash request and verify the response reqID++ - sendRequest(server.peer.app, GetBlockHeadersMsg, reqID, tt.query) - if err := expectResponse(server.peer.app, BlockHeadersMsg, reqID, testBufLimit, headers); err != nil { + sendRequest(rawPeer.app, GetBlockHeadersMsg, reqID, tt.query) + if err := expectResponse(rawPeer.app, BlockHeadersMsg, reqID, testBufLimit, headers); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } } @@ -182,9 +189,17 @@ func TestGetBlockBodiesLes3(t *testing.T) { testGetBlockBodies(t, 3) } func TestGetBlockBodiesLes4(t *testing.T) { testGetBlockBodies(t, 4) } func testGetBlockBodies(t *testing.T, protocol int) { - server, tearDown := newServerEnv(t, downloader.MaxBlockFetch+15, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: downloader.MaxHeaderFetch + 15, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain // Create a batch of tests for various scenarios @@ -247,8 +262,8 @@ func testGetBlockBodies(t *testing.T, protocol int) { reqID++ // Send the hash request and verify the response - sendRequest(server.peer.app, GetBlockBodiesMsg, reqID, hashes) - if err := expectResponse(server.peer.app, BlockBodiesMsg, reqID, testBufLimit, bodies); err != nil { + sendRequest(rawPeer.app, GetBlockBodiesMsg, reqID, hashes) + if err := expectResponse(rawPeer.app, BlockBodiesMsg, reqID, testBufLimit, bodies); err != nil { t.Errorf("test %d: bodies mismatch: %v", i, err) } } @@ -261,8 +276,17 @@ func TestGetCodeLes4(t *testing.T) { testGetCode(t, 4) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment - server, tearDown := newServerEnv(t, 4, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain var codereqs []*CodeReq @@ -279,8 +303,8 @@ func testGetCode(t *testing.T, protocol int) { } } - sendRequest(server.peer.app, GetCodeMsg, 42, codereqs) - if err := expectResponse(server.peer.app, CodeMsg, 42, testBufLimit, codes); err != nil { + sendRequest(rawPeer.app, GetCodeMsg, 42, codereqs) + if err := expectResponse(rawPeer.app, CodeMsg, 42, testBufLimit, codes); err != nil { t.Errorf("codes mismatch: %v", err) } } @@ -291,8 +315,17 @@ func TestGetStaleCodeLes3(t *testing.T) { testGetStaleCode(t, 3) } func TestGetStaleCodeLes4(t *testing.T) { testGetStaleCode(t, 4) } func testGetStaleCode(t *testing.T, protocol int) { - server, tearDown := newServerEnv(t, core.TriesInMemory+4, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: core.TriesInMemory + 4, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain check := func(number uint64, expected [][]byte) { @@ -300,8 +333,8 @@ func testGetStaleCode(t *testing.T, protocol int) { BHash: bc.GetHeaderByNumber(number).Hash(), AccKey: crypto.Keccak256(testContractAddr[:]), } - sendRequest(server.peer.app, GetCodeMsg, 42, []*CodeReq{req}) - if err := expectResponse(server.peer.app, CodeMsg, 42, testBufLimit, expected); err != nil { + sendRequest(rawPeer.app, GetCodeMsg, 42, []*CodeReq{req}) + if err := expectResponse(rawPeer.app, CodeMsg, 42, testBufLimit, expected); err != nil { t.Errorf("codes mismatch: %v", err) } } @@ -317,9 +350,17 @@ func TestGetReceiptLes4(t *testing.T) { testGetReceipt(t, 4) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment - server, tearDown := newServerEnv(t, 4, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain // Collect the hashes to request, and the response to expect @@ -332,8 +373,8 @@ func testGetReceipt(t *testing.T, protocol int) { receipts = append(receipts, rawdb.ReadRawReceipts(server.db, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response - sendRequest(server.peer.app, GetReceiptsMsg, 42, hashes) - if err := expectResponse(server.peer.app, ReceiptsMsg, 42, testBufLimit, receipts); err != nil { + sendRequest(rawPeer.app, GetReceiptsMsg, 42, hashes) + if err := expectResponse(rawPeer.app, ReceiptsMsg, 42, testBufLimit, receipts); err != nil { t.Errorf("receipts mismatch: %v", err) } } @@ -345,9 +386,17 @@ func TestGetProofsLes4(t *testing.T) { testGetProofs(t, 4) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment - server, tearDown := newServerEnv(t, 4, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain var proofreqs []ProofReq @@ -368,8 +417,8 @@ func testGetProofs(t *testing.T, protocol int) { } } // Send the proof request and verify the response - sendRequest(server.peer.app, GetProofsV2Msg, 42, proofreqs) - if err := expectResponse(server.peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { + sendRequest(rawPeer.app, GetProofsV2Msg, 42, proofreqs) + if err := expectResponse(rawPeer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -380,8 +429,17 @@ func TestGetStaleProofLes3(t *testing.T) { testGetStaleProof(t, 3) } func TestGetStaleProofLes4(t *testing.T) { testGetStaleProof(t, 4) } func testGetStaleProof(t *testing.T, protocol int) { - server, tearDown := newServerEnv(t, core.TriesInMemory+4, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + blocks: core.TriesInMemory + 4, + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain check := func(number uint64, wantOK bool) { @@ -393,7 +451,7 @@ func testGetStaleProof(t *testing.T, protocol int) { BHash: header.Hash(), Key: account, } - sendRequest(server.peer.app, GetProofsV2Msg, 42, []*ProofReq{req}) + sendRequest(rawPeer.app, GetProofsV2Msg, 42, []*ProofReq{req}) var expected []rlp.RawValue if wantOK { @@ -402,7 +460,7 @@ func testGetStaleProof(t *testing.T, protocol int) { t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } - if err := expectResponse(server.peer.app, ProofsV2Msg, 42, testBufLimit, expected); err != nil { + if err := expectResponse(rawPeer.app, ProofsV2Msg, 42, testBufLimit, expected); err != nil { t.Errorf("codes mismatch: %v", err) } } @@ -417,20 +475,30 @@ func TestGetCHTProofsLes3(t *testing.T) { testGetCHTProofs(t, 3) } func TestGetCHTProofsLes4(t *testing.T) { testGetCHTProofs(t, 4) } func testGetCHTProofs(t *testing.T, protocol int) { - config := light.TestServerIndexerConfig - - waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { - for { - cs, _, _ := cIndexer.Sections() - if cs >= 1 { - break + var ( + config = light.TestServerIndexerConfig + waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + cs, _, _ := cIndexer.Sections() + if cs >= 1 { + break + } + time.Sleep(10 * time.Millisecond) } - time.Sleep(10 * time.Millisecond) } - } - server, tearDown := newServerEnv(t, int(config.ChtSize+config.ChtConfirms), protocol, waitIndexers, false, true, 0) + netconfig = testnetConfig{ + blocks: int(config.ChtSize + config.ChtConfirms), + protocol: protocol, + indexFn: waitIndexers, + nopruning: true, + } + ) + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain // Assemble the proofs from the different protocols @@ -454,8 +522,8 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxReq: htAuxHeader, }} // Send the proof request and verify the response - sendRequest(server.peer.app, GetHelperTrieProofsMsg, 42, requestsV2) - if err := expectResponse(server.peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { + sendRequest(rawPeer.app, GetHelperTrieProofsMsg, 42, requestsV2) + if err := expectResponse(rawPeer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -466,20 +534,30 @@ func TestGetBloombitsProofsLes4(t *testing.T) { testGetBloombitsProofs(t, 4) } // Tests that bloombits proofs can be correctly retrieved. func testGetBloombitsProofs(t *testing.T, protocol int) { - config := light.TestServerIndexerConfig - - waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { - for { - bts, _, _ := btIndexer.Sections() - if bts >= 1 { - break + var ( + config = light.TestServerIndexerConfig + waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + bts, _, _ := btIndexer.Sections() + if bts >= 1 { + break + } + time.Sleep(10 * time.Millisecond) } - time.Sleep(10 * time.Millisecond) } - } - server, tearDown := newServerEnv(t, int(config.BloomTrieSize+config.BloomTrieConfirms), protocol, waitIndexers, false, true, 0) + netconfig = testnetConfig{ + blocks: int(config.BloomTrieSize + config.BloomTrieConfirms), + protocol: protocol, + indexFn: waitIndexers, + nopruning: true, + } + ) + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + bc := server.handler.blockchain // Request and verify each bit of the bloom bits proofs @@ -503,20 +581,28 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response - sendRequest(server.peer.app, GetHelperTrieProofsMsg, 42, requests) - if err := expectResponse(server.peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { + sendRequest(rawPeer.app, GetHelperTrieProofsMsg, 42, requests) + if err := expectResponse(rawPeer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { t.Errorf("bit %d: proofs mismatch: %v", bit, err) } } } -func TestTransactionStatusLes2(t *testing.T) { testTransactionStatus(t, 2) } -func TestTransactionStatusLes3(t *testing.T) { testTransactionStatus(t, 3) } -func TestTransactionStatusLes4(t *testing.T) { testTransactionStatus(t, 4) } +func TestTransactionStatusLes2(t *testing.T) { testTransactionStatus(t, lpv2) } +func TestTransactionStatusLes3(t *testing.T) { testTransactionStatus(t, lpv3) } +func TestTransactionStatusLes4(t *testing.T) { testTransactionStatus(t, lpv4) } func testTransactionStatus(t *testing.T, protocol int) { - server, tearDown := newServerEnv(t, 0, protocol, nil, false, true, 0) + netconfig := testnetConfig{ + protocol: protocol, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() + + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() + server.handler.addTxsSync = true chain := server.handler.blockchain @@ -526,11 +612,11 @@ func testTransactionStatus(t *testing.T, protocol int) { test := func(tx *types.Transaction, send bool, expStatus light.TxStatus) { reqID++ if send { - sendRequest(server.peer.app, SendTxV2Msg, reqID, types.Transactions{tx}) + sendRequest(rawPeer.app, SendTxV2Msg, reqID, types.Transactions{tx}) } else { - sendRequest(server.peer.app, GetTxStatusMsg, reqID, []common.Hash{tx.Hash()}) + sendRequest(rawPeer.app, GetTxStatusMsg, reqID, []common.Hash{tx.Hash()}) } - if err := expectResponse(server.peer.app, TxStatusMsg, reqID, testBufLimit, []light.TxStatus{expStatus}); err != nil { + if err := expectResponse(rawPeer.app, TxStatusMsg, reqID, testBufLimit, []light.TxStatus{expStatus}); err != nil { t.Errorf("transaction status mismatch") } } @@ -572,7 +658,7 @@ func testTransactionStatus(t *testing.T, protocol int) { t.Fatalf("pending count mismatch: have %d, want 1", pending) } // Discard new block announcement - msg, _ := server.peer.app.ReadMsg() + msg, _ := rawPeer.app.ReadMsg() msg.Discard() // check if their status is included now @@ -597,7 +683,7 @@ func testTransactionStatus(t *testing.T, protocol int) { t.Fatalf("pending count mismatch: have %d, want 3", pending) } // Discard new block announcement - msg, _ = server.peer.app.ReadMsg() + msg, _ = rawPeer.app.ReadMsg() msg.Discard() // check if their status is pending again @@ -605,11 +691,23 @@ func testTransactionStatus(t *testing.T, protocol int) { test(tx2, false, light.TxStatus{Status: core.TxStatusPending}) } -func TestStopResumeLes3(t *testing.T) { - server, tearDown := newServerEnv(t, 0, 3, nil, true, true, testBufLimit/10) +func TestStopResumeLES3(t *testing.T) { testStopResume(t, lpv3) } +func TestStopResumeLES4(t *testing.T) { testStopResume(t, lpv4) } + +func testStopResume(t *testing.T, protocol int) { + netconfig := testnetConfig{ + protocol: protocol, + simClock: true, + nopruning: true, + } + server, _, tearDown := newClientServerEnv(t, netconfig) defer tearDown() server.handler.server.costTracker.testing = true + server.handler.server.costTracker.testCostList = testCostList(testBufLimit / 10) + + rawPeer, closePeer, _ := server.newRawPeer(t, "peer", protocol) + defer closePeer() var ( reqID uint64 @@ -619,14 +717,14 @@ func TestStopResumeLes3(t *testing.T) { header := server.handler.blockchain.CurrentHeader() req := func() { reqID++ - sendRequest(server.peer.app, GetBlockHeadersMsg, reqID, &GetBlockHeadersData{Origin: hashOrNumber{Hash: header.Hash()}, Amount: 1}) + sendRequest(rawPeer.app, GetBlockHeadersMsg, reqID, &GetBlockHeadersData{Origin: hashOrNumber{Hash: header.Hash()}, Amount: 1}) } for i := 1; i <= 5; i++ { // send requests while we still have enough buffer and expect a response for expBuf >= testCost { req() expBuf -= testCost - if err := expectResponse(server.peer.app, BlockHeadersMsg, reqID, expBuf, []*types.Header{header}); err != nil { + if err := expectResponse(rawPeer.app, BlockHeadersMsg, reqID, expBuf, []*types.Header{header}); err != nil { t.Errorf("expected response and failed: %v", err) } } @@ -636,7 +734,7 @@ func TestStopResumeLes3(t *testing.T) { req() c-- } - if err := p2p.ExpectMsg(server.peer.app, StopMsg, nil); err != nil { + if err := p2p.ExpectMsg(rawPeer.app, StopMsg, nil); err != nil { t.Errorf("expected StopMsg and failed: %v", err) } // wait until the buffer is recharged by half of the limit @@ -645,7 +743,7 @@ func TestStopResumeLes3(t *testing.T) { // expect a ResumeMsg with the partially recharged buffer value expBuf += testBufRecharge * wait - if err := p2p.ExpectMsg(server.peer.app, ResumeMsg, expBuf); err != nil { + if err := p2p.ExpectMsg(rawPeer.app, ResumeMsg, expBuf); err != nil { t.Errorf("expected ResumeMsg and failed: %v", err) } } diff --git a/les/odr.go b/les/odr.go index 2c36f512de..d45c6a1a5d 100644 --- a/les/odr.go +++ b/les/odr.go @@ -18,6 +18,7 @@ package les import ( "context" + "sort" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -31,14 +32,16 @@ type LesOdr struct { db ethdb.Database indexerConfig *light.IndexerConfig chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer + peers *serverPeerSet retriever *retrieveManager stop chan struct{} } -func NewLesOdr(db ethdb.Database, config *light.IndexerConfig, retriever *retrieveManager) *LesOdr { +func NewLesOdr(db ethdb.Database, config *light.IndexerConfig, peers *serverPeerSet, retriever *retrieveManager) *LesOdr { return &LesOdr{ db: db, indexerConfig: config, + peers: peers, retriever: retriever, stop: make(chan struct{}), } @@ -98,7 +101,101 @@ type Msg struct { Obj interface{} } -// Retrieve tries to fetch an object from the LES network. +// peerByTxHistory is a heap.Interface implementation which can sort +// the peerset by transaction history. +type peerByTxHistory []*serverPeer + +func (h peerByTxHistory) Len() int { return len(h) } +func (h peerByTxHistory) Less(i, j int) bool { + if h[i].txHistory == txIndexUnlimited { + return false + } + if h[j].txHistory == txIndexUnlimited { + return true + } + return h[i].txHistory < h[j].txHistory +} +func (h peerByTxHistory) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +const ( + maxTxStatusRetry = 3 // The maximum retrys will be made for tx status request. + maxTxStatusCandidates = 5 // The maximum les servers the tx status requests will be sent to. +) + +// RetrieveTxStatus retrieves the transaction status from the LES network. +// There is no guarantee in the LES protocol that the mined transaction will +// be retrieved back for sure because of different reasons(the transaction +// is unindexed, the malicous server doesn't reply it deliberately, etc). +// Therefore, unretrieved transactions(UNKNOWN) will receive a certain number +// of retries, thus giving a weak guarantee. +func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequest) error { + // Sort according to the transaction history supported by the peer and + // select the peers with longest history. + var ( + retries int + peers []*serverPeer + missing = len(req.Hashes) + result = make([]light.TxStatus, len(req.Hashes)) + canSend = make(map[string]bool) + ) + for _, peer := range odr.peers.allPeers() { + if peer.txHistory == txIndexDisabled { + continue + } + peers = append(peers, peer) + } + sort.Sort(sort.Reverse(peerByTxHistory(peers))) + for i := 0; i < maxTxStatusCandidates && i < len(peers); i++ { + canSend[peers[i].id] = true + } + // Send out the request and assemble the result. + for { + if retries >= maxTxStatusRetry || len(canSend) == 0 { + break + } + var ( + // Deep copy the request, so that the partial result won't be mixed. + req = &TxStatusRequest{Hashes: req.Hashes} + id = genReqID() + distreq = &distReq{ + getCost: func(dp distPeer) uint64 { return req.GetCost(dp.(*serverPeer)) }, + canSend: func(dp distPeer) bool { return canSend[dp.(*serverPeer).id] }, + request: func(dp distPeer) func() { + p := dp.(*serverPeer) + p.fcServer.QueuedRequest(id, req.GetCost(p)) + delete(canSend, p.id) + return func() { req.Request(id, p) } + }, + } + ) + if err := odr.retriever.retrieve(ctx, id, distreq, func(p distPeer, msg *Msg) error { return req.Validate(odr.db, msg) }, odr.stop); err != nil { + return err + } + // Collect the response and assemble them to the final result. + // All the response is not verifiable, so always pick the first + // one we get. + for index, status := range req.Status { + if result[index].Status != core.TxStatusUnknown { + continue + } + if status.Status == core.TxStatusUnknown { + continue + } + result[index], missing = status, missing-1 + } + // Abort the procedure if all the status are retrieved + if missing == 0 { + break + } + retries += 1 + } + req.Status = result + return nil +} + +// Retrieve tries to fetch an object from the LES network. It's a common API +// for most of the LES requests except for the TxStatusRequest which needs +// the additional retry mechanism. // If the network retrieval was successful, it stores the object in local db. func (odr *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err error) { lreq := LesRequest(req) diff --git a/les/odr_requests.go b/les/odr_requests.go index 711c54b40f..d548fb1ee0 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -487,7 +487,7 @@ func (r *TxStatusRequest) GetCost(peer *serverPeer) uint64 { // CanSend tells if a certain peer is suitable for serving the given request func (r *TxStatusRequest) CanSend(peer *serverPeer) bool { - return peer.serveTxLookup + return peer.txHistory != txIndexDisabled } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -496,13 +496,12 @@ func (r *TxStatusRequest) Request(reqID uint64, peer *serverPeer) error { return peer.requestTxStatus(reqID, r.Hashes) } -// Valid processes an ODR request reply message from the LES network +// Validate processes an ODR request reply message from the LES network // returns true and stores results in memory if the message was a valid reply // to the request (implementation of LesOdrRequest) func (r *TxStatusRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating transaction status", "count", len(r.Hashes)) - // Ensure we have a correct message with a single block body if msg.MsgType != MsgTxStatus { return errInvalidMessageType } diff --git a/les/odr_test.go b/les/odr_test.go index 2a70a296c8..a43382e05e 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -19,7 +19,10 @@ package les import ( "bytes" "context" + "crypto/rand" + "fmt" "math/big" + "reflect" "testing" "time" @@ -190,7 +193,13 @@ func odrTxStatus(ctx context.Context, db ethdb.Database, config *params.ChainCon // testOdr tests odr requests whose validation guaranteed by block headers. func testOdr(t *testing.T, protocol int, expFail uint64, checkCached bool, fn odrTestFn) { // Assemble the test environment - server, client, tearDown := newClientServerEnv(t, 4, protocol, nil, nil, 0, false, true, true) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + connect: true, + nopruning: true, + } + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() // Ensure the client has synced all necessary data. @@ -246,3 +255,184 @@ func testOdr(t *testing.T, protocol int, expFail uint64, checkCached bool, fn od test(5) } } + +func TestGetTxStatusFromUnindexedPeersLES4(t *testing.T) { testGetTxStatusFromUnindexedPeers(t, lpv4) } + +func testGetTxStatusFromUnindexedPeers(t *testing.T, protocol int) { + var ( + blocks = 8 + netconfig = testnetConfig{ + blocks: blocks, + protocol: protocol, + nopruning: true, + } + ) + server, client, tearDown := newClientServerEnv(t, netconfig) + defer tearDown() + + // Iterate the chain, create the tx indexes locally + var ( + testHash common.Hash + testStatus light.TxStatus + + txs = make(map[common.Hash]*types.Transaction) // Transaction objects set + blockNumbers = make(map[common.Hash]uint64) // Transaction hash to block number mappings + blockHashes = make(map[common.Hash]common.Hash) // Transaction hash to block hash mappings + intraIndex = make(map[common.Hash]uint64) // Transaction intra-index in block + ) + for number := uint64(1); number < server.backend.Blockchain().CurrentBlock().NumberU64(); number++ { + block := server.backend.Blockchain().GetBlockByNumber(number) + if block == nil { + t.Fatalf("Failed to retrieve block %d", number) + } + for index, tx := range block.Transactions() { + txs[tx.Hash()] = tx + blockNumbers[tx.Hash()] = number + blockHashes[tx.Hash()] = block.Hash() + intraIndex[tx.Hash()] = uint64(index) + + if testHash == (common.Hash{}) { + testHash = tx.Hash() + testStatus = light.TxStatus{ + Status: core.TxStatusIncluded, + Lookup: &rawdb.LegacyTxLookupEntry{ + BlockHash: block.Hash(), + BlockIndex: block.NumberU64(), + Index: uint64(index), + }, + } + } + } + } + // serveMsg processes incoming GetTxStatusMsg and sends the response back. + serveMsg := func(peer *testPeer, txLookup uint64) error { + msg, err := peer.app.ReadMsg() + if err != nil { + return err + } + if msg.Code != GetTxStatusMsg { + return fmt.Errorf("message code mismatch: got %d, expected %d", msg.Code, GetTxStatusMsg) + } + var r GetTxStatusPacket + if err := msg.Decode(&r); err != nil { + return err + } + stats := make([]light.TxStatus, len(r.Hashes)) + for i, hash := range r.Hashes { + number, exist := blockNumbers[hash] + if !exist { + continue // Filter out unknown transactions + } + min := uint64(blocks) - txLookup + if txLookup != txIndexUnlimited && (txLookup == txIndexDisabled || number < min) { + continue // Filter out unindexed transactions + } + stats[i].Status = core.TxStatusIncluded + stats[i].Lookup = &rawdb.LegacyTxLookupEntry{ + BlockHash: blockHashes[hash], + BlockIndex: number, + Index: intraIndex[hash], + } + } + data, _ := rlp.EncodeToBytes(stats) + reply := &reply{peer.app, TxStatusMsg, r.ReqID, data} + reply.send(testBufLimit) + return nil + } + + var testspecs = []struct { + peers int + txLookups []uint64 + txs []common.Hash + results []light.TxStatus + }{ + // Retrieve mined transaction from the empty peerset + { + peers: 0, + txLookups: []uint64{}, + txs: []common.Hash{testHash}, + results: []light.TxStatus{{}}, + }, + // Retrieve unknown transaction from the full peers + { + peers: 3, + txLookups: []uint64{txIndexUnlimited, txIndexUnlimited, txIndexUnlimited}, + txs: []common.Hash{randomHash()}, + results: []light.TxStatus{{}}, + }, + // Retrieve mined transaction from the full peers + { + peers: 3, + txLookups: []uint64{txIndexUnlimited, txIndexUnlimited, txIndexUnlimited}, + txs: []common.Hash{testHash}, + results: []light.TxStatus{testStatus}, + }, + // Retrieve mixed transactions from the full peers + { + peers: 3, + txLookups: []uint64{txIndexUnlimited, txIndexUnlimited, txIndexUnlimited}, + txs: []common.Hash{randomHash(), testHash}, + results: []light.TxStatus{{}, testStatus}, + }, + // Retrieve mixed transactions from unindexed peer(but the target is still available) + { + peers: 3, + txLookups: []uint64{uint64(blocks) - testStatus.Lookup.BlockIndex, uint64(blocks) - testStatus.Lookup.BlockIndex - 1, uint64(blocks) - testStatus.Lookup.BlockIndex - 2}, + txs: []common.Hash{randomHash(), testHash}, + results: []light.TxStatus{{}, testStatus}, + }, + // Retrieve mixed transactions from unindexed peer(but the target is not available) + { + peers: 3, + txLookups: []uint64{uint64(blocks) - testStatus.Lookup.BlockIndex - 1, uint64(blocks) - testStatus.Lookup.BlockIndex - 1, uint64(blocks) - testStatus.Lookup.BlockIndex - 2}, + txs: []common.Hash{randomHash(), testHash}, + results: []light.TxStatus{{}, {}}, + }, + } + for _, testspec := range testspecs { + // Create a bunch of server peers with different tx history + var ( + serverPeers []*testPeer + closeFns []func() + ) + for i := 0; i < testspec.peers; i++ { + peer, closePeer, _ := client.newRawPeer(t, fmt.Sprintf("server-%d", i), protocol, testspec.txLookups[i]) + serverPeers = append(serverPeers, peer) + closeFns = append(closeFns, closePeer) + + // Create a one-time routine for serving message + go func(i int, peer *testPeer) { + serveMsg(peer, testspec.txLookups[i]) + }(i, peer) + } + + // Send out the GetTxStatus requests, compare the result with + // expected value. + r := &light.TxStatusRequest{Hashes: testspec.txs} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + err := client.handler.backend.odr.RetrieveTxStatus(ctx, r) + if err != nil { + t.Errorf("Failed to retrieve tx status %v", err) + } else { + if !reflect.DeepEqual(testspec.results, r.Status) { + t.Errorf("Result mismatch, diff") + } + } + + // Close all connected peers and start the next round + for _, closeFn := range closeFns { + closeFn() + } + } +} + +// randomHash generates a random blob of data and returns it as a hash. +func randomHash() common.Hash { + var hash common.Hash + if n, err := rand.Read(hash[:]); n != common.HashLength || err != nil { + panic(err) + } + return hash +} diff --git a/les/peer.go b/les/peer.go index 479b4034bc..0361167ee1 100644 --- a/les/peer.go +++ b/les/peer.go @@ -341,7 +341,7 @@ type serverPeer struct { onlyAnnounce bool // The flag whether the server sends announcement only. chainSince, chainRecent uint64 // The range of chain server peer can serve. stateSince, stateRecent uint64 // The range of state server peer can serve. - serveTxLookup bool // The server peer can serve tx lookups. + txHistory uint64 // The length of available tx history, 0 means all, 1 means disabled // Advertised checkpoint fields checkpointNumber uint64 // The block height which the checkpoint is registered. @@ -634,13 +634,13 @@ func (p *serverPeer) Handshake(genesis common.Hash, forkid forkid.ID, forkFilter if err := recv.get("recentTxLookup", &recentTx); err != nil { return err } - // Note: in the current version we only consider the tx index service useful - // if it is unlimited. This can be made configurable in the future. - p.serveTxLookup = recentTx == txIndexUnlimited + p.txHistory = uint64(recentTx) } else { - p.serveTxLookup = true + // The weak assumption is held here that legacy les server(les2,3) + // has unlimited transaction history. The les serving in these legacy + // versions is disabled if the transaction is unindexed. + p.txHistory = txIndexUnlimited } - if p.onlyAnnounce && !p.trusted { return errResp(ErrUselessPeer, "peer cannot serve requests") } diff --git a/les/pruner_test.go b/les/pruner_test.go index 62b4e9a950..c6f198c088 100644 --- a/les/pruner_test.go +++ b/les/pruner_test.go @@ -28,19 +28,26 @@ import ( ) func TestLightPruner(t *testing.T) { - config := light.TestClientIndexerConfig - - waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { - for { - cs, _, _ := cIndexer.Sections() - bts, _, _ := btIndexer.Sections() - if cs >= 3 && bts >= 3 { - break + var ( + waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + cs, _, _ := cIndexer.Sections() + bts, _, _ := btIndexer.Sections() + if cs >= 3 && bts >= 3 { + break + } + time.Sleep(10 * time.Millisecond) } - time.Sleep(10 * time.Millisecond) } - } - server, client, tearDown := newClientServerEnv(t, int(3*config.ChtSize+config.ChtConfirms), 2, waitIndexers, nil, 0, false, true, false) + config = light.TestClientIndexerConfig + netconfig = testnetConfig{ + blocks: int(3*config.ChtSize + config.ChtConfirms), + protocol: 3, + indexFn: waitIndexers, + connect: true, + } + ) + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() // checkDB iterates the chain with given prefix, resolves the block number diff --git a/les/request_test.go b/les/request_test.go index b054fd88df..c65405e375 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -83,7 +83,14 @@ func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrReq func testAccess(t *testing.T, protocol int, fn accessTestFn) { // Assemble the test environment - server, client, tearDown := newClientServerEnv(t, 4, protocol, nil, nil, 0, false, true, true) + netconfig := testnetConfig{ + blocks: 4, + protocol: protocol, + indexFn: nil, + connect: true, + nopruning: true, + } + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() // Ensure the client has synced all necessary data. diff --git a/les/sync_test.go b/les/sync_test.go index 64e7283663..d3bb90df02 100644 --- a/les/sync_test.go +++ b/les/sync_test.go @@ -31,15 +31,15 @@ import ( ) // Test light syncing which will download all headers from genesis. -func TestLightSyncingLes3(t *testing.T) { testCheckpointSyncing(t, 3, 0) } +func TestLightSyncingLes3(t *testing.T) { testCheckpointSyncing(t, lpv3, 0) } // Test legacy checkpoint syncing which will download tail headers // based on a hardcoded checkpoint. -func TestLegacyCheckpointSyncingLes3(t *testing.T) { testCheckpointSyncing(t, 3, 1) } +func TestLegacyCheckpointSyncingLes3(t *testing.T) { testCheckpointSyncing(t, lpv3, 1) } // Test checkpoint syncing which will download tail headers based // on a verified checkpoint. -func TestCheckpointSyncingLes3(t *testing.T) { testCheckpointSyncing(t, 3, 2) } +func TestCheckpointSyncingLes3(t *testing.T) { testCheckpointSyncing(t, lpv3, 2) } func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { config := light.TestServerIndexerConfig @@ -55,7 +55,13 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { } } // Generate 128+1 blocks (totally 1 CHT section) - server, client, tearDown := newClientServerEnv(t, int(config.ChtSize+config.ChtConfirms), protocol, waitIndexers, nil, 0, false, false, true) + netconfig := testnetConfig{ + blocks: int(config.ChtSize + config.ChtConfirms), + protocol: protocol, + indexFn: waitIndexers, + nopruning: true, + } + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() expected := config.ChtSize + config.ChtConfirms @@ -78,7 +84,7 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { // Register the assembled checkpoint into oracle. header := server.backend.Blockchain().CurrentHeader() - data := append([]byte{0x19, 0x00}, append(registrarAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) + data := append([]byte{0x19, 0x00}, append(oracleAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) sig, _ := crypto.Sign(crypto.Keccak256(data), signerKey) sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper auth, _ := bind.NewKeyedTransactorWithChainID(signerKey, big.NewInt(1337)) @@ -128,10 +134,10 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { } } -func TestMissOracleBackend(t *testing.T) { testMissOracleBackend(t, true) } -func TestMissOracleBackendNoCheckpoint(t *testing.T) { testMissOracleBackend(t, false) } +func TestMissOracleBackendLES3(t *testing.T) { testMissOracleBackend(t, true, lpv3) } +func TestMissOracleBackendNoCheckpointLES3(t *testing.T) { testMissOracleBackend(t, false, lpv3) } -func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { +func testMissOracleBackend(t *testing.T, hasCheckpoint bool, protocol int) { config := light.TestServerIndexerConfig waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { @@ -145,7 +151,13 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { } } // Generate 128+1 blocks (totally 1 CHT section) - server, client, tearDown := newClientServerEnv(t, int(config.ChtSize+config.ChtConfirms), 3, waitIndexers, nil, 0, false, false, true) + netconfig := testnetConfig{ + blocks: int(config.ChtSize + config.ChtConfirms), + protocol: protocol, + indexFn: waitIndexers, + nopruning: true, + } + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() expected := config.ChtSize + config.ChtConfirms @@ -160,7 +172,7 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { // Register the assembled checkpoint into oracle. header := server.backend.Blockchain().CurrentHeader() - data := append([]byte{0x19, 0x00}, append(registrarAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) + data := append([]byte{0x19, 0x00}, append(oracleAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) sig, _ := crypto.Sign(crypto.Keccak256(data), signerKey) sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper auth, _ := bind.NewKeyedTransactorWithChainID(signerKey, big.NewInt(1337)) @@ -220,7 +232,9 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { } } -func TestSyncFromConfiguredCheckpoint(t *testing.T) { +func TestSyncFromConfiguredCheckpointLES3(t *testing.T) { testSyncFromConfiguredCheckpoint(t, lpv3) } + +func testSyncFromConfiguredCheckpoint(t *testing.T, protocol int) { config := light.TestServerIndexerConfig waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { @@ -234,7 +248,13 @@ func TestSyncFromConfiguredCheckpoint(t *testing.T) { } } // Generate 256+1 blocks (totally 2 CHT sections) - server, client, tearDown := newClientServerEnv(t, int(2*config.ChtSize+config.ChtConfirms), 3, waitIndexers, nil, 0, false, false, true) + netconfig := testnetConfig{ + blocks: int(2*config.ChtSize + config.ChtConfirms), + protocol: protocol, + indexFn: waitIndexers, + nopruning: true, + } + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() // Configure the local checkpoint(the first section) @@ -296,7 +316,9 @@ func TestSyncFromConfiguredCheckpoint(t *testing.T) { } } -func TestSyncAll(t *testing.T) { +func TestSyncAll(t *testing.T) { testSyncAll(t, lpv3) } + +func testSyncAll(t *testing.T, protocol int) { config := light.TestServerIndexerConfig waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { @@ -310,7 +332,13 @@ func TestSyncAll(t *testing.T) { } } // Generate 256+1 blocks (totally 2 CHT sections) - server, client, tearDown := newClientServerEnv(t, int(2*config.ChtSize+config.ChtConfirms), 3, waitIndexers, nil, 0, false, false, true) + netconfig := testnetConfig{ + blocks: int(2*config.ChtSize + config.ChtConfirms), + protocol: protocol, + indexFn: waitIndexers, + nopruning: true, + } + server, client, tearDown := newClientServerEnv(t, netconfig) defer tearDown() client.handler.backend.config.SyncFromCheckpoint = true diff --git a/les/test_helper.go b/les/test_helper.go index 6f93c8a48f..39313b1e3b 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -14,8 +14,9 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// This file contains some shares testing functionality, common to multiple -// different files and modules being tested. +// This file contains some shares testing functionality, common to multiple +// different files and modules being tested. Client based network and Server +// based network can be created easily with available APIs. package les @@ -68,10 +69,10 @@ var ( testEventEmitterCode = common.Hex2Bytes("60606040523415600e57600080fd5b7f57050ab73f6b9ebdd9f76b8d4997793f48cf956e965ee070551b9ca0bb71584e60405160405180910390a160358060476000396000f3006060604052600080fd00a165627a7a723058203f727efcad8b5811f8cb1fc2620ce5e8c63570d697aef968172de296ea3994140029") - // Checkpoint registrar relative - registrarAddr common.Address - signerKey, _ = crypto.GenerateKey() - signerAddr = crypto.PubkeyToAddress(signerKey.PublicKey) + // Checkpoint oracle relative fields + oracleAddr common.Address + signerKey, _ = crypto.GenerateKey() + signerAddr = crypto.PubkeyToAddress(signerKey.PublicKey) ) var ( @@ -112,14 +113,23 @@ func prepare(n int, backend *backends.SimulatedBackend) { for i := 0; i < n; i++ { switch i { case 0: + // Builtin-block + // number: 1 + // txs: 2 + // deploy checkpoint contract auth, _ := bind.NewKeyedTransactorWithChainID(bankKey, big.NewInt(1337)) - registrarAddr, _, _, _ = contract.DeployCheckpointOracle(auth, backend, []common.Address{signerAddr}, sectionSize, processConfirms, big.NewInt(1)) + oracleAddr, _, _, _ = contract.DeployCheckpointOracle(auth, backend, []common.Address{signerAddr}, sectionSize, processConfirms, big.NewInt(1)) + // bankUser transfers some ether to user1 nonce, _ := backend.PendingNonceAt(ctx, bankAddr) tx, _ := types.SignTx(types.NewTransaction(nonce, userAddr1, big.NewInt(10000), params.TxGas, nil, nil), signer, bankKey) backend.SendTransaction(ctx, tx) case 1: + // Builtin-block + // number: 2 + // txs: 4 + bankNonce, _ := backend.PendingNonceAt(ctx, bankAddr) userNonce1, _ := backend.PendingNonceAt(ctx, userAddr1) @@ -140,6 +150,10 @@ func prepare(n int, backend *backends.SimulatedBackend) { tx4, _ := types.SignTx(types.NewContractCreation(userNonce1+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, userKey1) backend.SendTransaction(ctx, tx4) case 2: + // Builtin-block + // number: 3 + // txs: 2 + // bankUser transfer some ether to signer bankNonce, _ := backend.PendingNonceAt(ctx, bankAddr) tx1, _ := types.SignTx(types.NewTransaction(bankNonce, signerAddr, big.NewInt(1000000000), params.TxGas, nil, nil), signer, bankKey) @@ -150,6 +164,10 @@ func prepare(n int, backend *backends.SimulatedBackend) { tx2, _ := types.SignTx(types.NewTransaction(bankNonce+1, testContractAddr, big.NewInt(0), 100000, nil, data), signer, bankKey) backend.SendTransaction(ctx, tx2) case 3: + // Builtin-block + // number: 4 + // txs: 1 + // invoke test contract bankNonce, _ := backend.PendingNonceAt(ctx, bankAddr) data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002") @@ -310,45 +328,61 @@ type testPeer struct { app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side } -// newTestPeer creates a new peer registered at the given protocol manager. -func newTestPeer(t *testing.T, name string, version int, handler *serverHandler, shake bool, testCost uint64) (*testPeer, <-chan error) { - // Create a message pipe to communicate through - app, net := p2p.MsgPipe() - - // Generate a random id and create the peer - var id enode.ID - rand.Read(id[:]) - peer := newClientPeer(version, NetworkId, p2p.NewPeer(id, name, nil), net) +// handshakeWithServer executes the handshake with the remote server peer. +func (p *testPeer) handshakeWithServer(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID) { + // It only works for the simulated client peer + if p.cpeer == nil { + t.Fatal("handshake for client peer only") + } + var sendList keyValueList + sendList = sendList.add("protocolVersion", uint64(p.cpeer.version)) + sendList = sendList.add("networkId", uint64(NetworkId)) + sendList = sendList.add("headTd", td) + sendList = sendList.add("headHash", head) + sendList = sendList.add("headNum", headNum) + sendList = sendList.add("genesisHash", genesis) + if p.cpeer.version >= lpv4 { + sendList = sendList.add("forkID", &forkID) + } + if err := p2p.ExpectMsg(p.app, StatusMsg, nil); err != nil { + t.Fatalf("status recv: %v", err) + } + if err := p2p.Send(p.app, StatusMsg, sendList); err != nil { + t.Fatalf("status send: %v", err) + } +} - // Start the peer on a new thread - errCh := make(chan error, 1) - go func() { - select { - case <-handler.closeCh: - errCh <- p2p.DiscQuitting - case errCh <- handler.handle(peer): - } - }() - tp := &testPeer{ - app: app, - net: net, - cpeer: peer, +// handshakeWithClient executes the handshake with the remote client peer. +func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, costList RequestCostList, recentTxLookup uint64) { + // It only works for the simulated client peer + if p.speer == nil { + t.Fatal("handshake for server peer only") } - // Execute any implicitly requested handshakes and return - if shake { - // Customize the cost table if required. - if testCost != 0 { - handler.server.costTracker.testCostList = testCostList(testCost) - } - var ( - genesis = handler.blockchain.Genesis() - head = handler.blockchain.CurrentHeader() - td = handler.blockchain.GetTd(head.Hash(), head.Number.Uint64()) - ) - forkID := forkid.NewID(handler.blockchain.Config(), genesis.Hash(), head.Number.Uint64()) - tp.handshake(t, td, head.Hash(), head.Number.Uint64(), genesis.Hash(), forkID, testCostList(testCost)) + var sendList keyValueList + sendList = sendList.add("protocolVersion", uint64(p.speer.version)) + sendList = sendList.add("networkId", uint64(NetworkId)) + sendList = sendList.add("headTd", td) + sendList = sendList.add("headHash", head) + sendList = sendList.add("headNum", headNum) + sendList = sendList.add("genesisHash", genesis) + sendList = sendList.add("serveHeaders", nil) + sendList = sendList.add("serveChainSince", uint64(0)) + sendList = sendList.add("serveStateSince", uint64(0)) + sendList = sendList.add("serveRecentState", uint64(core.TriesInMemory-4)) + sendList = sendList.add("txRelay", nil) + sendList = sendList.add("flowControl/BL", testBufLimit) + sendList = sendList.add("flowControl/MRR", testBufRecharge) + sendList = sendList.add("flowControl/MRC", costList) + if p.speer.version >= lpv4 { + sendList = sendList.add("forkID", &forkID) + sendList = sendList.add("recentTxLookup", recentTxLookup) + } + if err := p2p.ExpectMsg(p.app, StatusMsg, nil); err != nil { + t.Fatalf("status recv: %v", err) + } + if err := p2p.Send(p.app, StatusMsg, sendList); err != nil { + t.Fatalf("status send: %v", err) } - return tp, errCh } // close terminates the local side of the peer, notifying the remote protocol @@ -402,48 +436,9 @@ func newTestPeerPair(name string, version int, server *serverHandler, client *cl return &testPeer{cpeer: peer1, net: net, app: app}, &testPeer{speer: peer2, net: app, app: net}, nil } -// handshake simulates a trivial handshake that expects the same state from the -// remote side as we are simulating locally. -func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, costList RequestCostList) { - var expList keyValueList - expList = expList.add("protocolVersion", uint64(p.cpeer.version)) - expList = expList.add("networkId", uint64(NetworkId)) - expList = expList.add("headTd", td) - expList = expList.add("headHash", head) - expList = expList.add("headNum", headNum) - expList = expList.add("genesisHash", genesis) - if p.cpeer.version >= lpv4 { - expList = expList.add("forkID", &forkID) - } - sendList := make(keyValueList, len(expList)) - copy(sendList, expList) - expList = expList.add("serveHeaders", nil) - expList = expList.add("serveChainSince", uint64(0)) - expList = expList.add("serveStateSince", uint64(0)) - expList = expList.add("serveRecentState", uint64(core.TriesInMemory-4)) - expList = expList.add("txRelay", nil) - if p.cpeer.version >= lpv4 { - expList = expList.add("recentTxLookup", uint64(0)) - } - expList = expList.add("flowControl/BL", testBufLimit) - expList = expList.add("flowControl/MRR", testBufRecharge) - expList = expList.add("flowControl/MRC", costList) - - if err := p2p.ExpectMsg(p.app, StatusMsg, expList); err != nil { - t.Fatalf("status recv: %v", err) - } - if err := p2p.Send(p.app, StatusMsg, sendList); err != nil { - t.Fatalf("status send: %v", err) - } - p.cpeer.fcParams = flowcontrol.ServerParams{ - BufLimit: testBufLimit, - MinRecharge: testBufRecharge, - } -} - type indexerCallback func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer) -// testClient represents a client for testing with necessary auxiliary fields. +// testClient represents a client object for testing with necessary auxiliary fields. type testClient struct { clock mclock.Clock db ethdb.Database @@ -455,7 +450,58 @@ type testClient struct { bloomTrieIndexer *core.ChainIndexer } -// testServer represents a server for testing with necessary auxiliary fields. +// newRawPeer creates a new server peer connects to the server and do the handshake. +func (client *testClient) newRawPeer(t *testing.T, name string, version int, recentTxLookup uint64) (*testPeer, func(), <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + // Generate a random id and create the peer + var id enode.ID + rand.Read(id[:]) + peer := newServerPeer(version, NetworkId, false, p2p.NewPeer(id, name, nil), net) + + // Start the peer on a new thread + errCh := make(chan error, 1) + go func() { + select { + case <-client.handler.closeCh: + errCh <- p2p.DiscQuitting + case errCh <- client.handler.handle(peer): + } + }() + tp := &testPeer{ + app: app, + net: net, + speer: peer, + } + var ( + genesis = client.handler.backend.blockchain.Genesis() + head = client.handler.backend.blockchain.CurrentHeader() + td = client.handler.backend.blockchain.GetTd(head.Hash(), head.Number.Uint64()) + ) + forkID := forkid.NewID(client.handler.backend.blockchain.Config(), genesis.Hash(), head.Number.Uint64()) + tp.handshakeWithClient(t, td, head.Hash(), head.Number.Uint64(), genesis.Hash(), forkID, testCostList(0), recentTxLookup) // disable flow control by default + + // Ensure the connection is established or exits when any error occurs + for { + select { + case <-errCh: + return nil, nil, nil + default: + } + if atomic.LoadUint32(&peer.serving) == 1 { + break + } + time.Sleep(50 * time.Millisecond) + } + closePeer := func() { + tp.speer.close() + tp.close() + } + return tp, closePeer, errCh +} + +// testServer represents a server object for testing with necessary auxiliary fields. type testServer struct { clock mclock.Clock backend *backends.SimulatedBackend @@ -468,89 +514,109 @@ type testServer struct { bloomTrieIndexer *core.ChainIndexer } -func newServerEnv(t *testing.T, blocks int, protocol int, callback indexerCallback, simClock bool, newPeer bool, testCost uint64) (*testServer, func()) { - db := rawdb.NewMemoryDatabase() - indexers := testIndexers(db, nil, light.TestServerIndexerConfig, true) +// newRawPeer creates a new client peer connects to the server and do the handshake. +func (server *testServer) newRawPeer(t *testing.T, name string, version int) (*testPeer, func(), <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() - var clock mclock.Clock = &mclock.System{} - if simClock { - clock = &mclock.Simulated{} - } - handler, b := newTestServerHandler(blocks, indexers, db, clock) + // Generate a random id and create the peer + var id enode.ID + rand.Read(id[:]) + peer := newClientPeer(version, NetworkId, p2p.NewPeer(id, name, nil), net) - var peer *testPeer - if newPeer { - peer, _ = newTestPeer(t, "peer", protocol, handler, true, testCost) + // Start the peer on a new thread + errCh := make(chan error, 1) + go func() { + select { + case <-server.handler.closeCh: + errCh <- p2p.DiscQuitting + case errCh <- server.handler.handle(peer): + } + }() + tp := &testPeer{ + app: app, + net: net, + cpeer: peer, } + var ( + genesis = server.handler.blockchain.Genesis() + head = server.handler.blockchain.CurrentHeader() + td = server.handler.blockchain.GetTd(head.Hash(), head.Number.Uint64()) + ) + forkID := forkid.NewID(server.handler.blockchain.Config(), genesis.Hash(), head.Number.Uint64()) + tp.handshakeWithServer(t, td, head.Hash(), head.Number.Uint64(), genesis.Hash(), forkID) - cIndexer, bIndexer, btIndexer := indexers[0], indexers[1], indexers[2] - cIndexer.Start(handler.blockchain) - bIndexer.Start(handler.blockchain) - - // Wait until indexers generate enough index data. - if callback != nil { - callback(cIndexer, bIndexer, btIndexer) - } - server := &testServer{ - clock: clock, - backend: b, - db: db, - peer: peer, - handler: handler, - chtIndexer: cIndexer, - bloomIndexer: bIndexer, - bloomTrieIndexer: btIndexer, - } - teardown := func() { - if newPeer { - peer.close() - peer.cpeer.close() - b.Close() + // Ensure the connection is established or exits when any error occurs + for { + select { + case <-errCh: + return nil, nil, nil + default: } - cIndexer.Close() - bIndexer.Close() + if atomic.LoadUint32(&peer.serving) == 1 { + break + } + time.Sleep(50 * time.Millisecond) + } + closePeer := func() { + tp.cpeer.close() + tp.close() } - return server, teardown + return tp, closePeer, errCh } -func newClientServerEnv(t *testing.T, blocks int, protocol int, callback indexerCallback, ulcServers []string, ulcFraction int, simClock bool, connect bool, disablePruning bool) (*testServer, *testClient, func()) { - sdb, cdb := rawdb.NewMemoryDatabase(), rawdb.NewMemoryDatabase() - speers := newServerPeerSet() +// testnetConfig wraps all the configurations for testing network. +type testnetConfig struct { + blocks int + protocol int + indexFn indexerCallback + ulcServers []string + ulcFraction int + simClock bool + connect bool + nopruning bool +} +func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testClient, func()) { + var ( + sdb = rawdb.NewMemoryDatabase() + cdb = rawdb.NewMemoryDatabase() + speers = newServerPeerSet() + ) var clock mclock.Clock = &mclock.System{} - if simClock { + if config.simClock { clock = &mclock.Simulated{} } dist := newRequestDistributor(speers, clock) rm := newRetrieveManager(speers, dist, func() time.Duration { return time.Millisecond * 500 }) - odr := NewLesOdr(cdb, light.TestClientIndexerConfig, rm) + odr := NewLesOdr(cdb, light.TestClientIndexerConfig, speers, rm) sindexers := testIndexers(sdb, nil, light.TestServerIndexerConfig, true) - cIndexers := testIndexers(cdb, odr, light.TestClientIndexerConfig, disablePruning) + cIndexers := testIndexers(cdb, odr, light.TestClientIndexerConfig, config.nopruning) scIndexer, sbIndexer, sbtIndexer := sindexers[0], sindexers[1], sindexers[2] ccIndexer, cbIndexer, cbtIndexer := cIndexers[0], cIndexers[1], cIndexers[2] odr.SetIndexers(ccIndexer, cbIndexer, cbtIndexer) - server, b := newTestServerHandler(blocks, sindexers, sdb, clock) - client := newTestClientHandler(b, odr, cIndexers, cdb, speers, ulcServers, ulcFraction) + server, b := newTestServerHandler(config.blocks, sindexers, sdb, clock) + client := newTestClientHandler(b, odr, cIndexers, cdb, speers, config.ulcServers, config.ulcFraction) scIndexer.Start(server.blockchain) sbIndexer.Start(server.blockchain) ccIndexer.Start(client.backend.blockchain) cbIndexer.Start(client.backend.blockchain) - if callback != nil { - callback(scIndexer, sbIndexer, sbtIndexer) + if config.indexFn != nil { + config.indexFn(scIndexer, sbIndexer, sbtIndexer) } var ( err error speer, cpeer *testPeer ) - if connect { + if config.connect { done := make(chan struct{}) client.syncEnd = func(_ *types.Header) { close(done) } - cpeer, speer, err = newTestPeerPair("peer", protocol, server, client) + cpeer, speer, err = newTestPeerPair("peer", config.protocol, server, client) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) } @@ -580,7 +646,7 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, callback indexer bloomTrieIndexer: cbtIndexer, } teardown := func() { - if connect { + if config.connect { speer.close() cpeer.close() cpeer.cpeer.close() diff --git a/les/ulc_test.go b/les/ulc_test.go index 657b13db2c..d7308fa593 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -126,7 +126,12 @@ func connect(server *serverHandler, serverId enode.ID, client *clientHandler, pr // newTestServerPeer creates server peer. func newTestServerPeer(t *testing.T, blocks int, protocol int) (*testServer, *enode.Node, func()) { - s, teardown := newServerEnv(t, blocks, protocol, nil, false, false, 0) + netconfig := testnetConfig{ + blocks: blocks, + protocol: protocol, + nopruning: true, + } + s, _, teardown := newClientServerEnv(t, netconfig) key, err := crypto.GenerateKey() if err != nil { t.Fatal("generate key err:", err) @@ -138,6 +143,12 @@ func newTestServerPeer(t *testing.T, blocks int, protocol int) (*testServer, *en // newTestLightPeer creates node with light sync mode func newTestLightPeer(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { - _, c, teardown := newClientServerEnv(t, 0, protocol, nil, ulcServers, ulcFraction, false, false, true) + netconfig := testnetConfig{ + protocol: protocol, + ulcServers: ulcServers, + ulcFraction: ulcFraction, + nopruning: true, + } + _, c, teardown := newClientServerEnv(t, netconfig) return c, teardown } diff --git a/light/odr.go b/light/odr.go index bb243f9152..9521dd53e8 100644 --- a/light/odr.go +++ b/light/odr.go @@ -42,6 +42,7 @@ type OdrBackend interface { BloomTrieIndexer() *core.ChainIndexer BloomIndexer() *core.ChainIndexer Retrieve(ctx context.Context, req OdrRequest) error + RetrieveTxStatus(ctx context.Context, req *TxStatusRequest) error IndexerConfig() *IndexerConfig } diff --git a/light/odr_util.go b/light/odr_util.go index ec2d1e6491..bbbcdbce21 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -269,10 +269,15 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint return result, nil } -// GetTransaction retrieves a canonical transaction by hash and also returns its position in the chain +// GetTransaction retrieves a canonical transaction by hash and also returns +// its position in the chain. There is no guarantee in the LES protocol that +// the mined transaction will be retrieved back for sure because of different +// reasons(the transaction is unindexed, the malicous server doesn't reply it +// deliberately, etc). Therefore, unretrieved transactions will receive a certain +// number of retrys, thus giving a weak guarantee. func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { r := &TxStatusRequest{Hashes: []common.Hash{txHash}} - if err := odr.Retrieve(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded { + if err := odr.RetrieveTxStatus(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded { return nil, common.Hash{}, 0, 0, err } pos := r.Status[0].Lookup From bbfb1e4008a359a8b57ec654330c0e674623e52f Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Thu, 25 Feb 2021 07:26:57 -0700 Subject: [PATCH 162/709] all: add support for EIP-2718, EIP-2930 transactions (#21502) This adds support for EIP-2718 typed transactions as well as EIP-2930 access list transactions (tx type 1). These EIPs are scheduled for the Berlin fork. There very few changes to existing APIs in core/types, and several new APIs to deal with access list transactions. In particular, there are two new constructor functions for transactions: types.NewTx and types.SignNewTx. Since the canonical encoding of typed transactions is not RLP-compatible, Transaction now has new methods for encoding and decoding: MarshalBinary and UnmarshalBinary. The existing EIP-155 signer does not support the new transaction types. All code dealing with transaction signatures should be updated to use the newer EIP-2930 signer. To make this easier for future updates, we have added new constructor functions for types.Signer: types.LatestSigner and types.LatestSignerForChainID. This change also adds support for the YoloV3 testnet. Co-authored-by: Martin Holst Swende Co-authored-by: Felix Lange Co-authored-by: Ryan Schneider --- accounts/abi/bind/auth.go | 4 +- accounts/abi/bind/backends/simulated.go | 25 +- accounts/keystore/keystore.go | 17 +- accounts/scwallet/wallet.go | 2 +- accounts/usbwallet/trezor.go | 2 + cmd/clef/main.go | 2 +- cmd/evm/README.md | 17 +- cmd/evm/internal/t8ntool/execution.go | 37 +- cmd/evm/internal/t8ntool/flags.go | 5 + cmd/evm/internal/t8ntool/transition.go | 135 +++++-- cmd/evm/main.go | 1 + cmd/evm/testdata/8/alloc.json | 11 + cmd/evm/testdata/8/env.json | 7 + cmd/evm/testdata/8/readme.md | 63 ++++ cmd/evm/testdata/8/txs.json | 58 +++ cmd/geth/main.go | 7 +- cmd/geth/usage.go | 3 +- cmd/utils/flags.go | 4 +- core/bench_test.go | 2 +- core/blockchain_test.go | 102 +++++- core/error.go | 10 +- core/genesis.go | 6 +- core/genesis_alloc.go | 3 +- core/state/statedb.go | 26 ++ core/state_prefetcher.go | 33 +- core/state_processor.go | 39 +- core/state_transition.go | 18 +- core/tx_pool.go | 12 +- core/types/access_list_tx.go | 115 ++++++ core/types/block.go | 21 +- core/types/block_test.go | 62 +++- core/types/derive_sha.go | 58 --- core/types/gen_access_tuple.go | 43 +++ core/types/gen_receipt_json.go | 6 + core/types/gen_tx_json.go | 101 ----- core/types/hashing.go | 112 ++++++ core/types/hashing_test.go | 212 +++++++++++ core/types/legacy_tx.go | 111 ++++++ core/types/receipt.go | 101 ++++- core/types/receipt_test.go | 76 +++- core/types/transaction.go | 466 ++++++++++++++---------- core/types/transaction_marshalling.go | 187 ++++++++++ core/types/transaction_signing.go | 259 +++++++++++-- core/types/transaction_test.go | 276 ++++++++++++-- core/vm/interface.go | 1 + core/vm/runtime/runtime.go | 18 +- eth/downloader/downloader.go | 2 +- eth/gasprice/gasprice_test.go | 2 +- eth/tracers/api.go | 5 +- eth/tracers/tracer.go | 2 +- ethclient/ethclient.go | 3 +- ethclient/ethclient_test.go | 2 +- ethclient/signer.go | 3 + graphql/graphql.go | 9 +- interfaces.go | 2 + internal/ethapi/api.go | 169 +++++---- internal/guide/guide_test.go | 4 +- les/benchmark.go | 2 +- les/odr_test.go | 4 +- light/odr_test.go | 2 +- light/txpool.go | 10 +- miner/worker.go | 7 +- miner/worker_test.go | 19 +- params/config.go | 6 +- params/protocol_params.go | 31 +- signer/core/api.go | 5 +- tests/state_test_util.go | 18 +- tests/transaction_test_util.go | 2 +- trie/stacktrie_test.go | 170 --------- 69 files changed, 2446 insertions(+), 909 deletions(-) create mode 100644 cmd/evm/testdata/8/alloc.json create mode 100644 cmd/evm/testdata/8/env.json create mode 100644 cmd/evm/testdata/8/readme.md create mode 100644 cmd/evm/testdata/8/txs.json create mode 100644 core/types/access_list_tx.go delete mode 100644 core/types/derive_sha.go create mode 100644 core/types/gen_access_tuple.go delete mode 100644 core/types/gen_tx_json.go create mode 100644 core/types/hashing.go create mode 100644 core/types/hashing_test.go create mode 100644 core/types/legacy_tx.go create mode 100644 core/types/transaction_marshalling.go diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index 1190772676..b8065e8488 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -120,7 +120,7 @@ func NewKeyStoreTransactorWithChainID(keystore *keystore.KeyStore, account accou if chainID == nil { return nil, ErrNoChainID } - signer := types.NewEIP155Signer(chainID) + signer := types.LatestSignerForChainID(chainID) return &TransactOpts{ From: account.Address, Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) { @@ -143,7 +143,7 @@ func NewKeyedTransactorWithChainID(key *ecdsa.PrivateKey, chainID *big.Int) (*Tr if chainID == nil { return nil, ErrNoChainID } - signer := types.NewEIP155Signer(chainID) + signer := types.LatestSignerForChainID(chainID) return &TransactOpts{ From: keyAddr, Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) { diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 8be364d08f..d6d525eae1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -559,7 +559,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa b.mu.Lock() defer b.mu.Unlock() - sender, err := types.Sender(types.NewEIP155Signer(b.config.ChainID), tx) + // Check transaction validity. + block := b.blockchain.CurrentBlock() + signer := types.MakeSigner(b.blockchain.Config(), block.Number()) + sender, err := types.Sender(signer, tx) if err != nil { panic(fmt.Errorf("invalid transaction: %v", err)) } @@ -568,7 +571,8 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa panic(fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce)) } - blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { + // Include tx in chain. + blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { for _, tx := range b.pendingBlock.Transactions() { block.AddTxWithChain(b.blockchain, tx) } @@ -707,14 +711,15 @@ type callMsg struct { ethereum.CallMsg } -func (m callMsg) From() common.Address { return m.CallMsg.From } -func (m callMsg) Nonce() uint64 { return 0 } -func (m callMsg) CheckNonce() bool { return false } -func (m callMsg) To() *common.Address { return m.CallMsg.To } -func (m callMsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } -func (m callMsg) Gas() uint64 { return m.CallMsg.Gas } -func (m callMsg) Value() *big.Int { return m.CallMsg.Value } -func (m callMsg) Data() []byte { return m.CallMsg.Data } +func (m callMsg) From() common.Address { return m.CallMsg.From } +func (m callMsg) Nonce() uint64 { return 0 } +func (m callMsg) CheckNonce() bool { return false } +func (m callMsg) To() *common.Address { return m.CallMsg.To } +func (m callMsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } +func (m callMsg) Gas() uint64 { return m.CallMsg.Gas } +func (m callMsg) Value() *big.Int { return m.CallMsg.Value } +func (m callMsg) Data() []byte { return m.CallMsg.Data } +func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList } // filterBackend implements filters.Backend to support filtering for logs without // taking bloom-bits acceleration structures into account. diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 9d5e2cf6a2..88dcfbeb69 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -283,11 +283,9 @@ func (ks *KeyStore) SignTx(a accounts.Account, tx *types.Transaction, chainID *b if !found { return nil, ErrLocked } - // Depending on the presence of the chain ID, sign with EIP155 or homestead - if chainID != nil { - return types.SignTx(tx, types.NewEIP155Signer(chainID), unlockedKey.PrivateKey) - } - return types.SignTx(tx, types.HomesteadSigner{}, unlockedKey.PrivateKey) + // Depending on the presence of the chain ID, sign with 2718 or homestead + signer := types.LatestSignerForChainID(chainID) + return types.SignTx(tx, signer, unlockedKey.PrivateKey) } // SignHashWithPassphrase signs hash if the private key matching the given address @@ -310,12 +308,9 @@ func (ks *KeyStore) SignTxWithPassphrase(a accounts.Account, passphrase string, return nil, err } defer zeroKey(key.PrivateKey) - - // Depending on the presence of the chain ID, sign with EIP155 or homestead - if chainID != nil { - return types.SignTx(tx, types.NewEIP155Signer(chainID), key.PrivateKey) - } - return types.SignTx(tx, types.HomesteadSigner{}, key.PrivateKey) + // Depending on the presence of the chain ID, sign with or without replay protection. + signer := types.LatestSignerForChainID(chainID) + return types.SignTx(tx, signer, key.PrivateKey) } // Unlock unlocks the given account indefinitely. diff --git a/accounts/scwallet/wallet.go b/accounts/scwallet/wallet.go index 6476646d7f..b4d229bc0b 100644 --- a/accounts/scwallet/wallet.go +++ b/accounts/scwallet/wallet.go @@ -699,7 +699,7 @@ func (w *Wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) // the needed details via SignTxWithPassphrase, or by other means (e.g. unlock // the account in a keystore). func (w *Wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - signer := types.NewEIP155Signer(chainID) + signer := types.LatestSignerForChainID(chainID) hash := signer.Hash(tx) sig, err := w.signHash(account, hash[:]) if err != nil { diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index 1892097baf..0546458c47 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -255,9 +255,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction if chainID == nil { signer = new(types.HomesteadSigner) } else { + // Trezor backend does not support typed transactions yet. signer = types.NewEIP155Signer(chainID) signature[64] -= byte(chainID.Uint64()*2 + 35) } + // Inject the final signature into the transaction and sanity check the sender signed, err := tx.WithSignature(signer, signature) if err != nil { diff --git a/cmd/clef/main.go b/cmd/clef/main.go index dbbb410fb6..8befce88dc 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -1106,7 +1106,7 @@ func GenDoc(ctx *cli.Context) { rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed") var tx types.Transaction - rlp.DecodeBytes(rlpdata, &tx) + tx.UnmarshalBinary(rlpdata) add("OnApproved - SignTransactionResult", desc, ðapi.SignTransactionResult{Raw: rlpdata, Tx: &tx}) } diff --git a/cmd/evm/README.md b/cmd/evm/README.md index 8f0848bde8..7742ccbbb7 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -30,7 +30,7 @@ Command line params that has to be supported are --trace.nomemory Disable full memory dump in traces --trace.nostack Disable stack output in traces --trace.noreturndata Disable return data output in traces - --output.basedir value Specifies where output files are placed. Will be created if it does not exist. (default: ".") + --output.basedir value Specifies where output files are placed. Will be created if it does not exist. --output.alloc alloc Determines where to put the alloc of the post-state. `stdout` - into the stdout output `stderr` - into the stderr output @@ -237,10 +237,10 @@ Example where blockhashes are provided: cat trace-0-0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81.jsonl | grep BLOCKHASH -C2 ``` ``` -{"pc":0,"op":96,"gas":"0x5f58ef8","gasCost":"0x3","memory":"0x","memSize":0,"stack":[],"returnStack":[],"returnData":null,"depth":1,"refund":0,"opName":"PUSH1","error":""} -{"pc":2,"op":64,"gas":"0x5f58ef5","gasCost":"0x14","memory":"0x","memSize":0,"stack":["0x1"],"returnStack":[],"returnData":null,"depth":1,"refund":0,"opName":"BLOCKHASH","error":""} -{"pc":3,"op":0,"gas":"0x5f58ee1","gasCost":"0x0","memory":"0x","memSize":0,"stack":["0xdac58aa524e50956d0c0bae7f3f8bb9d35381365d07804dd5b48a5a297c06af4"],"returnStack":[],"returnData":null,"depth":1,"refund":0,"opName":"STOP","error":""} -{"output":"","gasUsed":"0x17","time":112885} +{"pc":0,"op":96,"gas":"0x5f58ef8","gasCost":"0x3","memory":"0x","memSize":0,"stack":[],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"PUSH1","error":""} +{"pc":2,"op":64,"gas":"0x5f58ef5","gasCost":"0x14","memory":"0x","memSize":0,"stack":["0x1"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"BLOCKHASH","error":""} +{"pc":3,"op":0,"gas":"0x5f58ee1","gasCost":"0x0","memory":"0x","memSize":0,"stack":["0xdac58aa524e50956d0c0bae7f3f8bb9d35381365d07804dd5b48a5a297c06af4"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"STOP","error":""} +{"output":"","gasUsed":"0x17","time":142709} ``` In this example, the caller has not provided the required blockhash: @@ -256,9 +256,9 @@ Error code: 4 Another thing that can be done, is to chain invocations: ``` ./evm t8n --input.alloc=./testdata/1/alloc.json --input.txs=./testdata/1/txs.json --input.env=./testdata/1/env.json --output.alloc=stdout | ./evm t8n --input.alloc=stdin --input.env=./testdata/1/env.json --input.txs=./testdata/1/txs.json -INFO [08-03|15:25:15.168] rejected tx index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low" -INFO [08-03|15:25:15.169] rejected tx index=0 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low" -INFO [08-03|15:25:15.169] rejected tx index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low" +INFO [01-21|22:41:22.963] rejected tx index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [01-21|22:41:22.966] rejected tx index=0 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [01-21|22:41:22.967] rejected tx index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" ``` What happened here, is that we first applied two identical transactions, so the second one was rejected. @@ -267,4 +267,3 @@ the same two transactions: this time, both failed due to too low nonce. In order to meaningfully chain invocations, one would need to provide meaningful new `env`, otherwise the actual blocknumber (exposed to the EVM) would not increase. - diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 525723e3cb..c3f1b16efc 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -143,19 +143,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, vmConfig.Debug = (tracer != nil) statedb.Prepare(tx.Hash(), blockHash, txIndex) txContext := core.NewEVMTxContext(msg) - - evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) - if chainConfig.IsYoloV3(vmContext.BlockNumber) { - statedb.AddAddressToAccessList(msg.From()) - if dst := msg.To(); dst != nil { - statedb.AddAddressToAccessList(*dst) - // If it's a create-tx, the destination will be added inside evm.create - } - for _, addr := range evm.ActivePrecompiles() { - statedb.AddAddressToAccessList(addr) - } - } snapshot := statedb.Snapshot() + evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) + // (ret []byte, usedGas uint64, failed bool, err error) msgResult, err := core.ApplyMessage(evm, msg, gaspool) if err != nil { @@ -169,7 +159,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return nil, nil, NewError(ErrorMissingBlockhash, hashError) } gasUsed += msgResult.UsedGas - // Create a new receipt for the transaction, storing the intermediate root and gas used by the tx + + // Receipt: { var root []byte if chainConfig.IsByzantium(vmContext.BlockNumber) { @@ -178,22 +169,32 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, root = statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber)).Bytes() } - receipt := types.NewReceipt(root, msgResult.Failed(), gasUsed) + // Create a new receipt for the transaction, storing the intermediate root and + // gas used by the tx. + receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: gasUsed} + if msgResult.Failed() { + receipt.Status = types.ReceiptStatusFailed + } else { + receipt.Status = types.ReceiptStatusSuccessful + } receipt.TxHash = tx.Hash() receipt.GasUsed = msgResult.UsedGas - // if the transaction created a contract, store the creation address in the receipt. + + // If the transaction created a contract, store the creation address in the receipt. if msg.To() == nil { receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce()) } - // Set the receipt logs and create a bloom for filtering + + // Set the receipt logs and create the bloom filter. receipt.Logs = statedb.GetLogs(tx.Hash()) receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) - // These three are non-consensus fields + // These three are non-consensus fields: //receipt.BlockHash - //receipt.BlockNumber = + //receipt.BlockNumber receipt.TransactionIndex = uint(txIndex) receipts = append(receipts, receipt) } + txIndex++ } statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber)) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index 424156ba82..a599462cc6 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -47,6 +47,11 @@ var ( Usage: "Specifies where output files are placed. Will be created if it does not exist.", Value: "", } + OutputBodyFlag = cli.StringFlag{ + Name: "output.body", + Usage: "If set, the RLP of the transactions (block body) will be written to this file.", + Value: "", + } OutputAllocFlag = cli.StringFlag{ Name: "output.alloc", Usage: "Determines where to put the `alloc` of the post-state.\n" + diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 5119ed5fb7..fedcd12435 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -17,6 +17,7 @@ package t8ntool import ( + "crypto/ecdsa" "encoding/json" "fmt" "io/ioutil" @@ -25,12 +26,15 @@ import ( "path" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" "gopkg.in/urfave/cli.v1" ) @@ -64,9 +68,9 @@ func (n *NumberedError) Code() int { } type input struct { - Alloc core.GenesisAlloc `json:"alloc,omitempty"` - Env *stEnv `json:"env,omitempty"` - Txs types.Transactions `json:"txs,omitempty"` + Alloc core.GenesisAlloc `json:"alloc,omitempty"` + Env *stEnv `json:"env,omitempty"` + Txs []*txWithKey `json:"txs,omitempty"` } func Main(ctx *cli.Context) error { @@ -135,7 +139,7 @@ func Main(ctx *cli.Context) error { txStr = ctx.String(InputTxsFlag.Name) inputData = &input{} ) - + // Figure out the prestate alloc if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) decoder.Decode(inputData) @@ -151,7 +155,9 @@ func Main(ctx *cli.Context) error { return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling alloc-file: %v", err)) } } + prestate.Pre = inputData.Alloc + // Set the block environment if envStr != stdinSelector { inFile, err := os.Open(envStr) if err != nil { @@ -165,26 +171,8 @@ func Main(ctx *cli.Context) error { } inputData.Env = &env } - - if txStr != stdinSelector { - inFile, err := os.Open(txStr) - if err != nil { - return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err)) - } - defer inFile.Close() - decoder := json.NewDecoder(inFile) - var txs types.Transactions - if err := decoder.Decode(&txs); err != nil { - return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling txs-file: %v", err)) - } - inputData.Txs = txs - } - - prestate.Pre = inputData.Alloc prestate.Env = *inputData.Env - txs = inputData.Txs - // Iterate over all the tests, run them and aggregate the results vmConfig := vm.Config{ Tracer: tracer, Debug: (tracer != nil), @@ -200,17 +188,105 @@ func Main(ctx *cli.Context) error { // Set the chain id chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name)) + var txsWithKeys []*txWithKey + if txStr != stdinSelector { + inFile, err := os.Open(txStr) + if err != nil { + return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err)) + } + defer inFile.Close() + decoder := json.NewDecoder(inFile) + if err := decoder.Decode(&txsWithKeys); err != nil { + return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling txs-file: %v", err)) + } + } else { + txsWithKeys = inputData.Txs + } + // We may have to sign the transactions. + signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number))) + + if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil { + return NewError(ErrorJson, fmt.Errorf("Failed signing transactions: %v", err)) + } + + // Iterate over all the tests, run them and aggregate the results + // Run the test and aggregate the result state, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { return err } + body, _ := rlp.EncodeToBytes(txs) // Dump the excution result - //postAlloc := state.DumpGenesisFormat(false, false, false) collector := make(Alloc) state.DumpToCollector(collector, false, false, false, nil, -1) - return dispatchOutput(ctx, baseDir, result, collector) + return dispatchOutput(ctx, baseDir, result, collector, body) + +} +// txWithKey is a helper-struct, to allow us to use the types.Transaction along with +// a `secretKey`-field, for input +type txWithKey struct { + key *ecdsa.PrivateKey + tx *types.Transaction +} + +func (t *txWithKey) UnmarshalJSON(input []byte) error { + // Read the secretKey, if present + type sKey struct { + Key *common.Hash `json:"secretKey"` + } + var key sKey + if err := json.Unmarshal(input, &key); err != nil { + return err + } + if key.Key != nil { + k := key.Key.Hex()[2:] + if ecdsaKey, err := crypto.HexToECDSA(k); err != nil { + return err + } else { + t.key = ecdsaKey + } + } + // Now, read the transaction itself + var tx types.Transaction + if err := json.Unmarshal(input, &tx); err != nil { + return err + } + t.tx = &tx + return nil +} + +// signUnsignedTransactions converts the input txs to canonical transactions. +// +// The transactions can have two forms, either +// 1. unsigned or +// 2. signed +// For (1), r, s, v, need so be zero, and the `secretKey` needs to be set. +// If so, we sign it here and now, with the given `secretKey` +// If the condition above is not met, then it's considered a signed transaction. +// +// To manage this, we read the transactions twice, first trying to read the secretKeys, +// and secondly to read them with the standard tx json format +func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) { + var signedTxs []*types.Transaction + for i, txWithKey := range txs { + tx := txWithKey.tx + key := txWithKey.key + v, r, s := tx.RawSignatureValues() + if key != nil && v.BitLen()+r.BitLen()+s.BitLen() == 0 { + // This transaction needs to be signed + signed, err := types.SignTx(tx, signer, key) + if err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("Tx %d: failed to sign tx: %v", i, err)) + } + signedTxs = append(signedTxs, signed) + } else { + // Already signed + signedTxs = append(signedTxs, tx) + } + } + return signedTxs, nil } type Alloc map[common.Address]core.GenesisAccount @@ -241,15 +317,17 @@ func saveFile(baseDir, filename string, data interface{}) error { if err != nil { return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } - if err = ioutil.WriteFile(path.Join(baseDir, filename), b, 0644); err != nil { + location := path.Join(baseDir, filename) + if err = ioutil.WriteFile(location, b, 0644); err != nil { return NewError(ErrorIO, fmt.Errorf("failed writing output: %v", err)) } + log.Info("Wrote file", "file", location) return nil } // dispatchOutput writes the output data to either stderr or stdout, or to the specified // files -func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc) error { +func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes) error { stdOutObject := make(map[string]interface{}) stdErrObject := make(map[string]interface{}) dispatch := func(baseDir, fName, name string, obj interface{}) error { @@ -258,6 +336,8 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a stdOutObject[name] = obj case "stderr": stdErrObject[name] = obj + case "": + // don't save default: // save to file if err := saveFile(baseDir, fName, obj); err != nil { return err @@ -271,6 +351,9 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a if err := dispatch(baseDir, ctx.String(OutputResultFlag.Name), "result", result); err != nil { return err } + if err := dispatch(baseDir, ctx.String(OutputBodyFlag.Name), "body", body); err != nil { + return err + } if len(stdOutObject) > 0 { b, err := json.MarshalIndent(stdOutObject, "", " ") if err != nil { diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 35c672142d..8a3e4e0ea2 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -149,6 +149,7 @@ var stateTransitionCommand = cli.Command{ t8ntool.OutputBasedir, t8ntool.OutputAllocFlag, t8ntool.OutputResultFlag, + t8ntool.OutputBodyFlag, t8ntool.InputAllocFlag, t8ntool.InputEnvFlag, t8ntool.InputTxsFlag, diff --git a/cmd/evm/testdata/8/alloc.json b/cmd/evm/testdata/8/alloc.json new file mode 100644 index 0000000000..1d1b5f86c6 --- /dev/null +++ b/cmd/evm/testdata/8/alloc.json @@ -0,0 +1,11 @@ +{ + "0x000000000000000000000000000000000000aaaa": { + "balance": "0x03", + "code": "0x5854505854", + "nonce": "0x1" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x100000", + "nonce": "0x00" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/8/env.json b/cmd/evm/testdata/8/env.json new file mode 100644 index 0000000000..8b91934724 --- /dev/null +++ b/cmd/evm/testdata/8/env.json @@ -0,0 +1,7 @@ +{ + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x20000", + "currentGasLimit": "0x1000000000", + "currentNumber": "0x1000000", + "currentTimestamp": "0x04" +} \ No newline at end of file diff --git a/cmd/evm/testdata/8/readme.md b/cmd/evm/testdata/8/readme.md new file mode 100644 index 0000000000..778fc6151a --- /dev/null +++ b/cmd/evm/testdata/8/readme.md @@ -0,0 +1,63 @@ +## EIP-2930 testing + +This test contains testcases for EIP-2930, which uses transactions with access lists. + +### Prestate + +The alloc portion contains one contract (`0x000000000000000000000000000000000000aaaa`), containing the +following code: `0x5854505854`: `PC ;SLOAD; POP; PC; SLOAD`. + +Essentialy, this contract does `SLOAD(0)` and `SLOAD(3)`. + +The alloc also contains some funds on `0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b`. + +## Transactions + +There are three transactions, each invokes the contract above. + +1. ACL-transaction, which contains some non-used slots +2. Regular transaction +3. ACL-transaction, which contains the slots `1` and `3` in `0x000000000000000000000000000000000000aaaa` + +## Execution + +Running it yields: +``` +dir=./testdata/8 && ./evm t8n --state.fork=Berlin --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --trace && cat trace-* | grep SLOAD +{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":4,"op":84,"gas":"0x47c86","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":1,"op":84,"gas":"0x49cf6","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":4,"op":84,"gas":"0x494be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":4,"op":84,"gas":"0x48456","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} + +``` + +Simlarly, we can provide the input transactions via `stdin` instead of as file: + +``` +dir=./testdata/8 \ + && cat $dir/txs.json | jq "{txs: .}" \ + | ./evm t8n --state.fork=Berlin \ + --input.alloc=$dir/alloc.json \ + --input.txs=stdin \ + --input.env=$dir/env.json \ + --trace \ + && cat trace-* | grep SLOAD + +{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":4,"op":84,"gas":"0x47c86","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":1,"op":84,"gas":"0x49cf6","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":4,"op":84,"gas":"0x494be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":4,"op":84,"gas":"0x48456","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +``` + +If we try to execute it on older rules: +``` +dir=./testdata/8 && ./evm t8n --state.fork=Istanbul --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json +INFO [01-21|23:21:51.265] rejected tx index=0 hash="d2818d…6ab3da" error="tx type not supported" +INFO [01-21|23:21:51.265] rejected tx index=1 hash="26ea00…81c01b" from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0" +INFO [01-21|23:21:51.265] rejected tx index=2 hash="698d01…369cee" error="tx type not supported" +``` +Number `1` and `3` are not applicable, and therefore number `2` has wrong nonce. \ No newline at end of file diff --git a/cmd/evm/testdata/8/txs.json b/cmd/evm/testdata/8/txs.json new file mode 100644 index 0000000000..35142ba234 --- /dev/null +++ b/cmd/evm/testdata/8/txs.json @@ -0,0 +1,58 @@ +[ + { + "gas": "0x4ef00", + "gasPrice": "0x1", + "chainId": "0x1", + "input": "0x", + "nonce": "0x0", + "to": "0x000000000000000000000000000000000000aaaa", + "value": "0x1", + "type" : "0x1", + "accessList": [ + {"address": "0x0000000000000000000000000000000000000000", + "storageKeys": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "v": "0x0", + "r": "0x0", + "s": "0x0", + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + }, + { + "gas": "0x4ef00", + "gasPrice": "0x1", + "input": "0x", + "nonce": "0x1", + "to": "0x000000000000000000000000000000000000aaaa", + "value": "0x2", + "v": "0x0", + "r": "0x0", + "s": "0x0", + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + }, + { + "gas": "0x4ef00", + "gasPrice": "0x1", + "chainId": "0x1", + "input": "0x", + "nonce": "0x2", + "to": "0x000000000000000000000000000000000000aaaa", + "value": "0x1", + "type" : "0x1", + "accessList": [ + {"address": "0x000000000000000000000000000000000000aaaa", + "storageKeys": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000003" + ] + } + ], + "v": "0x0", + "r": "0x0", + "s": "0x0", + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + } +] diff --git a/cmd/geth/main.go b/cmd/geth/main.go index b4c622ce2e..a3af44f6c4 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -140,9 +140,7 @@ var ( utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - // YOLOv3 is not yet complete! - // TODO: enable this once 2718/2930 is added - //utils.YoloV3Flag, + utils.YoloV3Flag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, @@ -286,6 +284,9 @@ func prepare(ctx *cli.Context) { case ctx.GlobalIsSet(utils.GoerliFlag.Name): log.Info("Starting Geth on Görli testnet...") + case ctx.GlobalIsSet(utils.YoloV3Flag.Name): + log.Info("Starting Geth on YOLOv3 testnet...") + case ctx.GlobalIsSet(utils.DeveloperFlag.Name): log.Info("Starting Geth in ephemeral dev mode...") diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 24215f55a8..6935adabc7 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -44,8 +44,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MainnetFlag, utils.GoerliFlag, utils.RinkebyFlag, - // TODO: Re-enable this when 2718/2930 is added - //utils.YoloV3Flag, + utils.YoloV3Flag, utils.RopstenFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 9fff183a14..57547d0c49 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1277,7 +1277,7 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") case ctx.GlobalBool(YoloV3Flag.Name) && cfg.DataDir == node.DefaultDataDir(): - cfg.DataDir = filepath.Join(node.DefaultDataDir(), "yolo-v2") + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "yolo-v3") } } @@ -1609,7 +1609,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) case ctx.GlobalBool(YoloV3Flag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = new(big.Int).SetBytes([]byte("yolov3")).Uint64() // "yolov3" + cfg.NetworkId = new(big.Int).SetBytes([]byte("yolov3x")).Uint64() // "yolov3x" } cfg.Genesis = core.DefaultYoloV3GenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): diff --git a/core/bench_test.go b/core/bench_test.go index 0f4cabd837..85653ea5db 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -85,7 +85,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) { return func(i int, gen *BlockGen) { toaddr := common.Address{} data := make([]byte, nbytes) - gas, _ := IntrinsicGas(data, false, false, false) + gas, _ := IntrinsicGas(data, nil, false, false, false) tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(benchRootAddr), toaddr, big.NewInt(1), gas, nil, data), types.HomesteadSigner{}, benchRootKey) gen.AddTx(tx) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index c33e7321ec..3e4757f8b6 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -600,7 +600,7 @@ func TestFastVsFullChains(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, } genesis = gspec.MustCommit(gendb) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, 1024, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) @@ -839,7 +839,7 @@ func TestChainTxReorgs(t *testing.T) { }, } genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) // Create two transactions shared between the chains: @@ -944,7 +944,7 @@ func TestLogReorgs(t *testing.T) { code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -998,7 +998,7 @@ func TestLogRebirth(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() blockchain, _ = NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil) ) @@ -1062,7 +1062,7 @@ func TestSideLogRebirth(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) ) @@ -1135,7 +1135,7 @@ func TestReorgSideEvent(t *testing.T) { Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}, } genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -1295,7 +1295,7 @@ func TestEIP155Transition(t *testing.T) { } block.AddTx(tx) - tx, err = basicTx(types.NewEIP155Signer(gspec.Config.ChainID)) + tx, err = basicTx(types.LatestSigner(gspec.Config)) if err != nil { t.Fatal(err) } @@ -1307,7 +1307,7 @@ func TestEIP155Transition(t *testing.T) { } block.AddTx(tx) - tx, err = basicTx(types.NewEIP155Signer(gspec.Config.ChainID)) + tx, err = basicTx(types.LatestSigner(gspec.Config)) if err != nil { t.Fatal(err) } @@ -1345,7 +1345,7 @@ func TestEIP155Transition(t *testing.T) { } ) if i == 0 { - tx, err = basicTx(types.NewEIP155Signer(big.NewInt(2))) + tx, err = basicTx(types.LatestSigner(config)) if err != nil { t.Fatal(err) } @@ -1385,7 +1385,7 @@ func TestEIP161AccountRemoval(t *testing.T) { var ( tx *types.Transaction err error - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) switch i { case 0: @@ -2078,7 +2078,7 @@ func TestTransactionIndices(t *testing.T) { funds = big.NewInt(1000000000) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} genesis = gspec.MustCommit(gendb) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) height := uint64(128) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), func(i int, block *BlockGen) { @@ -2205,7 +2205,7 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) { funds = big.NewInt(1000000000) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} genesis = gspec.MustCommit(gendb) - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) height := uint64(128) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), func(i int, block *BlockGen) { @@ -3030,3 +3030,81 @@ func TestInitThenFailCreateContract(t *testing.T) { } } } + +// TestEIP2718Transition tests that an EIP-2718 transaction will be accepted +// after the fork block has passed. This is verified by sending an EIP-2930 +// access list transaction, which specifies a single slot access, and then +// checking that the gas usage of a hot SLOAD and a cold SLOAD are calculated +// correctly. +func TestEIP2718Transition(t *testing.T) { + var ( + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") + + // Generate a canonical chain to act as the main dataset + engine = ethash.NewFaker() + db = rawdb.NewMemoryDatabase() + + // A sender who makes transactions, has some funds + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{ + Config: params.YoloV3ChainConfig, + Alloc: GenesisAlloc{ + address: {Balance: funds}, + // The address 0xAAAA sloads 0x00 and 0x01 + aa: { + Code: []byte{ + byte(vm.PC), + byte(vm.PC), + byte(vm.SLOAD), + byte(vm.SLOAD), + }, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + genesis = gspec.MustCommit(db) + ) + + blocks, _ := GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{1}) + + // One transaction to 0xAAAA + signer := types.LatestSigner(gspec.Config) + tx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: gspec.Config.ChainID, + Nonce: 0, + To: &aa, + Gas: 30000, + GasPrice: big.NewInt(1), + AccessList: types.AccessList{{ + Address: aa, + StorageKeys: []common.Hash{{0}}, + }}, + }) + b.AddTx(tx) + }) + + // Import the canonical chain + diskdb := rawdb.NewMemoryDatabase() + gspec.MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + + block := chain.GetBlockByNumber(1) + + // Expected gas is intrinsic + 2 * pc + hot load + cold load, since only one load is in the access list + expected := params.TxGas + params.TxAccessListAddressGas + params.TxAccessListStorageKeyGas + vm.GasQuickStep*2 + vm.WarmStorageReadCostEIP2929 + vm.ColdSloadCostEIP2929 + if block.GasUsed() != expected { + t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expected, block.GasUsed()) + + } +} diff --git a/core/error.go b/core/error.go index 5a28be7e1c..197dd81567 100644 --- a/core/error.go +++ b/core/error.go @@ -16,7 +16,11 @@ package core -import "errors" +import ( + "errors" + + "github.com/ethereum/go-ethereum/core/types" +) var ( // ErrKnownBlock is returned when a block to import is already known locally. @@ -63,4 +67,8 @@ var ( // ErrIntrinsicGas is returned if the transaction is specified to use less gas // than required to start the invocation. ErrIntrinsicGas = errors.New("intrinsic gas too low") + + // ErrTxTypeNotSupported is returned if a transaction is not supported in the + // current network configuration. + ErrTxTypeNotSupported = types.ErrTxTypeNotSupported ) diff --git a/core/genesis.go b/core/genesis.go index 462aec45bd..6bcc50b050 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -377,11 +377,11 @@ func DefaultGoerliGenesisBlock() *Genesis { } func DefaultYoloV3GenesisBlock() *Genesis { - // Full genesis: https://gist.github.com/holiman/b2c32a05ff2e2712e11c0787d987d46f + // Full genesis: https://gist.github.com/holiman/c6ed9269dce28304ad176314caa75e97 return &Genesis{ Config: params.YoloV3ChainConfig, - Timestamp: 0x60117f8b, - ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000008a37866fd3627c9205a37c8685666f32ec07bb1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + Timestamp: 0x6027dd2e, + ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000001041afbcb359d5a8dc58c15b2ff51354ff8a217d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), GasLimit: 0x47b760, Difficulty: big.NewInt(1), Alloc: decodePrealloc(yoloV3AllocData), diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 6eecbbf0e8..5b0e933d7a 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,5 +25,4 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const yoloV3AllocData = "\xf9\x05\x01\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - +const yoloV3AllocData = "\xf9\x05o\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xdf\n\x88\xb2\xb6\x8cg7\x13\xa8\xec\x82`\x03go'.5s\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" diff --git a/core/state/statedb.go b/core/state/statedb.go index 8fd066e24f..2e5d6e47c8 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -983,6 +983,32 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { return root, err } +// PrepareAccessList handles the preparatory steps for executing a state transition with +// regards to both EIP-2929 and EIP-2930: +// +// - Add sender to access list (2929) +// - Add destination to access list (2929) +// - Add precompiles to access list (2929) +// - Add the contents of the optional tx access list (2930) +// +// This method should only be called if Yolov3/Berlin/2929+2930 is applicable at the current number. +func (s *StateDB) PrepareAccessList(sender common.Address, dst *common.Address, precompiles []common.Address, list types.AccessList) { + s.AddAddressToAccessList(sender) + if dst != nil { + s.AddAddressToAccessList(*dst) + // If it's a create-tx, the destination will be added inside evm.create + } + for _, addr := range precompiles { + s.AddAddressToAccessList(addr) + } + for _, el := range list { + s.AddAddressToAccessList(el.Address) + for _, key := range el.StorageKeys { + s.AddSlotToAccessList(el.Address, key) + } + } +} + // AddAddressToAccessList adds the given address to the access list func (s *StateDB) AddAddressToAccessList(addr common.Address) { if s.accessList.AddAddress(addr) { diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 6fa52c2d9b..05394321f7 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -19,7 +19,6 @@ package core import ( "sync/atomic" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -50,8 +49,11 @@ func newStatePrefetcher(config *params.ChainConfig, bc *BlockChain, engine conse // only goal is to pre-cache transaction signatures and state trie nodes. func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, cfg vm.Config, interrupt *uint32) { var ( - header = block.Header() - gaspool = new(GasPool).AddGas(block.GasLimit()) + header = block.Header() + gaspool = new(GasPool).AddGas(block.GasLimit()) + blockContext = NewEVMBlockContext(header, p.bc, nil) + evm = vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg) + signer = types.MakeSigner(p.config, header.Number) ) // Iterate over and process the individual transactions byzantium := p.config.IsByzantium(block.Number()) @@ -60,9 +62,13 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c if interrupt != nil && atomic.LoadUint32(interrupt) == 1 { return } - // Block precaching permitted to continue, execute the transaction + // Convert the transaction into an executable message and pre-cache its sender + msg, err := tx.AsMessage(signer) + if err != nil { + return // Also invalid block, bail out + } statedb.Prepare(tx.Hash(), block.Hash(), i) - if err := precacheTransaction(p.config, p.bc, nil, gaspool, statedb, header, tx, cfg); err != nil { + if err := precacheTransaction(msg, p.config, gaspool, statedb, header, evm); err != nil { return // Ugh, something went horribly wrong, bail out } // If we're pre-byzantium, pre-load trie nodes for the intermediate root @@ -79,17 +85,10 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c // precacheTransaction attempts to apply a transaction to the given state database // and uses the input parameters for its environment. The goal is not to execute // the transaction successfully, rather to warm up touched data slots. -func precacheTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gaspool *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, cfg vm.Config) error { - // Convert the transaction into an executable message and pre-cache its sender - msg, err := tx.AsMessage(types.MakeSigner(config, header.Number)) - if err != nil { - return err - } - // Create the EVM and execute the transaction - context := NewEVMBlockContext(header, bc, author) - txContext := NewEVMTxContext(msg) - vm := vm.NewEVM(context, txContext, statedb, config, cfg) - - _, err = ApplyMessage(vm, msg, gaspool) +func precacheTransaction(msg types.Message, config *params.ChainConfig, gaspool *GasPool, statedb *state.StateDB, header *types.Header, evm *vm.EVM) error { + // Update the evm with the new transaction context. + evm.Reset(NewEVMTxContext(msg), statedb) + // Add addresses to access list if applicable + _, err := ApplyMessage(evm, msg, gaspool) return err } diff --git a/core/state_processor.go b/core/state_processor.go index c5b0bb1609..40a953f0d4 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -90,28 +90,17 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { - // Create a new context to be used in the EVM environment + // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) - // Add addresses to access list if applicable - if config.IsYoloV3(header.Number) { - statedb.AddAddressToAccessList(msg.From()) - if dst := msg.To(); dst != nil { - statedb.AddAddressToAccessList(*dst) - // If it's a create-tx, the destination will be added inside evm.create - } - for _, addr := range evm.ActivePrecompiles() { - statedb.AddAddressToAccessList(addr) - } - } - - // Update the evm with the new transaction context. evm.Reset(txContext, statedb) - // Apply the transaction to the current state (included in the env) + + // Apply the transaction to the current state (included in the env). result, err := ApplyMessage(evm, msg, gp) if err != nil { return nil, err } - // Update the state with pending changes + + // Update the state with pending changes. var root []byte if config.IsByzantium(header.Number) { statedb.Finalise(true) @@ -120,22 +109,28 @@ func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainCon } *usedGas += result.UsedGas - // Create a new receipt for the transaction, storing the intermediate root and gas used by the tx - // based on the eip phase, we're passing whether the root touch-delete accounts. - receipt := types.NewReceipt(root, result.Failed(), *usedGas) + // Create a new receipt for the transaction, storing the intermediate root and gas used + // by the tx. + receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: *usedGas} + if result.Failed() { + receipt.Status = types.ReceiptStatusFailed + } else { + receipt.Status = types.ReceiptStatusSuccessful + } receipt.TxHash = tx.Hash() receipt.GasUsed = result.UsedGas - // if the transaction created a contract, store the creation address in the receipt. + + // If the transaction created a contract, store the creation address in the receipt. if msg.To() == nil { receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce()) } - // Set the receipt logs and create a bloom for filtering + + // Set the receipt logs and create the bloom filter. receipt.Logs = statedb.GetLogs(tx.Hash()) receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) receipt.BlockHash = statedb.BlockHash() receipt.BlockNumber = header.Number receipt.TransactionIndex = uint(statedb.TxIndex()) - return receipt, err } diff --git a/core/state_transition.go b/core/state_transition.go index a8d1936c1f..0d589a6119 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -22,6 +22,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" ) @@ -67,6 +68,7 @@ type Message interface { Nonce() uint64 CheckNonce() bool Data() []byte + AccessList() types.AccessList } // ExecutionResult includes all output after executing given evm @@ -105,10 +107,10 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, contractCreation, isHomestead bool, isEIP2028 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 - if contractCreation && isHomestead { + if isContractCreation && isHomestead { gas = params.TxGasContractCreation } else { gas = params.TxGas @@ -138,6 +140,10 @@ func IntrinsicGas(data []byte, contractCreation, isHomestead bool, isEIP2028 boo } gas += z * params.TxDataZeroGas } + if accessList != nil { + gas += uint64(len(accessList)) * params.TxAccessListAddressGas + gas += uint64(accessList.StorageKeys()) * params.TxAccessListStorageKeyGas + } return gas, nil } @@ -238,7 +244,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { contractCreation := msg.To() == nil // Check clauses 4-5, subtract intrinsic gas if everything is correct - gas, err := IntrinsicGas(st.data, contractCreation, homestead, istanbul) + gas, err := IntrinsicGas(st.data, st.msg.AccessList(), contractCreation, homestead, istanbul) if err != nil { return nil, err } @@ -251,6 +257,12 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if msg.Value().Sign() > 0 && !st.evm.Context.CanTransfer(st.state, msg.From(), msg.Value()) { return nil, fmt.Errorf("%w: address %v", ErrInsufficientFundsForTransfer, msg.From().Hex()) } + + // Set up the initial access list. + if st.evm.ChainConfig().IsYoloV3(st.evm.Context.BlockNumber) { + st.state.PrepareAccessList(msg.From(), msg.To(), st.evm.ActivePrecompiles(), msg.AccessList()) + } + var ( ret []byte vmerr error // vm errors do not effect consensus and are therefore not assigned to err diff --git a/core/tx_pool.go b/core/tx_pool.go index 36546cde75..28ac822131 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -229,6 +229,7 @@ type TxPool struct { mu sync.RWMutex istanbul bool // Fork indicator whether we are in the istanbul stage. + eip2718 bool // Fork indicator whether we are using EIP-2718 type transactions. currentState *state.StateDB // Current state in the blockchain head pendingNonces *txNoncer // Pending state tracking virtual nonces @@ -268,7 +269,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block config: config, chainconfig: chainconfig, chain: chain, - signer: types.NewEIP155Signer(chainconfig.ChainID), + signer: types.LatestSigner(chainconfig), pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), @@ -522,6 +523,10 @@ func (pool *TxPool) local() map[common.Address]types.Transactions { // validateTx checks whether a transaction is valid according to the consensus // rules and adheres to some heuristic limits of the local node (price and size). func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { + // Accept only legacy transactions until EIP-2718/2930 activates. + if !pool.eip2718 && tx.Type() != types.LegacyTxType { + return ErrTxTypeNotSupported + } // Reject transactions over defined size to prevent DOS attacks if uint64(tx.Size()) > txMaxSize { return ErrOversizedData @@ -535,7 +540,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { if pool.currentMaxGas < tx.Gas() { return ErrGasLimit } - // Make sure the transaction is signed properly + // Make sure the transaction is signed properly. from, err := types.Sender(pool.signer, tx) if err != nil { return ErrInvalidSender @@ -554,7 +559,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { return ErrInsufficientFunds } // Ensure the transaction has more gas than the basic tx fee. - intrGas, err := IntrinsicGas(tx.Data(), tx.To() == nil, true, pool.istanbul) + intrGas, err := IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, true, pool.istanbul) if err != nil { return err } @@ -1199,6 +1204,7 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { // Update all fork indicator by next pending block number. next := new(big.Int).Add(newHead.Number, big.NewInt(1)) pool.istanbul = pool.chainconfig.IsIstanbul(next) + pool.eip2718 = pool.chainconfig.IsYoloV3(next) } // promoteExecutables moves transactions that have become processable from the diff --git a/core/types/access_list_tx.go b/core/types/access_list_tx.go new file mode 100644 index 0000000000..65ee95adf6 --- /dev/null +++ b/core/types/access_list_tx.go @@ -0,0 +1,115 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +//go:generate gencodec -type AccessTuple -out gen_access_tuple.go + +// AccessList is an EIP-2930 access list. +type AccessList []AccessTuple + +// AccessTuple is the element type of an access list. +type AccessTuple struct { + Address common.Address `json:"address" gencodec:"required"` + StorageKeys []common.Hash `json:"storageKeys" gencodec:"required"` +} + +// StorageKeys returns the total number of storage keys in the access list. +func (al AccessList) StorageKeys() int { + sum := 0 + for _, tuple := range al { + sum += len(tuple.StorageKeys) + } + return sum +} + +// AccessListTx is the data of EIP-2930 access list transactions. +type AccessListTx struct { + ChainID *big.Int // destination chain ID + Nonce uint64 // nonce of sender account + GasPrice *big.Int // wei per gas + Gas uint64 // gas limit + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int // wei amount + Data []byte // contract invocation input data + AccessList AccessList // EIP-2930 access list + V, R, S *big.Int // signature values +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *AccessListTx) copy() TxData { + cpy := &AccessListTx{ + Nonce: tx.Nonce, + To: tx.To, // TODO: copy pointed-to address + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are copied below. + AccessList: make(AccessList, len(tx.AccessList)), + Value: new(big.Int), + ChainID: new(big.Int), + GasPrice: new(big.Int), + V: new(big.Int), + R: new(big.Int), + S: new(big.Int), + } + copy(cpy.AccessList, tx.AccessList) + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.ChainID != nil { + cpy.ChainID.Set(tx.ChainID) + } + if tx.GasPrice != nil { + cpy.GasPrice.Set(tx.GasPrice) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. + +func (tx *AccessListTx) txType() byte { return AccessListTxType } +func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } +func (tx *AccessListTx) protected() bool { return true } +func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } +func (tx *AccessListTx) data() []byte { return tx.Data } +func (tx *AccessListTx) gas() uint64 { return tx.Gas } +func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) value() *big.Int { return tx.Value } +func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } +func (tx *AccessListTx) to() *common.Address { return tx.To } + +func (tx *AccessListTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V, tx.R, tx.S +} + +func (tx *AccessListTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s +} diff --git a/core/types/block.go b/core/types/block.go index 8096ebb755..553db003bb 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -23,15 +23,12 @@ import ( "io" "math/big" "reflect" - "sync" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" ) var ( @@ -131,22 +128,6 @@ func (h *Header) SanityCheck() error { return nil } -// hasherPool holds LegacyKeccak hashers. -var hasherPool = sync.Pool{ - New: func() interface{} { - return sha3.NewLegacyKeccak256() - }, -} - -func rlpHash(x interface{}) (h common.Hash) { - sha := hasherPool.Get().(crypto.KeccakState) - defer hasherPool.Put(sha) - sha.Reset() - rlp.Encode(sha, x) - sha.Read(h[:]) - return h -} - // EmptyBody returns true if there is no additional 'body' to complete the header // that is: no transactions and no uncles. func (h *Header) EmptyBody() bool { @@ -221,7 +202,7 @@ type storageblock struct { // The values of TxHash, UncleHash, ReceiptHash and Bloom in header // are ignored and set to values derived from the given txs, uncles // and receipts. -func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher Hasher) *Block { +func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher TrieHasher) *Block { b := &Block{header: CopyHeader(header), td: new(big.Int)} // TODO: panic if len(txs) != len(receipts) diff --git a/core/types/block_test.go b/core/types/block_test.go index 4dfdcf9545..63904f882c 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -59,6 +59,66 @@ func TestBlockEncoding(t *testing.T) { tx1, _ = tx1.WithSignature(HomesteadSigner{}, common.Hex2Bytes("9bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094f8a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b100")) check("len(Transactions)", len(block.Transactions()), 1) check("Transactions[0].Hash", block.Transactions()[0].Hash(), tx1.Hash()) + ourBlockEnc, err := rlp.EncodeToBytes(&block) + if err != nil { + t.Fatal("encode error: ", err) + } + if !bytes.Equal(ourBlockEnc, blockEnc) { + t.Errorf("encoded block mismatch:\ngot: %x\nwant: %x", ourBlockEnc, blockEnc) + } +} + +func TestEIP2718BlockEncoding(t *testing.T) { + blockEnc := common.FromHex("f90319f90211a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a0e6e49996c7ec59f7a23d22b83239a60151512c65613bf84a0d7da336399ebc4aa0cafe75574d59780665a97fbfd11365c7545aa8f1abf4e5e12e8243334ef7286bb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000820200832fefd882a410845506eb0796636f6f6c65737420626c6f636b206f6e20636861696ea0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4f90101f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1b89e01f89b01800a8301e24194095e7baea6a6c7c4c2dfeb977efac326af552d878080f838f7940000000000000000000000000000000000000001e1a0000000000000000000000000000000000000000000000000000000000000000001a03dbacc8d0259f2508625e97fdfc57cd85fdd16e5821bc2c10bdd1a52649e8335a0476e10695b183a87b0aa292a7f4b78ef0c3fbe62aa2c42c84e1d9c3da159ef14c0") + var block Block + if err := rlp.DecodeBytes(blockEnc, &block); err != nil { + t.Fatal("decode error: ", err) + } + + check := func(f string, got, want interface{}) { + if !reflect.DeepEqual(got, want) { + t.Errorf("%s mismatch: got %v, want %v", f, got, want) + } + } + check("Difficulty", block.Difficulty(), big.NewInt(131072)) + check("GasLimit", block.GasLimit(), uint64(3141592)) + check("GasUsed", block.GasUsed(), uint64(42000)) + check("Coinbase", block.Coinbase(), common.HexToAddress("8888f1f195afa192cfee860698584c030f4c9db1")) + check("MixDigest", block.MixDigest(), common.HexToHash("bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff498")) + check("Root", block.Root(), common.HexToHash("ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017")) + check("Nonce", block.Nonce(), uint64(0xa13a5a8c8f2bb1c4)) + check("Time", block.Time(), uint64(1426516743)) + check("Size", block.Size(), common.StorageSize(len(blockEnc))) + + // Create legacy tx. + to := common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87") + tx1 := NewTx(&LegacyTx{ + Nonce: 0, + To: &to, + Value: big.NewInt(10), + Gas: 50000, + GasPrice: big.NewInt(10), + }) + sig := common.Hex2Bytes("9bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094f8a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b100") + tx1, _ = tx1.WithSignature(HomesteadSigner{}, sig) + + // Create ACL tx. + addr := common.HexToAddress("0x0000000000000000000000000000000000000001") + tx2 := NewTx(&AccessListTx{ + ChainID: big.NewInt(1), + Nonce: 0, + To: &to, + Gas: 123457, + GasPrice: big.NewInt(10), + AccessList: AccessList{{Address: addr, StorageKeys: []common.Hash{{0}}}}, + }) + sig2 := common.Hex2Bytes("3dbacc8d0259f2508625e97fdfc57cd85fdd16e5821bc2c10bdd1a52649e8335476e10695b183a87b0aa292a7f4b78ef0c3fbe62aa2c42c84e1d9c3da159ef1401") + tx2, _ = tx2.WithSignature(NewEIP2930Signer(big.NewInt(1)), sig2) + + check("len(Transactions)", len(block.Transactions()), 2) + check("Transactions[0].Hash", block.Transactions()[0].Hash(), tx1.Hash()) + check("Transactions[1].Hash", block.Transactions()[1].Hash(), tx2.Hash()) + check("Transactions[1].Type()", block.Transactions()[1].Type(), uint8(AccessListTxType)) ourBlockEnc, err := rlp.EncodeToBytes(&block) if err != nil { @@ -121,7 +181,7 @@ func makeBenchBlock() *Block { key, _ = crypto.GenerateKey() txs = make([]*Transaction, 70) receipts = make([]*Receipt, len(txs)) - signer = NewEIP155Signer(params.TestChainConfig.ChainID) + signer = LatestSigner(params.TestChainConfig) uncles = make([]*Header, 3) ) header := &Header{ diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go deleted file mode 100644 index 51a10f3f3d..0000000000 --- a/core/types/derive_sha.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package types - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" -) - -type DerivableList interface { - Len() int - GetRlp(i int) []byte -} - -// Hasher is the tool used to calculate the hash of derivable list. -type Hasher interface { - Reset() - Update([]byte, []byte) - Hash() common.Hash -} - -func DeriveSha(list DerivableList, hasher Hasher) common.Hash { - hasher.Reset() - - // StackTrie requires values to be inserted in increasing - // hash order, which is not the order that `list` provides - // hashes in. This insertion sequence ensures that the - // order is correct. - - var buf []byte - for i := 1; i < list.Len() && i <= 0x7f; i++ { - buf = rlp.AppendUint64(buf[:0], uint64(i)) - hasher.Update(buf, list.GetRlp(i)) - } - if list.Len() > 0 { - buf = rlp.AppendUint64(buf[:0], 0) - hasher.Update(buf, list.GetRlp(0)) - } - for i := 0x80; i < list.Len(); i++ { - buf = rlp.AppendUint64(buf[:0], uint64(i)) - hasher.Update(buf, list.GetRlp(i)) - } - return hasher.Hash() -} diff --git a/core/types/gen_access_tuple.go b/core/types/gen_access_tuple.go new file mode 100644 index 0000000000..fc48a84cc0 --- /dev/null +++ b/core/types/gen_access_tuple.go @@ -0,0 +1,43 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package types + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" +) + +// MarshalJSON marshals as JSON. +func (a AccessTuple) MarshalJSON() ([]byte, error) { + type AccessTuple struct { + Address common.Address `json:"address" gencodec:"required"` + StorageKeys []common.Hash `json:"storageKeys" gencodec:"required"` + } + var enc AccessTuple + enc.Address = a.Address + enc.StorageKeys = a.StorageKeys + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (a *AccessTuple) UnmarshalJSON(input []byte) error { + type AccessTuple struct { + Address *common.Address `json:"address" gencodec:"required"` + StorageKeys []common.Hash `json:"storageKeys" gencodec:"required"` + } + var dec AccessTuple + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Address == nil { + return errors.New("missing required field 'address' for AccessTuple") + } + a.Address = *dec.Address + if dec.StorageKeys == nil { + return errors.New("missing required field 'storageKeys' for AccessTuple") + } + a.StorageKeys = dec.StorageKeys + return nil +} diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index 790ed65b58..bb892f85be 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -16,6 +16,7 @@ var _ = (*receiptMarshaling)(nil) // MarshalJSON marshals as JSON. func (r Receipt) MarshalJSON() ([]byte, error) { type Receipt struct { + Type hexutil.Uint64 `json:"type,omitempty"` PostState hexutil.Bytes `json:"root"` Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -29,6 +30,7 @@ func (r Receipt) MarshalJSON() ([]byte, error) { TransactionIndex hexutil.Uint `json:"transactionIndex"` } var enc Receipt + enc.Type = hexutil.Uint64(r.Type) enc.PostState = r.PostState enc.Status = hexutil.Uint64(r.Status) enc.CumulativeGasUsed = hexutil.Uint64(r.CumulativeGasUsed) @@ -46,6 +48,7 @@ func (r Receipt) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (r *Receipt) UnmarshalJSON(input []byte) error { type Receipt struct { + Type *hexutil.Uint64 `json:"type,omitempty"` PostState *hexutil.Bytes `json:"root"` Status *hexutil.Uint64 `json:"status"` CumulativeGasUsed *hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -62,6 +65,9 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { if err := json.Unmarshal(input, &dec); err != nil { return err } + if dec.Type != nil { + r.Type = uint8(*dec.Type) + } if dec.PostState != nil { r.PostState = *dec.PostState } diff --git a/core/types/gen_tx_json.go b/core/types/gen_tx_json.go deleted file mode 100644 index e676058ecc..0000000000 --- a/core/types/gen_tx_json.go +++ /dev/null @@ -1,101 +0,0 @@ -// Code generated by github.com/fjl/gencodec. DO NOT EDIT. - -package types - -import ( - "encoding/json" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" -) - -var _ = (*txdataMarshaling)(nil) - -// MarshalJSON marshals as JSON. -func (t txdata) MarshalJSON() ([]byte, error) { - type txdata struct { - AccountNonce hexutil.Uint64 `json:"nonce" gencodec:"required"` - Price *hexutil.Big `json:"gasPrice" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gas" gencodec:"required"` - Recipient *common.Address `json:"to" rlp:"nil"` - Amount *hexutil.Big `json:"value" gencodec:"required"` - Payload hexutil.Bytes `json:"input" gencodec:"required"` - V *hexutil.Big `json:"v" gencodec:"required"` - R *hexutil.Big `json:"r" gencodec:"required"` - S *hexutil.Big `json:"s" gencodec:"required"` - Hash *common.Hash `json:"hash" rlp:"-"` - } - var enc txdata - enc.AccountNonce = hexutil.Uint64(t.AccountNonce) - enc.Price = (*hexutil.Big)(t.Price) - enc.GasLimit = hexutil.Uint64(t.GasLimit) - enc.Recipient = t.Recipient - enc.Amount = (*hexutil.Big)(t.Amount) - enc.Payload = t.Payload - enc.V = (*hexutil.Big)(t.V) - enc.R = (*hexutil.Big)(t.R) - enc.S = (*hexutil.Big)(t.S) - enc.Hash = t.Hash - return json.Marshal(&enc) -} - -// UnmarshalJSON unmarshals from JSON. -func (t *txdata) UnmarshalJSON(input []byte) error { - type txdata struct { - AccountNonce *hexutil.Uint64 `json:"nonce" gencodec:"required"` - Price *hexutil.Big `json:"gasPrice" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gas" gencodec:"required"` - Recipient *common.Address `json:"to" rlp:"nil"` - Amount *hexutil.Big `json:"value" gencodec:"required"` - Payload *hexutil.Bytes `json:"input" gencodec:"required"` - V *hexutil.Big `json:"v" gencodec:"required"` - R *hexutil.Big `json:"r" gencodec:"required"` - S *hexutil.Big `json:"s" gencodec:"required"` - Hash *common.Hash `json:"hash" rlp:"-"` - } - var dec txdata - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - if dec.AccountNonce == nil { - return errors.New("missing required field 'nonce' for txdata") - } - t.AccountNonce = uint64(*dec.AccountNonce) - if dec.Price == nil { - return errors.New("missing required field 'gasPrice' for txdata") - } - t.Price = (*big.Int)(dec.Price) - if dec.GasLimit == nil { - return errors.New("missing required field 'gas' for txdata") - } - t.GasLimit = uint64(*dec.GasLimit) - if dec.Recipient != nil { - t.Recipient = dec.Recipient - } - if dec.Amount == nil { - return errors.New("missing required field 'value' for txdata") - } - t.Amount = (*big.Int)(dec.Amount) - if dec.Payload == nil { - return errors.New("missing required field 'input' for txdata") - } - t.Payload = *dec.Payload - if dec.V == nil { - return errors.New("missing required field 'v' for txdata") - } - t.V = (*big.Int)(dec.V) - if dec.R == nil { - return errors.New("missing required field 'r' for txdata") - } - t.R = (*big.Int)(dec.R) - if dec.S == nil { - return errors.New("missing required field 's' for txdata") - } - t.S = (*big.Int)(dec.S) - if dec.Hash != nil { - t.Hash = dec.Hash - } - return nil -} diff --git a/core/types/hashing.go b/core/types/hashing.go new file mode 100644 index 0000000000..71efb25a9a --- /dev/null +++ b/core/types/hashing.go @@ -0,0 +1,112 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "bytes" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/crypto/sha3" +) + +// hasherPool holds LegacyKeccak256 hashers for rlpHash. +var hasherPool = sync.Pool{ + New: func() interface{} { return sha3.NewLegacyKeccak256() }, +} + +// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. +var encodeBufferPool = sync.Pool{ + New: func() interface{} { return new(bytes.Buffer) }, +} + +func rlpHash(x interface{}) (h common.Hash) { + sha := hasherPool.Get().(crypto.KeccakState) + defer hasherPool.Put(sha) + sha.Reset() + rlp.Encode(sha, x) + sha.Read(h[:]) + return h +} + +// prefixedRlpHash writes the prefix into the hasher before rlp-encoding the +// given interface. It's used for typed transactions. +func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) { + sha := hasherPool.Get().(crypto.KeccakState) + defer hasherPool.Put(sha) + sha.Reset() + sha.Write([]byte{prefix}) + rlp.Encode(sha, x) + sha.Read(h[:]) + return h +} + +// TrieHasher is the tool used to calculate the hash of derivable list. +// This is internal, do not use. +type TrieHasher interface { + Reset() + Update([]byte, []byte) + Hash() common.Hash +} + +// DerivableList is the input to DeriveSha. +// It is implemented by the 'Transactions' and 'Receipts' types. +// This is internal, do not use these methods. +type DerivableList interface { + Len() int + EncodeIndex(int, *bytes.Buffer) +} + +func encodeForDerive(list DerivableList, i int, buf *bytes.Buffer) []byte { + buf.Reset() + list.EncodeIndex(i, buf) + // It's really unfortunate that we need to do perform this copy. + // StackTrie holds onto the values until Hash is called, so the values + // written to it must not alias. + return common.CopyBytes(buf.Bytes()) +} + +// DeriveSha creates the tree hashes of transactions and receipts in a block header. +func DeriveSha(list DerivableList, hasher TrieHasher) common.Hash { + hasher.Reset() + + valueBuf := encodeBufferPool.Get().(*bytes.Buffer) + defer encodeBufferPool.Put(valueBuf) + + // StackTrie requires values to be inserted in increasing hash order, which is not the + // order that `list` provides hashes in. This insertion sequence ensures that the + // order is correct. + var indexBuf []byte + for i := 1; i < list.Len() && i <= 0x7f; i++ { + indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i)) + value := encodeForDerive(list, i, valueBuf) + hasher.Update(indexBuf, value) + } + if list.Len() > 0 { + indexBuf = rlp.AppendUint64(indexBuf[:0], 0) + value := encodeForDerive(list, 0, valueBuf) + hasher.Update(indexBuf, value) + } + for i := 0x80; i < list.Len(); i++ { + indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i)) + value := encodeForDerive(list, i, valueBuf) + hasher.Update(indexBuf, value) + } + return hasher.Hash() +} diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go new file mode 100644 index 0000000000..a948b10ef6 --- /dev/null +++ b/core/types/hashing_test.go @@ -0,0 +1,212 @@ +package types_test + +import ( + "bytes" + "fmt" + "io" + "math/big" + mrand "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +func TestDeriveSha(t *testing.T) { + txs, err := genTxs(0) + if err != nil { + t.Fatal(err) + } + for len(txs) < 1000 { + exp := types.DeriveSha(txs, new(trie.Trie)) + got := types.DeriveSha(txs, trie.NewStackTrie(nil)) + if !bytes.Equal(got[:], exp[:]) { + t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) + } + newTxs, err := genTxs(uint64(len(txs) + 1)) + if err != nil { + t.Fatal(err) + } + txs = append(txs, newTxs...) + } +} + +// TestEIP2718DeriveSha tests that the input to the DeriveSha function is correct. +func TestEIP2718DeriveSha(t *testing.T) { + for _, tc := range []struct { + rlpData string + exp string + }{ + { + rlpData: "0xb8a701f8a486796f6c6f763380843b9aca008262d4948a8eafb1cf62bfbeb1741769dae1a9dd479961928080f838f7940000000000000000000000000000000000001337e1a0000000000000000000000000000000000000000000000000000000000000000080a0775101f92dcca278a56bfe4d613428624a1ebfc3cd9e0bcc1de80c41455b9021a06c9deac205afe7b124907d4ba54a9f46161498bd3990b90d175aac12c9a40ee9", + exp: "01 01f8a486796f6c6f763380843b9aca008262d4948a8eafb1cf62bfbeb1741769dae1a9dd479961928080f838f7940000000000000000000000000000000000001337e1a0000000000000000000000000000000000000000000000000000000000000000080a0775101f92dcca278a56bfe4d613428624a1ebfc3cd9e0bcc1de80c41455b9021a06c9deac205afe7b124907d4ba54a9f46161498bd3990b90d175aac12c9a40ee9\n80 01f8a486796f6c6f763380843b9aca008262d4948a8eafb1cf62bfbeb1741769dae1a9dd479961928080f838f7940000000000000000000000000000000000001337e1a0000000000000000000000000000000000000000000000000000000000000000080a0775101f92dcca278a56bfe4d613428624a1ebfc3cd9e0bcc1de80c41455b9021a06c9deac205afe7b124907d4ba54a9f46161498bd3990b90d175aac12c9a40ee9\n", + }, + } { + d := &hashToHumanReadable{} + var t1, t2 types.Transaction + rlp.DecodeBytes(common.FromHex(tc.rlpData), &t1) + rlp.DecodeBytes(common.FromHex(tc.rlpData), &t2) + txs := types.Transactions{&t1, &t2} + types.DeriveSha(txs, d) + if tc.exp != string(d.data) { + t.Fatalf("Want\n%v\nhave:\n%v", tc.exp, string(d.data)) + } + } +} + +func BenchmarkDeriveSha200(b *testing.B) { + txs, err := genTxs(200) + if err != nil { + b.Fatal(err) + } + var exp common.Hash + var got common.Hash + b.Run("std_trie", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + exp = types.DeriveSha(txs, new(trie.Trie)) + } + }) + + b.Run("stack_trie", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + got = types.DeriveSha(txs, trie.NewStackTrie(nil)) + } + }) + if got != exp { + b.Errorf("got %x exp %x", got, exp) + } +} + +func TestFuzzDeriveSha(t *testing.T) { + // increase this for longer runs -- it's set to quite low for travis + rndSeed := mrand.Int() + for i := 0; i < 10; i++ { + seed := rndSeed + i + exp := types.DeriveSha(newDummy(i), new(trie.Trie)) + got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) + if !bytes.Equal(got[:], exp[:]) { + printList(newDummy(seed)) + t.Fatalf("seed %d: got %x exp %x", seed, got, exp) + } + } +} + +// TestDerivableList contains testcases found via fuzzing +func TestDerivableList(t *testing.T) { + type tcase []string + tcs := []tcase{ + { + "0xc041", + }, + { + "0xf04cf757812428b0763112efb33b6f4fad7deb445e", + "0xf04cf757812428b0763112efb33b6f4fad7deb445e", + }, + { + "0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d", + "0x6cd850eca0a7ac46bb1748d7b9cb88aa3bd21c57d852c28198ad8fa422c4595032e88a4494b4778b36b944fe47a52b8c5cd312910139dfcb4147ab8e972cc456bcb063f25dd78f54c4d34679e03142c42c662af52947d45bdb6e555751334ace76a5080ab5a0256a1d259855dfc5c0b8023b25befbb13fd3684f9f755cbd3d63544c78ee2001452dd54633a7593ade0b183891a0a4e9c7844e1254005fbe592b1b89149a502c24b6e1dca44c158aebedf01beae9c30cabe16a", + "0x14abd5c47c0be87b0454596baad2", + "0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d", + }, + } + for i, tc := range tcs[1:] { + exp := types.DeriveSha(flatList(tc), new(trie.Trie)) + got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) + if !bytes.Equal(got[:], exp[:]) { + t.Fatalf("case %d: got %x exp %x", i, got, exp) + } + } +} + +func genTxs(num uint64) (types.Transactions, error) { + key, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + if err != nil { + return nil, err + } + var addr = crypto.PubkeyToAddress(key.PublicKey) + newTx := func(i uint64) (*types.Transaction, error) { + signer := types.NewEIP155Signer(big.NewInt(18)) + utx := types.NewTransaction(i, addr, new(big.Int), 0, new(big.Int).SetUint64(10000000), nil) + tx, err := types.SignTx(utx, signer, key) + return tx, err + } + var txs types.Transactions + for i := uint64(0); i < num; i++ { + tx, err := newTx(i) + if err != nil { + return nil, err + } + txs = append(txs, tx) + } + return txs, nil +} + +type dummyDerivableList struct { + len int + seed int +} + +func newDummy(seed int) *dummyDerivableList { + d := &dummyDerivableList{} + src := mrand.NewSource(int64(seed)) + // don't use lists longer than 4K items + d.len = int(src.Int63() & 0x0FFF) + d.seed = seed + return d +} + +func (d *dummyDerivableList) Len() int { + return d.len +} + +func (d *dummyDerivableList) EncodeIndex(i int, w *bytes.Buffer) { + src := mrand.NewSource(int64(d.seed + i)) + // max item size 256, at least 1 byte per item + size := 1 + src.Int63()&0x00FF + io.CopyN(w, mrand.New(src), size) +} + +func printList(l types.DerivableList) { + fmt.Printf("list length: %d\n", l.Len()) + fmt.Printf("{\n") + for i := 0; i < l.Len(); i++ { + var buf bytes.Buffer + l.EncodeIndex(i, &buf) + fmt.Printf("\"0x%x\",\n", buf.Bytes()) + } + fmt.Printf("},\n") +} + +type flatList []string + +func (f flatList) Len() int { + return len(f) +} +func (f flatList) EncodeIndex(i int, w *bytes.Buffer) { + w.Write(hexutil.MustDecode(f[i])) +} + +type hashToHumanReadable struct { + data []byte +} + +func (d *hashToHumanReadable) Reset() { + d.data = make([]byte, 0) +} + +func (d *hashToHumanReadable) Update(i []byte, i2 []byte) { + l := fmt.Sprintf("%x %x\n", i, i2) + d.data = append(d.data, []byte(l)...) +} + +func (d *hashToHumanReadable) Hash() common.Hash { + return common.Hash{} +} diff --git a/core/types/legacy_tx.go b/core/types/legacy_tx.go new file mode 100644 index 0000000000..41ad44f379 --- /dev/null +++ b/core/types/legacy_tx.go @@ -0,0 +1,111 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +// LegacyTx is the transaction data of regular Ethereum transactions. +type LegacyTx struct { + Nonce uint64 // nonce of sender account + GasPrice *big.Int // wei per gas + Gas uint64 // gas limit + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int // wei amount + Data []byte // contract invocation input data + V, R, S *big.Int // signature values +} + +// NewTransaction creates an unsigned legacy transaction. +// Deprecated: use NewTx instead. +func NewTransaction(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { + return NewTx(&LegacyTx{ + Nonce: nonce, + To: &to, + Value: amount, + Gas: gasLimit, + GasPrice: gasPrice, + Data: data, + }) +} + +// NewContractCreation creates an unsigned legacy transaction. +// Deprecated: use NewTx instead. +func NewContractCreation(nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { + return NewTx(&LegacyTx{ + Nonce: nonce, + Value: amount, + Gas: gasLimit, + GasPrice: gasPrice, + Data: data, + }) +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *LegacyTx) copy() TxData { + cpy := &LegacyTx{ + Nonce: tx.Nonce, + To: tx.To, // TODO: copy pointed-to address + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are initialized below. + Value: new(big.Int), + GasPrice: new(big.Int), + V: new(big.Int), + R: new(big.Int), + S: new(big.Int), + } + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.GasPrice != nil { + cpy.GasPrice.Set(tx.GasPrice) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. + +func (tx *LegacyTx) txType() byte { return LegacyTxType } +func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } +func (tx *LegacyTx) accessList() AccessList { return nil } +func (tx *LegacyTx) data() []byte { return tx.Data } +func (tx *LegacyTx) gas() uint64 { return tx.Gas } +func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) value() *big.Int { return tx.Value } +func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } +func (tx *LegacyTx) to() *common.Address { return tx.To } + +func (tx *LegacyTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V, tx.R, tx.S +} + +func (tx *LegacyTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.V, tx.R, tx.S = v, r, s +} diff --git a/core/types/receipt.go b/core/types/receipt.go index a96c7525ef..48f4aef06a 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -38,6 +38,8 @@ var ( receiptStatusSuccessfulRLP = []byte{0x01} ) +var errEmptyTypedReceipt = errors.New("empty typed receipt bytes") + const ( // ReceiptStatusFailed is the status code of a transaction if execution failed. ReceiptStatusFailed = uint64(0) @@ -49,6 +51,7 @@ const ( // Receipt represents the results of a transaction. type Receipt struct { // Consensus fields: These fields are defined by the Yellow Paper + Type uint8 `json:"type,omitempty"` PostState []byte `json:"root"` Status uint64 `json:"status"` CumulativeGasUsed uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -69,6 +72,7 @@ type Receipt struct { } type receiptMarshaling struct { + Type hexutil.Uint64 PostState hexutil.Bytes Status hexutil.Uint64 CumulativeGasUsed hexutil.Uint64 @@ -114,8 +118,13 @@ type v3StoredReceiptRLP struct { } // NewReceipt creates a barebone transaction receipt, copying the init fields. +// Deprecated: create receipts using a struct literal instead. func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { - r := &Receipt{PostState: common.CopyBytes(root), CumulativeGasUsed: cumulativeGasUsed} + r := &Receipt{ + Type: LegacyTxType, + PostState: common.CopyBytes(root), + CumulativeGasUsed: cumulativeGasUsed, + } if failed { r.Status = ReceiptStatusFailed } else { @@ -127,21 +136,65 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { // EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt // into an RLP stream. If no post state is present, byzantium fork is assumed. func (r *Receipt) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}) + data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs} + if r.Type == LegacyTxType { + return rlp.Encode(w, data) + } + // It's an EIP-2718 typed TX receipt. + if r.Type != AccessListTxType { + return ErrTxTypeNotSupported + } + buf := encodeBufferPool.Get().(*bytes.Buffer) + defer encodeBufferPool.Put(buf) + buf.Reset() + buf.WriteByte(r.Type) + if err := rlp.Encode(buf, data); err != nil { + return err + } + return rlp.Encode(w, buf.Bytes()) } // DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt // from an RLP stream. func (r *Receipt) DecodeRLP(s *rlp.Stream) error { - var dec receiptRLP - if err := s.Decode(&dec); err != nil { - return err - } - if err := r.setStatus(dec.PostStateOrStatus); err != nil { + kind, _, err := s.Kind() + switch { + case err != nil: return err + case kind == rlp.List: + // It's a legacy receipt. + var dec receiptRLP + if err := s.Decode(&dec); err != nil { + return err + } + r.Type = LegacyTxType + return r.setFromRLP(dec) + case kind == rlp.String: + // It's an EIP-2718 typed tx receipt. + b, err := s.Bytes() + if err != nil { + return err + } + if len(b) == 0 { + return errEmptyTypedReceipt + } + r.Type = b[0] + if r.Type == AccessListTxType { + var dec receiptRLP + if err := rlp.DecodeBytes(b[1:], &dec); err != nil { + return err + } + return r.setFromRLP(dec) + } + return ErrTxTypeNotSupported + default: + return rlp.ErrExpectedList } - r.CumulativeGasUsed, r.Bloom, r.Logs = dec.CumulativeGasUsed, dec.Bloom, dec.Logs - return nil +} + +func (r *Receipt) setFromRLP(data receiptRLP) error { + r.CumulativeGasUsed, r.Bloom, r.Logs = data.CumulativeGasUsed, data.Bloom, data.Logs + return r.setStatus(data.PostStateOrStatus) } func (r *Receipt) setStatus(postStateOrStatus []byte) error { @@ -172,7 +225,6 @@ func (r *Receipt) statusEncoding() []byte { // to approximate and limit the memory consumption of various caches. func (r *Receipt) Size() common.StorageSize { size := common.StorageSize(unsafe.Sizeof(*r)) + common.StorageSize(len(r.PostState)) - size += common.StorageSize(len(r.Logs)) * common.StorageSize(unsafe.Sizeof(Log{})) for _, log := range r.Logs { size += common.StorageSize(len(log.Topics)*common.HashLength + len(log.Data)) @@ -277,19 +329,27 @@ func decodeV3StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { return nil } -// Receipts is a wrapper around a Receipt array to implement DerivableList. +// Receipts implements DerivableList for receipts. type Receipts []*Receipt // Len returns the number of receipts in this list. -func (r Receipts) Len() int { return len(r) } - -// GetRlp returns the RLP encoding of one receipt from the list. -func (r Receipts) GetRlp(i int) []byte { - bytes, err := rlp.EncodeToBytes(r[i]) - if err != nil { - panic(err) +func (rs Receipts) Len() int { return len(rs) } + +// EncodeIndex encodes the i'th receipt to w. +func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) { + r := rs[i] + data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs} + switch r.Type { + case LegacyTxType: + rlp.Encode(w, data) + case AccessListTxType: + w.WriteByte(AccessListTxType) + rlp.Encode(w, data) + default: + // For unsupported types, write nothing. Since this is for + // DeriveSha, the error will be caught matching the derived hash + // to the block. } - return bytes } // DeriveFields fills the receipts with their computed fields based on consensus @@ -302,7 +362,8 @@ func (r Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, num return errors.New("transaction and receipt count mismatch") } for i := 0; i < len(r); i++ { - // The transaction hash can be retrieved from the transaction itself + // The transaction type and hash can be retrieved from the transaction itself + r[i].Type = txs[i].Type() r[i].TxHash = txs[i].Hash() // block location fields diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 806b3dd2ab..22a316c237 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -29,6 +29,15 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +func TestDecodeEmptyTypedReceipt(t *testing.T) { + input := []byte{0x80} + var r Receipt + err := rlp.DecodeBytes(input, &r) + if err != errEmptyTypedReceipt { + t.Fatal("wrong error:", err) + } +} + func TestLegacyReceiptDecoding(t *testing.T) { tests := []struct { name string @@ -154,9 +163,29 @@ func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) { // Tests that receipt data can be correctly derived from the contextual infos func TestDeriveFields(t *testing.T) { // Create a few transactions to have receipts for + to2 := common.HexToAddress("0x2") + to3 := common.HexToAddress("0x3") txs := Transactions{ - NewContractCreation(1, big.NewInt(1), 1, big.NewInt(1), nil), - NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil), + NewTx(&LegacyTx{ + Nonce: 1, + Value: big.NewInt(1), + Gas: 1, + GasPrice: big.NewInt(1), + }), + NewTx(&LegacyTx{ + To: &to2, + Nonce: 2, + Value: big.NewInt(2), + Gas: 2, + GasPrice: big.NewInt(2), + }), + NewTx(&AccessListTx{ + To: &to3, + Nonce: 3, + Value: big.NewInt(3), + Gas: 3, + GasPrice: big.NewInt(3), + }), } // Create the corresponding receipts receipts := Receipts{ @@ -182,6 +211,18 @@ func TestDeriveFields(t *testing.T) { ContractAddress: common.BytesToAddress([]byte{0x02, 0x22, 0x22}), GasUsed: 2, }, + &Receipt{ + Type: AccessListTxType, + PostState: common.Hash{3}.Bytes(), + CumulativeGasUsed: 6, + Logs: []*Log{ + {Address: common.BytesToAddress([]byte{0x33})}, + {Address: common.BytesToAddress([]byte{0x03, 0x33})}, + }, + TxHash: txs[2].Hash(), + ContractAddress: common.BytesToAddress([]byte{0x03, 0x33, 0x33}), + GasUsed: 3, + }, } // Clear all the computed fields and re-derive them number := big.NewInt(1) @@ -196,6 +237,9 @@ func TestDeriveFields(t *testing.T) { logIndex := uint(0) for i := range receipts { + if receipts[i].Type != txs[i].Type() { + t.Errorf("receipts[%d].Type = %d, want %d", i, receipts[i].Type, txs[i].Type()) + } if receipts[i].TxHash != txs[i].Hash() { t.Errorf("receipts[%d].TxHash = %s, want %s", i, receipts[i].TxHash.String(), txs[i].Hash().String()) } @@ -243,6 +287,34 @@ func TestDeriveFields(t *testing.T) { } } +// TestTypedReceiptEncodingDecoding reproduces a flaw that existed in the receipt +// rlp decoder, which failed due to a shadowing error. +func TestTypedReceiptEncodingDecoding(t *testing.T) { + var payload = common.FromHex("f9043eb9010c01f90108018262d4b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0b9010c01f901080182cd14b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0b9010d01f901090183013754b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0b9010d01f90109018301a194b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0") + check := func(bundle []*Receipt) { + t.Helper() + for i, receipt := range bundle { + if got, want := receipt.Type, uint8(1); got != want { + t.Fatalf("bundle %d: got %x, want %x", i, got, want) + } + } + } + { + var bundle []*Receipt + rlp.DecodeBytes(payload, &bundle) + check(bundle) + } + { + var bundle []*Receipt + r := bytes.NewReader(payload) + s := rlp.NewStream(r, uint64(len(payload))) + if err := s.Decode(&bundle); err != nil { + t.Fatal(err) + } + check(bundle) + } +} + func clearComputedFieldsOnReceipts(t *testing.T, receipts Receipts) { t.Helper() diff --git a/core/types/transaction.go b/core/types/transaction.go index 177bfbb70b..49127630ae 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -17,6 +17,7 @@ package types import ( + "bytes" "container/heap" "errors" "io" @@ -25,20 +26,28 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) -//go:generate gencodec -type txdata -field-override txdataMarshaling -out gen_tx_json.go - var ( - ErrInvalidSig = errors.New("invalid transaction v, r, s values") + ErrInvalidSig = errors.New("invalid transaction v, r, s values") + ErrUnexpectedProtection = errors.New("transaction type does not supported EIP-155 protected signatures") + ErrInvalidTxType = errors.New("transaction type not valid in this context") + ErrTxTypeNotSupported = errors.New("transaction type not supported") + errEmptyTypedTx = errors.New("empty typed transaction bytes") +) + +// Transaction types. +const ( + LegacyTxType = iota + AccessListTxType ) +// Transaction is an Ethereum transaction. type Transaction struct { - data txdata // Consensus contents of a transaction - time time.Time // Time first seen locally (spam avoidance) + inner TxData // Consensus contents of a transaction + time time.Time // Time first seen locally (spam avoidance) // caches hash atomic.Value @@ -46,170 +55,266 @@ type Transaction struct { from atomic.Value } -type txdata struct { - AccountNonce uint64 `json:"nonce" gencodec:"required"` - Price *big.Int `json:"gasPrice" gencodec:"required"` - GasLimit uint64 `json:"gas" gencodec:"required"` - Recipient *common.Address `json:"to" rlp:"nil"` // nil means contract creation - Amount *big.Int `json:"value" gencodec:"required"` - Payload []byte `json:"input" gencodec:"required"` +// NewTx creates a new transaction. +func NewTx(inner TxData) *Transaction { + tx := new(Transaction) + tx.setDecoded(inner.copy(), 0) + return tx +} + +// TxData is the underlying data of a transaction. +// +// This is implemented by LegacyTx and AccessListTx. +type TxData interface { + txType() byte // returns the type ID + copy() TxData // creates a deep copy and initializes all fields - // Signature values - V *big.Int `json:"v" gencodec:"required"` - R *big.Int `json:"r" gencodec:"required"` - S *big.Int `json:"s" gencodec:"required"` + chainID() *big.Int + accessList() AccessList + data() []byte + gas() uint64 + gasPrice() *big.Int + value() *big.Int + nonce() uint64 + to() *common.Address - // This is only used when marshaling to JSON. - Hash *common.Hash `json:"hash" rlp:"-"` + rawSignatureValues() (v, r, s *big.Int) + setSignatureValues(chainID, v, r, s *big.Int) } -type txdataMarshaling struct { - AccountNonce hexutil.Uint64 - Price *hexutil.Big - GasLimit hexutil.Uint64 - Amount *hexutil.Big - Payload hexutil.Bytes - V *hexutil.Big - R *hexutil.Big - S *hexutil.Big +// EncodeRLP implements rlp.Encoder +func (tx *Transaction) EncodeRLP(w io.Writer) error { + if tx.Type() == LegacyTxType { + return rlp.Encode(w, tx.inner) + } + // It's an EIP-2718 typed TX envelope. + buf := encodeBufferPool.Get().(*bytes.Buffer) + defer encodeBufferPool.Put(buf) + buf.Reset() + if err := tx.encodeTyped(buf); err != nil { + return err + } + return rlp.Encode(w, buf.Bytes()) } -func NewTransaction(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { - return newTransaction(nonce, &to, amount, gasLimit, gasPrice, data) +// encodeTyped writes the canonical encoding of a typed transaction to w. +func (tx *Transaction) encodeTyped(w *bytes.Buffer) error { + w.WriteByte(tx.Type()) + return rlp.Encode(w, tx.inner) } -func NewContractCreation(nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { - return newTransaction(nonce, nil, amount, gasLimit, gasPrice, data) +// MarshalBinary returns the canonical encoding of the transaction. +// For legacy transactions, it returns the RLP encoding. For EIP-2718 typed +// transactions, it returns the type and payload. +func (tx *Transaction) MarshalBinary() ([]byte, error) { + if tx.Type() == LegacyTxType { + return rlp.EncodeToBytes(tx.inner) + } + var buf bytes.Buffer + err := tx.encodeTyped(&buf) + return buf.Bytes(), err } -func newTransaction(nonce uint64, to *common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { - if len(data) > 0 { - data = common.CopyBytes(data) +// DecodeRLP implements rlp.Decoder +func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { + kind, size, err := s.Kind() + switch { + case err != nil: + return err + case kind == rlp.List: + // It's a legacy transaction. + var inner LegacyTx + err := s.Decode(&inner) + if err == nil { + tx.setDecoded(&inner, int(rlp.ListSize(size))) + } + return err + case kind == rlp.String: + // It's an EIP-2718 typed TX envelope. + var b []byte + if b, err = s.Bytes(); err != nil { + return err + } + inner, err := tx.decodeTyped(b) + if err == nil { + tx.setDecoded(inner, len(b)) + } + return err + default: + return rlp.ErrExpectedList } - d := txdata{ - AccountNonce: nonce, - Recipient: to, - Payload: data, - Amount: new(big.Int), - GasLimit: gasLimit, - Price: new(big.Int), - V: new(big.Int), - R: new(big.Int), - S: new(big.Int), +} + +// UnmarshalBinary decodes the canonical encoding of transactions. +// It supports legacy RLP transactions and EIP2718 typed transactions. +func (tx *Transaction) UnmarshalBinary(b []byte) error { + if len(b) > 0 && b[0] > 0x7f { + // It's a legacy transaction. + var data LegacyTx + err := rlp.DecodeBytes(b, &data) + if err != nil { + return err + } + tx.setDecoded(&data, len(b)) + return nil } - if amount != nil { - d.Amount.Set(amount) + // It's an EIP2718 typed transaction envelope. + inner, err := tx.decodeTyped(b) + if err != nil { + return err } - if gasPrice != nil { - d.Price.Set(gasPrice) + tx.setDecoded(inner, len(b)) + return nil +} + +// decodeTyped decodes a typed transaction from the canonical format. +func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { + if len(b) == 0 { + return nil, errEmptyTypedTx } - return &Transaction{ - data: d, - time: time.Now(), + switch b[0] { + case AccessListTxType: + var inner AccessListTx + err := rlp.DecodeBytes(b[1:], &inner) + return &inner, err + default: + return nil, ErrTxTypeNotSupported } } -// ChainId returns which chain id this transaction was signed for (if at all) -func (tx *Transaction) ChainId() *big.Int { - return deriveChainId(tx.data.V) +// setDecoded sets the inner transaction and size after decoding. +func (tx *Transaction) setDecoded(inner TxData, size int) { + tx.inner = inner + tx.time = time.Now() + if size > 0 { + tx.size.Store(common.StorageSize(size)) + } } -// Protected returns whether the transaction is protected from replay protection. -func (tx *Transaction) Protected() bool { - return isProtectedV(tx.data.V) +func sanityCheckSignature(v *big.Int, r *big.Int, s *big.Int, maybeProtected bool) error { + if isProtectedV(v) && !maybeProtected { + return ErrUnexpectedProtection + } + + var plainV byte + if isProtectedV(v) { + chainID := deriveChainId(v).Uint64() + plainV = byte(v.Uint64() - 35 - 2*chainID) + } else if maybeProtected { + // Only EIP-155 signatures can be optionally protected. Since + // we determined this v value is not protected, it must be a + // raw 27 or 28. + plainV = byte(v.Uint64() - 27) + } else { + // If the signature is not optionally protected, we assume it + // must already be equal to the recovery id. + plainV = byte(v.Uint64()) + } + if !crypto.ValidateSignatureValues(plainV, r, s, false) { + return ErrInvalidSig + } + + return nil } func isProtectedV(V *big.Int) bool { if V.BitLen() <= 8 { v := V.Uint64() - return v != 27 && v != 28 + return v != 27 && v != 28 && v != 1 && v != 0 } // anything not 27 or 28 is considered protected return true } -// EncodeRLP implements rlp.Encoder -func (tx *Transaction) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, &tx.data) -} - -// DecodeRLP implements rlp.Decoder -func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - err := s.Decode(&tx.data) - if err == nil { - tx.size.Store(common.StorageSize(rlp.ListSize(size))) - tx.time = time.Now() +// Protected says whether the transaction is replay-protected. +func (tx *Transaction) Protected() bool { + switch tx := tx.inner.(type) { + case *LegacyTx: + return tx.V != nil && isProtectedV(tx.V) + default: + return true } - return err } -// MarshalJSON encodes the web3 RPC transaction format. -func (tx *Transaction) MarshalJSON() ([]byte, error) { - hash := tx.Hash() - data := tx.data - data.Hash = &hash - return data.MarshalJSON() +// Type returns the transaction type. +func (tx *Transaction) Type() uint8 { + return tx.inner.txType() } -// UnmarshalJSON decodes the web3 RPC transaction format. -func (tx *Transaction) UnmarshalJSON(input []byte) error { - var dec txdata - if err := dec.UnmarshalJSON(input); err != nil { - return err - } - withSignature := dec.V.Sign() != 0 || dec.R.Sign() != 0 || dec.S.Sign() != 0 - if withSignature { - var V byte - if isProtectedV(dec.V) { - chainID := deriveChainId(dec.V).Uint64() - V = byte(dec.V.Uint64() - 35 - 2*chainID) - } else { - V = byte(dec.V.Uint64() - 27) - } - if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) { - return ErrInvalidSig - } - } - *tx = Transaction{ - data: dec, - time: time.Now(), - } - return nil +// ChainId returns the EIP155 chain ID of the transaction. The return value will always be +// non-nil. For legacy transactions which are not replay-protected, the return value is +// zero. +func (tx *Transaction) ChainId() *big.Int { + return tx.inner.chainID() } -func (tx *Transaction) Data() []byte { return common.CopyBytes(tx.data.Payload) } -func (tx *Transaction) Gas() uint64 { return tx.data.GasLimit } -func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.data.Price) } -func (tx *Transaction) GasPriceCmp(other *Transaction) int { - return tx.data.Price.Cmp(other.data.Price) -} -func (tx *Transaction) GasPriceIntCmp(other *big.Int) int { - return tx.data.Price.Cmp(other) -} -func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.data.Amount) } -func (tx *Transaction) Nonce() uint64 { return tx.data.AccountNonce } -func (tx *Transaction) CheckNonce() bool { return true } +// Data returns the input data of the transaction. +func (tx *Transaction) Data() []byte { return tx.inner.data() } + +// AccessList returns the access list of the transaction. +func (tx *Transaction) AccessList() AccessList { return tx.inner.accessList() } + +// Gas returns the gas limit of the transaction. +func (tx *Transaction) Gas() uint64 { return tx.inner.gas() } + +// GasPrice returns the gas price of the transaction. +func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.inner.gasPrice()) } + +// Value returns the ether amount of the transaction. +func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) } + +// Nonce returns the sender account nonce of the transaction. +func (tx *Transaction) Nonce() uint64 { return tx.inner.nonce() } // To returns the recipient address of the transaction. -// It returns nil if the transaction is a contract creation. +// For contract-creation transactions, To returns nil. func (tx *Transaction) To() *common.Address { - if tx.data.Recipient == nil { + // Copy the pointed-to address. + ito := tx.inner.to() + if ito == nil { return nil } - to := *tx.data.Recipient - return &to + cpy := *ito + return &cpy +} + +// Cost returns gas * gasPrice + value. +func (tx *Transaction) Cost() *big.Int { + total := new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas())) + total.Add(total, tx.Value()) + return total +} + +// RawSignatureValues returns the V, R, S signature values of the transaction. +// The return values should not be modified by the caller. +func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) { + return tx.inner.rawSignatureValues() +} + +// GasPriceCmp compares the gas prices of two transactions. +func (tx *Transaction) GasPriceCmp(other *Transaction) int { + return tx.inner.gasPrice().Cmp(other.GasPrice()) } -// Hash hashes the RLP encoding of tx. -// It uniquely identifies the transaction. +// GasPriceIntCmp compares the gas price of the transaction against the given price. +func (tx *Transaction) GasPriceIntCmp(other *big.Int) int { + return tx.inner.gasPrice().Cmp(other) +} + +// Hash returns the transaction hash. func (tx *Transaction) Hash() common.Hash { if hash := tx.hash.Load(); hash != nil { return hash.(common.Hash) } - v := rlpHash(tx) - tx.hash.Store(v) - return v + + var h common.Hash + if tx.Type() == LegacyTxType { + h = rlpHash(tx.inner) + } else { + h = prefixedRlpHash(tx.Type(), tx.inner) + } + tx.hash.Store(h) + return h } // Size returns the true RLP encoded storage size of the transaction, either by @@ -219,32 +324,11 @@ func (tx *Transaction) Size() common.StorageSize { return size.(common.StorageSize) } c := writeCounter(0) - rlp.Encode(&c, &tx.data) + rlp.Encode(&c, &tx.inner) tx.size.Store(common.StorageSize(c)) return common.StorageSize(c) } -// AsMessage returns the transaction as a core.Message. -// -// AsMessage requires a signer to derive the sender. -// -// XXX Rename message to something less arbitrary? -func (tx *Transaction) AsMessage(s Signer) (Message, error) { - msg := Message{ - nonce: tx.data.AccountNonce, - gasLimit: tx.data.GasLimit, - gasPrice: new(big.Int).Set(tx.data.Price), - to: tx.data.Recipient, - amount: tx.data.Amount, - data: tx.data.Payload, - checkNonce: true, - } - - var err error - msg.from, err = Sender(s, tx) - return msg, err -} - // WithSignature returns a new transaction with the given signature. // This signature needs to be in the [R || S || V] format where V is 0 or 1. func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, error) { @@ -252,40 +336,27 @@ func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, e if err != nil { return nil, err } - cpy := &Transaction{ - data: tx.data, - time: tx.time, - } - cpy.data.R, cpy.data.S, cpy.data.V = r, s, v - return cpy, nil + cpy := tx.inner.copy() + cpy.setSignatureValues(signer.ChainID(), v, r, s) + return &Transaction{inner: cpy, time: tx.time}, nil } -// Cost returns amount + gasprice * gaslimit. -func (tx *Transaction) Cost() *big.Int { - total := new(big.Int).Mul(tx.data.Price, new(big.Int).SetUint64(tx.data.GasLimit)) - total.Add(total, tx.data.Amount) - return total -} - -// RawSignatureValues returns the V, R, S signature values of the transaction. -// The return values should not be modified by the caller. -func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) { - return tx.data.V, tx.data.R, tx.data.S -} - -// Transactions is a Transaction slice type for basic sorting. +// Transactions implements DerivableList for transactions. type Transactions []*Transaction // Len returns the length of s. func (s Transactions) Len() int { return len(s) } -// Swap swaps the i'th and the j'th element in s. -func (s Transactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// GetRlp implements Rlpable and returns the i'th element of s in rlp. -func (s Transactions) GetRlp(i int) []byte { - enc, _ := rlp.EncodeToBytes(s[i]) - return enc +// EncodeIndex encodes the i'th transaction to w. Note that this does not check for errors +// because we assume that *Transaction will only ever contain valid txs that were either +// constructed by decoding or via public API in this package. +func (s Transactions) EncodeIndex(i int, w *bytes.Buffer) { + tx := s[i] + if tx.Type() == LegacyTxType { + rlp.Encode(w, tx.inner) + } else { + tx.encodeTyped(w) + } } // TxDifference returns a new set which is the difference between a and b. @@ -312,7 +383,7 @@ func TxDifference(a, b Transactions) Transactions { type TxByNonce Transactions func (s TxByNonce) Len() int { return len(s) } -func (s TxByNonce) Less(i, j int) bool { return s[i].data.AccountNonce < s[j].data.AccountNonce } +func (s TxByNonce) Less(i, j int) bool { return s[i].Nonce() < s[j].Nonce() } func (s TxByNonce) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // TxByPriceAndTime implements both the sort and the heap interface, making it useful @@ -323,7 +394,7 @@ func (s TxByPriceAndTime) Len() int { return len(s) } func (s TxByPriceAndTime) Less(i, j int) bool { // If the prices are equal, use the time the transaction was first seen for // deterministic sorting - cmp := s[i].data.Price.Cmp(s[j].data.Price) + cmp := s[i].GasPrice().Cmp(s[j].GasPrice()) if cmp == 0 { return s[i].time.Before(s[j].time) } @@ -361,13 +432,13 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa // Initialize a price and received time based heap with the head transactions heads := make(TxByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { - heads = append(heads, accTxs[0]) // Ensure the sender address is from the signer - acc, _ := Sender(signer, accTxs[0]) - txs[acc] = accTxs[1:] - if from != acc { + if acc, _ := Sender(signer, accTxs[0]); acc != from { delete(txs, from) + continue } + heads = append(heads, accTxs[0]) + txs[from] = accTxs[1:] } heap.Init(&heads) @@ -416,10 +487,11 @@ type Message struct { gasLimit uint64 gasPrice *big.Int data []byte + accessList AccessList checkNonce bool } -func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte, checkNonce bool) Message { +func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte, accessList AccessList, checkNonce bool) Message { return Message{ from: from, to: to, @@ -428,15 +500,35 @@ func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *b gasLimit: gasLimit, gasPrice: gasPrice, data: data, + accessList: accessList, checkNonce: checkNonce, } } -func (m Message) From() common.Address { return m.from } -func (m Message) To() *common.Address { return m.to } -func (m Message) GasPrice() *big.Int { return m.gasPrice } -func (m Message) Value() *big.Int { return m.amount } -func (m Message) Gas() uint64 { return m.gasLimit } -func (m Message) Nonce() uint64 { return m.nonce } -func (m Message) Data() []byte { return m.data } -func (m Message) CheckNonce() bool { return m.checkNonce } +// AsMessage returns the transaction as a core.Message. +func (tx *Transaction) AsMessage(s Signer) (Message, error) { + msg := Message{ + nonce: tx.Nonce(), + gasLimit: tx.Gas(), + gasPrice: new(big.Int).Set(tx.GasPrice()), + to: tx.To(), + amount: tx.Value(), + data: tx.Data(), + accessList: tx.AccessList(), + checkNonce: true, + } + + var err error + msg.from, err = Sender(s, tx) + return msg, err +} + +func (m Message) From() common.Address { return m.from } +func (m Message) To() *common.Address { return m.to } +func (m Message) GasPrice() *big.Int { return m.gasPrice } +func (m Message) Value() *big.Int { return m.amount } +func (m Message) Gas() uint64 { return m.gasLimit } +func (m Message) Nonce() uint64 { return m.nonce } +func (m Message) Data() []byte { return m.data } +func (m Message) AccessList() AccessList { return m.accessList } +func (m Message) CheckNonce() bool { return m.checkNonce } diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go new file mode 100644 index 0000000000..184a17d5b5 --- /dev/null +++ b/core/types/transaction_marshalling.go @@ -0,0 +1,187 @@ +package types + +import ( + "encoding/json" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +// txJSON is the JSON representation of transactions. +type txJSON struct { + Type hexutil.Uint64 `json:"type"` + + // Common transaction fields: + Nonce *hexutil.Uint64 `json:"nonce"` + GasPrice *hexutil.Big `json:"gasPrice"` + Gas *hexutil.Uint64 `json:"gas"` + Value *hexutil.Big `json:"value"` + Data *hexutil.Bytes `json:"input"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + To *common.Address `json:"to"` + + // Access list transaction fields: + ChainID *hexutil.Big `json:"chainId,omitempty"` + AccessList *AccessList `json:"accessList,omitempty"` + + // Only used for encoding: + Hash common.Hash `json:"hash"` +} + +// MarshalJSON marshals as JSON with a hash. +func (t *Transaction) MarshalJSON() ([]byte, error) { + var enc txJSON + // These are set for all tx types. + enc.Hash = t.Hash() + enc.Type = hexutil.Uint64(t.Type()) + + // Other fields are set conditionally depending on tx type. + switch tx := t.inner.(type) { + case *LegacyTx: + enc.Nonce = (*hexutil.Uint64)(&tx.Nonce) + enc.Gas = (*hexutil.Uint64)(&tx.Gas) + enc.GasPrice = (*hexutil.Big)(tx.GasPrice) + enc.Value = (*hexutil.Big)(tx.Value) + enc.Data = (*hexutil.Bytes)(&tx.Data) + enc.To = t.To() + enc.V = (*hexutil.Big)(tx.V) + enc.R = (*hexutil.Big)(tx.R) + enc.S = (*hexutil.Big)(tx.S) + case *AccessListTx: + enc.ChainID = (*hexutil.Big)(tx.ChainID) + enc.AccessList = &tx.AccessList + enc.Nonce = (*hexutil.Uint64)(&tx.Nonce) + enc.Gas = (*hexutil.Uint64)(&tx.Gas) + enc.GasPrice = (*hexutil.Big)(tx.GasPrice) + enc.Value = (*hexutil.Big)(tx.Value) + enc.Data = (*hexutil.Bytes)(&tx.Data) + enc.To = t.To() + enc.V = (*hexutil.Big)(tx.V) + enc.R = (*hexutil.Big)(tx.R) + enc.S = (*hexutil.Big)(tx.S) + } + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (t *Transaction) UnmarshalJSON(input []byte) error { + var dec txJSON + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + + // Decode / verify fields according to transaction type. + var inner TxData + switch dec.Type { + case LegacyTxType: + var itx LegacyTx + inner = &itx + if dec.To != nil { + itx.To = dec.To + } + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.GasPrice == nil { + return errors.New("missing required field 'gasPrice' in transaction") + } + itx.GasPrice = (*big.Int)(dec.GasPrice) + if dec.Gas == nil { + return errors.New("missing required field 'gas' in transaction") + } + itx.Gas = uint64(*dec.Gas) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = (*big.Int)(dec.Value) + if dec.Data == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Data + if dec.V == nil { + return errors.New("missing required field 'v' in transaction") + } + itx.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S = (*big.Int)(dec.S) + withSignature := itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 + if withSignature { + if err := sanityCheckSignature(itx.V, itx.R, itx.S, true); err != nil { + return err + } + } + + case AccessListTxType: + var itx AccessListTx + inner = &itx + // Access list is optional for now. + if dec.AccessList != nil { + itx.AccessList = *dec.AccessList + } + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' in transaction") + } + itx.ChainID = (*big.Int)(dec.ChainID) + if dec.To != nil { + itx.To = dec.To + } + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.GasPrice == nil { + return errors.New("missing required field 'gasPrice' in transaction") + } + itx.GasPrice = (*big.Int)(dec.GasPrice) + if dec.Gas == nil { + return errors.New("missing required field 'gas' in transaction") + } + itx.Gas = uint64(*dec.Gas) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = (*big.Int)(dec.Value) + if dec.Data == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Data + if dec.V == nil { + return errors.New("missing required field 'v' in transaction") + } + itx.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S = (*big.Int)(dec.S) + withSignature := itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 + if withSignature { + if err := sanityCheckSignature(itx.V, itx.R, itx.S, false); err != nil { + return err + } + } + + default: + return ErrTxTypeNotSupported + } + + // Now set the inner transaction. + t.setDecoded(inner, 0) + + // TODO: check hash here? + return nil +} diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 842fedbd03..1645369b4f 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -27,9 +27,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -var ( - ErrInvalidChainId = errors.New("invalid chain id for signer") -) +var ErrInvalidChainId = errors.New("invalid chain id for signer") // sigCache is used to cache the derived sender and contains // the signer used to derive it. @@ -42,6 +40,8 @@ type sigCache struct { func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { var signer Signer switch { + case config.IsYoloV3(blockNumber): + signer = NewEIP2930Signer(config.ChainID) case config.IsEIP155(blockNumber): signer = NewEIP155Signer(config.ChainID) case config.IsHomestead(blockNumber): @@ -52,7 +52,40 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { return signer } -// SignTx signs the transaction using the given signer and private key +// LatestSigner returns the 'most permissive' Signer available for the given chain +// configuration. Specifically, this enables support of EIP-155 replay protection and +// EIP-2930 access list transactions when their respective forks are scheduled to occur at +// any block number in the chain config. +// +// Use this in transaction-handling code where the current block number is unknown. If you +// have the current block number available, use MakeSigner instead. +func LatestSigner(config *params.ChainConfig) Signer { + if config.ChainID != nil { + if config.YoloV3Block != nil { + return NewEIP2930Signer(config.ChainID) + } + if config.EIP155Block != nil { + return NewEIP155Signer(config.ChainID) + } + } + return HomesteadSigner{} +} + +// LatestSignerForChainID returns the 'most permissive' Signer available. Specifically, +// this enables support for EIP-155 replay protection and all implemented EIP-2718 +// transaction types if chainID is non-nil. +// +// Use this in transaction-handling code where the current block number and fork +// configuration are unknown. If you have a ChainConfig, use LatestSigner instead. +// If you have a ChainConfig and know the current block number, use MakeSigner instead. +func LatestSignerForChainID(chainID *big.Int) Signer { + if chainID == nil { + return HomesteadSigner{} + } + return NewEIP2930Signer(chainID) +} + +// SignTx signs the transaction using the given signer and private key. func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, error) { h := s.Hash(tx) sig, err := crypto.Sign(h[:], prv) @@ -62,6 +95,27 @@ func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, err return tx.WithSignature(s, sig) } +// SignNewTx creates a transaction and signs it. +func SignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) (*Transaction, error) { + tx := NewTx(txdata) + h := s.Hash(tx) + sig, err := crypto.Sign(h[:], prv) + if err != nil { + return nil, err + } + return tx.WithSignature(s, sig) +} + +// MustSignNewTx creates a transaction and signs it. +// This panics if the transaction cannot be signed. +func MustSignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) *Transaction { + tx, err := SignNewTx(prv, s, txdata) + if err != nil { + panic(err) + } + return tx +} + // Sender returns the address derived from the signature (V, R, S) using secp256k1 // elliptic curve and an error if it failed deriving or upon an incorrect // signature. @@ -88,21 +142,128 @@ func Sender(signer Signer, tx *Transaction) (common.Address, error) { return addr, nil } -// Signer encapsulates transaction signature handling. Note that this interface is not a -// stable API and may change at any time to accommodate new protocol rules. +// Signer encapsulates transaction signature handling. The name of this type is slightly +// misleading because Signers don't actually sign, they're just for validating and +// processing of signatures. +// +// Note that this interface is not a stable API and may change at any time to accommodate +// new protocol rules. type Signer interface { // Sender returns the sender address of the transaction. Sender(tx *Transaction) (common.Address, error) + // SignatureValues returns the raw R, S, V values corresponding to the // given signature. SignatureValues(tx *Transaction, sig []byte) (r, s, v *big.Int, err error) - // Hash returns the hash to be signed. + ChainID() *big.Int + + // Hash returns 'signature hash', i.e. the transaction hash that is signed by the + // private key. This hash does not uniquely identify the transaction. Hash(tx *Transaction) common.Hash + // Equal returns true if the given signer is the same as the receiver. Equal(Signer) bool } -// EIP155Transaction implements Signer using the EIP155 rules. +type eip2930Signer struct{ EIP155Signer } + +// NewEIP2930Signer returns a signer that accepts EIP-2930 access list transactions, +// EIP-155 replay protected transactions, and legacy Homestead transactions. +func NewEIP2930Signer(chainId *big.Int) Signer { + return eip2930Signer{NewEIP155Signer(chainId)} +} + +func (s eip2930Signer) ChainID() *big.Int { + return s.chainId +} + +func (s eip2930Signer) Equal(s2 Signer) bool { + x, ok := s2.(eip2930Signer) + return ok && x.chainId.Cmp(s.chainId) == 0 +} + +func (s eip2930Signer) Sender(tx *Transaction) (common.Address, error) { + V, R, S := tx.RawSignatureValues() + switch tx.Type() { + case LegacyTxType: + if !tx.Protected() { + return HomesteadSigner{}.Sender(tx) + } + V = new(big.Int).Sub(V, s.chainIdMul) + V.Sub(V, big8) + case AccessListTxType: + // ACL txs are defined to use 0 and 1 as their recovery id, add + // 27 to become equivalent to unprotected Homestead signatures. + V = new(big.Int).Add(V, big.NewInt(27)) + default: + return common.Address{}, ErrTxTypeNotSupported + } + if tx.ChainId().Cmp(s.chainId) != 0 { + return common.Address{}, ErrInvalidChainId + } + return recoverPlain(s.Hash(tx), R, S, V, true) +} + +func (s eip2930Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { + switch txdata := tx.inner.(type) { + case *LegacyTx: + R, S, V = decodeSignature(sig) + if s.chainId.Sign() != 0 { + V = big.NewInt(int64(sig[64] + 35)) + V.Add(V, s.chainIdMul) + } + case *AccessListTx: + // Check that chain ID of tx matches the signer. We also accept ID zero here, + // because it indicates that the chain ID was not specified in the tx. + if txdata.ChainID.Sign() != 0 && txdata.ChainID.Cmp(s.chainId) != 0 { + return nil, nil, nil, ErrInvalidChainId + } + R, S, _ = decodeSignature(sig) + V = big.NewInt(int64(sig[64])) + default: + return nil, nil, nil, ErrTxTypeNotSupported + } + return R, S, V, nil +} + +// Hash returns the hash to be signed by the sender. +// It does not uniquely identify the transaction. +func (s eip2930Signer) Hash(tx *Transaction) common.Hash { + switch tx.Type() { + case LegacyTxType: + return rlpHash([]interface{}{ + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + s.chainId, uint(0), uint(0), + }) + case AccessListTxType: + return prefixedRlpHash( + tx.Type(), + []interface{}{ + s.chainId, + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + tx.AccessList(), + }) + default: + // This _should_ not happen, but in case someone sends in a bad + // json struct via RPC, it's probably more prudent to return an + // empty hash instead of killing the node with a panic + //panic("Unsupported transaction type: %d", tx.typ) + return common.Hash{} + } +} + +// EIP155Signer implements Signer using the EIP-155 rules. This accepts transactions which +// are replay-protected as well as unprotected homestead transactions. type EIP155Signer struct { chainId, chainIdMul *big.Int } @@ -117,6 +278,10 @@ func NewEIP155Signer(chainId *big.Int) EIP155Signer { } } +func (s EIP155Signer) ChainID() *big.Int { + return s.chainId +} + func (s EIP155Signer) Equal(s2 Signer) bool { eip155, ok := s2.(EIP155Signer) return ok && eip155.chainId.Cmp(s.chainId) == 0 @@ -125,24 +290,28 @@ func (s EIP155Signer) Equal(s2 Signer) bool { var big8 = big.NewInt(8) func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { + if tx.Type() != LegacyTxType { + return common.Address{}, ErrTxTypeNotSupported + } if !tx.Protected() { return HomesteadSigner{}.Sender(tx) } if tx.ChainId().Cmp(s.chainId) != 0 { return common.Address{}, ErrInvalidChainId } - V := new(big.Int).Sub(tx.data.V, s.chainIdMul) + V, R, S := tx.RawSignatureValues() + V = new(big.Int).Sub(V, s.chainIdMul) V.Sub(V, big8) - return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true) + return recoverPlain(s.Hash(tx), R, S, V, true) } // SignatureValues returns signature values. This signature // needs to be in the [R || S || V] format where V is 0 or 1. func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { - R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig) - if err != nil { - return nil, nil, nil, err + if tx.Type() != LegacyTxType { + return nil, nil, nil, ErrTxTypeNotSupported } + R, S, V = decodeSignature(sig) if s.chainId.Sign() != 0 { V = big.NewInt(int64(sig[64] + 35)) V.Add(V, s.chainIdMul) @@ -154,12 +323,12 @@ func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big // It does not uniquely identify the transaction. func (s EIP155Signer) Hash(tx *Transaction) common.Hash { return rlpHash([]interface{}{ - tx.data.AccountNonce, - tx.data.Price, - tx.data.GasLimit, - tx.data.Recipient, - tx.data.Amount, - tx.data.Payload, + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), s.chainId, uint(0), uint(0), }) } @@ -168,6 +337,10 @@ func (s EIP155Signer) Hash(tx *Transaction) common.Hash { // homestead rules. type HomesteadSigner struct{ FrontierSigner } +func (s HomesteadSigner) ChainID() *big.Int { + return nil +} + func (s HomesteadSigner) Equal(s2 Signer) bool { _, ok := s2.(HomesteadSigner) return ok @@ -180,25 +353,39 @@ func (hs HomesteadSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v } func (hs HomesteadSigner) Sender(tx *Transaction) (common.Address, error) { - return recoverPlain(hs.Hash(tx), tx.data.R, tx.data.S, tx.data.V, true) + if tx.Type() != LegacyTxType { + return common.Address{}, ErrTxTypeNotSupported + } + v, r, s := tx.RawSignatureValues() + return recoverPlain(hs.Hash(tx), r, s, v, true) } type FrontierSigner struct{} +func (s FrontierSigner) ChainID() *big.Int { + return nil +} + func (s FrontierSigner) Equal(s2 Signer) bool { _, ok := s2.(FrontierSigner) return ok } +func (fs FrontierSigner) Sender(tx *Transaction) (common.Address, error) { + if tx.Type() != LegacyTxType { + return common.Address{}, ErrTxTypeNotSupported + } + v, r, s := tx.RawSignatureValues() + return recoverPlain(fs.Hash(tx), r, s, v, false) +} + // SignatureValues returns signature values. This signature // needs to be in the [R || S || V] format where V is 0 or 1. func (fs FrontierSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v *big.Int, err error) { - if len(sig) != crypto.SignatureLength { - panic(fmt.Sprintf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength)) + if tx.Type() != LegacyTxType { + return nil, nil, nil, ErrTxTypeNotSupported } - r = new(big.Int).SetBytes(sig[:32]) - s = new(big.Int).SetBytes(sig[32:64]) - v = new(big.Int).SetBytes([]byte{sig[64] + 27}) + r, s, v = decodeSignature(sig) return r, s, v, nil } @@ -206,17 +393,23 @@ func (fs FrontierSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v * // It does not uniquely identify the transaction. func (fs FrontierSigner) Hash(tx *Transaction) common.Hash { return rlpHash([]interface{}{ - tx.data.AccountNonce, - tx.data.Price, - tx.data.GasLimit, - tx.data.Recipient, - tx.data.Amount, - tx.data.Payload, + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), }) } -func (fs FrontierSigner) Sender(tx *Transaction) (common.Address, error) { - return recoverPlain(fs.Hash(tx), tx.data.R, tx.data.S, tx.data.V, false) +func decodeSignature(sig []byte) (r, s, v *big.Int) { + if len(sig) != crypto.SignatureLength { + panic(fmt.Sprintf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength)) + } + r = new(big.Int).SetBytes(sig[:32]) + s = new(big.Int).SetBytes(sig[32:64]) + v = new(big.Int).SetBytes([]byte{sig[64] + 27}) + return r, s, v } func recoverPlain(sighash common.Hash, R, S, Vb *big.Int, homestead bool) (common.Address, error) { diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 159cb0c4c4..3cece9c235 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -20,7 +20,9 @@ import ( "bytes" "crypto/ecdsa" "encoding/json" + "fmt" "math/big" + "reflect" "testing" "time" @@ -32,6 +34,8 @@ import ( // The values in those tests are from the Transaction Tests // at github.com/ethereum/tests. var ( + testAddr = common.HexToAddress("b94f5374fce5edbc8e2a8697c15331677e6ebf0b") + emptyTx = NewTransaction( 0, common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87"), @@ -41,7 +45,7 @@ var ( rightvrsTx, _ = NewTransaction( 3, - common.HexToAddress("b94f5374fce5edbc8e2a8697c15331677e6ebf0b"), + testAddr, big.NewInt(10), 2000, big.NewInt(1), @@ -50,8 +54,32 @@ var ( HomesteadSigner{}, common.Hex2Bytes("98ff921201554726367d2be8c804a7ff89ccf285ebc57dff8ae4c44b9c19ac4a8887321be575c8095f789dd4c743dfe42c1820f9231f98a962b210e3ac2452a301"), ) + + emptyEip2718Tx = NewTx(&AccessListTx{ + ChainID: big.NewInt(1), + Nonce: 3, + To: &testAddr, + Value: big.NewInt(10), + Gas: 25000, + GasPrice: big.NewInt(1), + Data: common.FromHex("5544"), + }) + + signedEip2718Tx, _ = emptyEip2718Tx.WithSignature( + NewEIP2930Signer(big.NewInt(1)), + common.Hex2Bytes("c9519f4f2b30335884581971573fadf60c6204f59a911df35ee8a540456b266032f1e8e2c5dd761f9e4f88f41c8310aeaba26a8bfcdacfedfa12ec3862d3752101"), + ) ) +func TestDecodeEmptyTypedTx(t *testing.T) { + input := []byte{0x80} + var tx Transaction + err := rlp.DecodeBytes(input, &tx) + if err != errEmptyTypedTx { + t.Fatal("wrong error:", err) + } +} + func TestTransactionSigHash(t *testing.T) { var homestead HomesteadSigner if homestead.Hash(emptyTx) != common.HexToHash("c775b99e7ad12f50d819fcd602390467e28141316969f4b57f0626f74fe3b386") { @@ -73,10 +101,121 @@ func TestTransactionEncode(t *testing.T) { } } +func TestEIP2718TransactionSigHash(t *testing.T) { + s := NewEIP2930Signer(big.NewInt(1)) + if s.Hash(emptyEip2718Tx) != common.HexToHash("49b486f0ec0a60dfbbca2d30cb07c9e8ffb2a2ff41f29a1ab6737475f6ff69f3") { + t.Errorf("empty EIP-2718 transaction hash mismatch, got %x", s.Hash(emptyEip2718Tx)) + } + if s.Hash(signedEip2718Tx) != common.HexToHash("49b486f0ec0a60dfbbca2d30cb07c9e8ffb2a2ff41f29a1ab6737475f6ff69f3") { + t.Errorf("signed EIP-2718 transaction hash mismatch, got %x", s.Hash(signedEip2718Tx)) + } +} + +// This test checks signature operations on access list transactions. +func TestEIP2930Signer(t *testing.T) { + + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + keyAddr = crypto.PubkeyToAddress(key.PublicKey) + signer1 = NewEIP2930Signer(big.NewInt(1)) + signer2 = NewEIP2930Signer(big.NewInt(2)) + tx0 = NewTx(&AccessListTx{Nonce: 1}) + tx1 = NewTx(&AccessListTx{ChainID: big.NewInt(1), Nonce: 1}) + tx2, _ = SignNewTx(key, signer2, &AccessListTx{ChainID: big.NewInt(2), Nonce: 1}) + ) + + tests := []struct { + tx *Transaction + signer Signer + wantSignerHash common.Hash + wantSenderErr error + wantSignErr error + wantHash common.Hash // after signing + }{ + { + tx: tx0, + signer: signer1, + wantSignerHash: common.HexToHash("846ad7672f2a3a40c1f959cd4a8ad21786d620077084d84c8d7c077714caa139"), + wantSenderErr: ErrInvalidChainId, + wantHash: common.HexToHash("1ccd12d8bbdb96ea391af49a35ab641e219b2dd638dea375f2bc94dd290f2549"), + }, + { + tx: tx1, + signer: signer1, + wantSenderErr: ErrInvalidSig, + wantSignerHash: common.HexToHash("846ad7672f2a3a40c1f959cd4a8ad21786d620077084d84c8d7c077714caa139"), + wantHash: common.HexToHash("1ccd12d8bbdb96ea391af49a35ab641e219b2dd638dea375f2bc94dd290f2549"), + }, + { + // This checks what happens when trying to sign an unsigned tx for the wrong chain. + tx: tx1, + signer: signer2, + wantSenderErr: ErrInvalidChainId, + wantSignerHash: common.HexToHash("367967247499343401261d718ed5aa4c9486583e4d89251afce47f4a33c33362"), + wantSignErr: ErrInvalidChainId, + }, + { + // This checks what happens when trying to re-sign a signed tx for the wrong chain. + tx: tx2, + signer: signer1, + wantSenderErr: ErrInvalidChainId, + wantSignerHash: common.HexToHash("846ad7672f2a3a40c1f959cd4a8ad21786d620077084d84c8d7c077714caa139"), + wantSignErr: ErrInvalidChainId, + }, + } + + for i, test := range tests { + sigHash := test.signer.Hash(test.tx) + if sigHash != test.wantSignerHash { + t.Errorf("test %d: wrong sig hash: got %x, want %x", i, sigHash, test.wantSignerHash) + } + sender, err := Sender(test.signer, test.tx) + if err != test.wantSenderErr { + t.Errorf("test %d: wrong Sender error %q", i, err) + } + if err == nil && sender != keyAddr { + t.Errorf("test %d: wrong sender address %x", i, sender) + } + signedTx, err := SignTx(test.tx, test.signer, key) + if err != test.wantSignErr { + t.Fatalf("test %d: wrong SignTx error %q", i, err) + } + if signedTx != nil { + if signedTx.Hash() != test.wantHash { + t.Errorf("test %d: wrong tx hash after signing: got %x, want %x", i, signedTx.Hash(), test.wantHash) + } + } + } +} + +func TestEIP2718TransactionEncode(t *testing.T) { + // RLP representation + { + have, err := rlp.EncodeToBytes(signedEip2718Tx) + if err != nil { + t.Fatalf("encode error: %v", err) + } + want := common.FromHex("b86601f8630103018261a894b94f5374fce5edbc8e2a8697c15331677e6ebf0b0a825544c001a0c9519f4f2b30335884581971573fadf60c6204f59a911df35ee8a540456b2660a032f1e8e2c5dd761f9e4f88f41c8310aeaba26a8bfcdacfedfa12ec3862d37521") + if !bytes.Equal(have, want) { + t.Errorf("encoded RLP mismatch, got %x", have) + } + } + // Binary representation + { + have, err := signedEip2718Tx.MarshalBinary() + if err != nil { + t.Fatalf("encode error: %v", err) + } + want := common.FromHex("01f8630103018261a894b94f5374fce5edbc8e2a8697c15331677e6ebf0b0a825544c001a0c9519f4f2b30335884581971573fadf60c6204f59a911df35ee8a540456b2660a032f1e8e2c5dd761f9e4f88f41c8310aeaba26a8bfcdacfedfa12ec3862d37521") + if !bytes.Equal(have, want) { + t.Errorf("encoded RLP mismatch, got %x", have) + } + } +} + func decodeTx(data []byte) (*Transaction, error) { var tx Transaction t, err := &tx, rlp.Decode(bytes.NewReader(data), &tx) - return t, err } @@ -219,50 +358,125 @@ func TestTransactionTimeSort(t *testing.T) { } } -// TestTransactionJSON tests serializing/de-serializing to/from JSON. -func TestTransactionJSON(t *testing.T) { +// TestTransactionCoding tests serializing/de-serializing to/from rlp and JSON. +func TestTransactionCoding(t *testing.T) { key, err := crypto.GenerateKey() if err != nil { t.Fatalf("could not generate key: %v", err) } - signer := NewEIP155Signer(common.Big1) - - transactions := make([]*Transaction, 0, 50) - for i := uint64(0); i < 25; i++ { - var tx *Transaction - switch i % 2 { + var ( + signer = NewEIP2930Signer(common.Big1) + addr = common.HexToAddress("0x0000000000000000000000000000000000000001") + recipient = common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87") + accesses = AccessList{{Address: addr, StorageKeys: []common.Hash{{0}}}} + ) + for i := uint64(0); i < 500; i++ { + var txdata TxData + switch i % 5 { case 0: - tx = NewTransaction(i, common.Address{1}, common.Big0, 1, common.Big2, []byte("abcdef")) + // Legacy tx. + txdata = &LegacyTx{ + Nonce: i, + To: &recipient, + Gas: 1, + GasPrice: big.NewInt(2), + Data: []byte("abcdef"), + } case 1: - tx = NewContractCreation(i, common.Big0, 1, common.Big2, []byte("abcdef")) + // Legacy tx contract creation. + txdata = &LegacyTx{ + Nonce: i, + Gas: 1, + GasPrice: big.NewInt(2), + Data: []byte("abcdef"), + } + case 2: + // Tx with non-zero access list. + txdata = &AccessListTx{ + ChainID: big.NewInt(1), + Nonce: i, + To: &recipient, + Gas: 123457, + GasPrice: big.NewInt(10), + AccessList: accesses, + Data: []byte("abcdef"), + } + case 3: + // Tx with empty access list. + txdata = &AccessListTx{ + ChainID: big.NewInt(1), + Nonce: i, + To: &recipient, + Gas: 123457, + GasPrice: big.NewInt(10), + Data: []byte("abcdef"), + } + case 4: + // Contract creation with access list. + txdata = &AccessListTx{ + ChainID: big.NewInt(1), + Nonce: i, + Gas: 123457, + GasPrice: big.NewInt(10), + AccessList: accesses, + } } - transactions = append(transactions, tx) - - signedTx, err := SignTx(tx, signer, key) + tx, err := SignNewTx(key, signer, txdata) if err != nil { t.Fatalf("could not sign transaction: %v", err) } - - transactions = append(transactions, signedTx) - } - - for _, tx := range transactions { - data, err := json.Marshal(tx) + // RLP + parsedTx, err := encodeDecodeBinary(tx) if err != nil { - t.Fatalf("json.Marshal failed: %v", err) + t.Fatal(err) } + assertEqual(parsedTx, tx) - var parsedTx *Transaction - if err := json.Unmarshal(data, &parsedTx); err != nil { - t.Fatalf("json.Unmarshal failed: %v", err) + // JSON + parsedTx, err = encodeDecodeJSON(tx) + if err != nil { + t.Fatal(err) } + assertEqual(parsedTx, tx) + } +} - // compare nonce, price, gaslimit, recipient, amount, payload, V, R, S - if tx.Hash() != parsedTx.Hash() { - t.Errorf("parsed tx differs from original tx, want %v, got %v", tx, parsedTx) - } - if tx.ChainId().Cmp(parsedTx.ChainId()) != 0 { - t.Errorf("invalid chain id, want %d, got %d", tx.ChainId(), parsedTx.ChainId()) +func encodeDecodeJSON(tx *Transaction) (*Transaction, error) { + data, err := json.Marshal(tx) + if err != nil { + return nil, fmt.Errorf("json encoding failed: %v", err) + } + var parsedTx = &Transaction{} + if err := json.Unmarshal(data, &parsedTx); err != nil { + return nil, fmt.Errorf("json decoding failed: %v", err) + } + return parsedTx, nil +} + +func encodeDecodeBinary(tx *Transaction) (*Transaction, error) { + data, err := tx.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("rlp encoding failed: %v", err) + } + var parsedTx = &Transaction{} + if err := parsedTx.UnmarshalBinary(data); err != nil { + return nil, fmt.Errorf("rlp decoding failed: %v", err) + } + return parsedTx, nil +} + +func assertEqual(orig *Transaction, cpy *Transaction) error { + // compare nonce, price, gaslimit, recipient, amount, payload, V, R, S + if want, got := orig.Hash(), cpy.Hash(); want != got { + return fmt.Errorf("parsed tx differs from original tx, want %v, got %v", want, got) + } + if want, got := orig.ChainId(), cpy.ChainId(); want.Cmp(got) != 0 { + return fmt.Errorf("invalid chain id, want %d, got %d", want, got) + } + if orig.AccessList() != nil { + if !reflect.DeepEqual(orig.AccessList(), cpy.AccessList()) { + return fmt.Errorf("access list wrong!") } } + return nil } diff --git a/core/vm/interface.go b/core/vm/interface.go index fb5bbca48f..ad9b05d666 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -57,6 +57,7 @@ type StateDB interface { // is defined according to EIP161 (balance = nonce = code = 0). Empty(common.Address) bool + PrepareAccessList(sender common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) AddressInAccessList(addr common.Address) bool SlotInAccessList(addr common.Address, slot common.Hash) (addressOk bool, slotOk bool) // AddAddressToAccessList adds the given address to the access list. This operation is safe to perform diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 7cdb4ebd22..c97bdc420c 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -114,11 +114,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { sender = vm.AccountRef(cfg.Origin) ) if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { - cfg.State.AddAddressToAccessList(cfg.Origin) - cfg.State.AddAddressToAccessList(address) - for _, addr := range vmenv.ActivePrecompiles() { - cfg.State.AddAddressToAccessList(addr) - } + cfg.State.PrepareAccessList(cfg.Origin, &address, vmenv.ActivePrecompiles(), nil) } cfg.State.CreateAccount(address) // set the receiver's (the executing contract) code for execution. @@ -150,10 +146,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { sender = vm.AccountRef(cfg.Origin) ) if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { - cfg.State.AddAddressToAccessList(cfg.Origin) - for _, addr := range vmenv.ActivePrecompiles() { - cfg.State.AddAddressToAccessList(addr) - } + cfg.State.PrepareAccessList(cfg.Origin, nil, vmenv.ActivePrecompiles(), nil) } // Call the code with the given configuration. @@ -177,12 +170,9 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er vmenv := NewEnv(cfg) sender := cfg.State.GetOrNewStateObject(cfg.Origin) + statedb := cfg.State if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { - cfg.State.AddAddressToAccessList(cfg.Origin) - cfg.State.AddAddressToAccessList(address) - for _, addr := range vmenv.ActivePrecompiles() { - cfg.State.AddAddressToAccessList(addr) - } + statedb.PrepareAccessList(cfg.Origin, &address, vmenv.ActivePrecompiles(), nil) } // Call the code with the given configuration. diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 416a387e31..5ddd2f9848 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -1387,7 +1387,7 @@ func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) case err == nil: peer.log.Trace("Delivered new batch of data", "type", kind, "count", packet.Stats()) default: - peer.log.Trace("Failed to deliver retrieved data", "type", kind, "err", err) + peer.log.Debug("Failed to deliver retrieved data", "type", kind, "err", err) } } // Blocks assembled, try to update the progress diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 89caeeb45b..4fd2df10e2 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -63,7 +63,7 @@ func newTestBackend(t *testing.T) *testBackend { Config: params.TestChainConfig, Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, } - signer = types.NewEIP155Signer(gspec.Config.ChainID) + signer = types.LatestSigner(gspec.Config) ) engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 1bf59552c9..bd995d08a9 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -753,14 +753,15 @@ func (api *API) traceTx(ctx context.Context, message core.Message, vmctx vm.Bloc default: tracer = vm.NewStructLogger(config.LogConfig) } + // Run the transaction with tracing enabled. vmenv := vm.NewEVM(vmctx, txContext, statedb, api.backend.ChainConfig(), vm.Config{Debug: true, Tracer: tracer}) - result, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) if err != nil { return nil, fmt.Errorf("tracing failed: %v", err) } - // Depending on the tracer type, format and return the output + + // Depending on the tracer type, format and return the output. switch tracer := tracer.(type) { case *vm.StructLogger: // If the result contains a revert reason, return it. diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index dba7ce87cf..80775caa8e 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -556,7 +556,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost if data, ok := jst.ctx["input"].([]byte); ok { input = data } - intrinsicGas, err := core.IntrinsicGas(input, jst.ctx["type"] == "CREATE", isHomestead, isIstanbul) + intrinsicGas, err := core.IntrinsicGas(input, nil, jst.ctx["type"] == "CREATE", isHomestead, isIstanbul) if err != nil { return err } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 8dc34a835e..a17696356c 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -521,7 +520,7 @@ func (ec *Client) EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64 // If the transaction was a contract creation use the TransactionReceipt method to get the // contract address after the transaction has been mined. func (ec *Client) SendTransaction(ctx context.Context, tx *types.Transaction) error { - data, err := rlp.EncodeToBytes(tx) + data, err := tx.MarshalBinary() if err != nil { return err } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 8b175ee066..9a5a45e34f 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -560,7 +560,7 @@ func sendTransaction(ec *Client) error { } // Create transaction tx := types.NewTransaction(0, common.Address{1}, big.NewInt(1), 22000, big.NewInt(1), nil) - signer := types.NewEIP155Signer(chainID) + signer := types.LatestSignerForChainID(chainID) signature, err := crypto.Sign(signer.Hash(tx).Bytes(), testKey) if err != nil { return err diff --git a/ethclient/signer.go b/ethclient/signer.go index 74a93f1e2f..9de020b352 100644 --- a/ethclient/signer.go +++ b/ethclient/signer.go @@ -51,6 +51,9 @@ func (s *senderFromServer) Sender(tx *types.Transaction) (common.Address, error) return s.addr, nil } +func (s *senderFromServer) ChainID() *big.Int { + panic("can't sign with senderFromServer") +} func (s *senderFromServer) Hash(tx *types.Transaction) common.Hash { panic("can't sign with senderFromServer") } diff --git a/graphql/graphql.go b/graphql/graphql.go index 5dc0f723d3..2374beb8e1 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -33,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/internal/ethapi" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -246,12 +245,8 @@ func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, if err != nil || tx == nil { return nil, err } - var signer types.Signer = types.HomesteadSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } + signer := types.LatestSigner(t.backend.ChainConfig()) from, _ := types.Sender(signer, tx) - return &Account{ backend: t.backend, address: from, @@ -1022,7 +1017,7 @@ func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Has func (r *Resolver) SendRawTransaction(ctx context.Context, args struct{ Data hexutil.Bytes }) (common.Hash, error) { tx := new(types.Transaction) - if err := rlp.DecodeBytes(args.Data, tx); err != nil { + if err := tx.UnmarshalBinary(args.Data); err != nil { return common.Hash{}, err } hash, err := ethapi.SubmitTransaction(ctx, r.backend, tx) diff --git a/interfaces.go b/interfaces.go index 1ff31f96b6..afcdc17e58 100644 --- a/interfaces.go +++ b/interfaces.go @@ -119,6 +119,8 @@ type CallMsg struct { GasPrice *big.Int // wei <-> gas exchange ratio Value *big.Int // amount of wei sent along with the call Data []byte // input data, usually an ABI-encoded contract method invocation + + AccessList types.AccessList // EIP-2930 access list. } // A ContractCaller provides contract calls, essentially transactions that are executed by diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 52b4f5f506..622063cf64 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -410,7 +410,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs log.Warn("Failed transaction sign attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err) return nil, err } - data, err := rlp.EncodeToBytes(signed) + data, err := signed.MarshalBinary() if err != nil { return nil, err } @@ -748,12 +748,13 @@ func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.A // CallArgs represents the arguments for a call. type CallArgs struct { - From *common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Value *hexutil.Big `json:"value"` - Data *hexutil.Bytes `json:"data"` + From *common.Address `json:"from"` + To *common.Address `json:"to"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + Value *hexutil.Big `json:"value"` + Data *hexutil.Bytes `json:"data"` + AccessList *types.AccessList `json:"accessList"` } // ToMessage converts CallArgs to the Message type used by the core evm @@ -780,18 +781,20 @@ func (args *CallArgs) ToMessage(globalGasCap uint64) types.Message { if args.GasPrice != nil { gasPrice = args.GasPrice.ToInt() } - value := new(big.Int) if args.Value != nil { value = args.Value.ToInt() } - var data []byte if args.Data != nil { data = *args.Data } + var accessList types.AccessList + if args.AccessList != nil { + accessList = *args.AccessList + } - msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, data, false) + msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, data, accessList, false) return msg } @@ -869,13 +872,13 @@ func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.Blo evm.Cancel() }() - // Setup the gas pool (also for unmetered requests) - // and apply the message. + // Execute the message. gp := new(core.GasPool).AddGas(math.MaxUint64) result, err := core.ApplyMessage(evm, msg, gp) if err := vmError(); err != nil { return nil, err } + // If the timer caused an abort, return an appropriate error message if evm.Cancelled() { return nil, fmt.Errorf("execution aborted (timeout = %v)", timeout) @@ -1200,33 +1203,43 @@ func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Bloc // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction type RPCTransaction struct { - BlockHash *common.Hash `json:"blockHash"` - BlockNumber *hexutil.Big `json:"blockNumber"` - From common.Address `json:"from"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Hash common.Hash `json:"hash"` - Input hexutil.Bytes `json:"input"` - Nonce hexutil.Uint64 `json:"nonce"` - To *common.Address `json:"to"` - TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` - Value *hexutil.Big `json:"value"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` + BlockHash *common.Hash `json:"blockHash"` + BlockNumber *hexutil.Big `json:"blockNumber"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + Hash common.Hash `json:"hash"` + Input hexutil.Bytes `json:"input"` + Nonce hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` + Value *hexutil.Big `json:"value"` + Type hexutil.Uint64 `json:"type"` + Accesses *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` } // newRPCTransaction returns a transaction that will serialize to the RPC // representation, with the given location metadata set (if available). func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction { - var signer types.Signer = types.FrontierSigner{} + // Determine the signer. For replay-protected transactions, use the most permissive + // signer, because we assume that signers are backwards-compatible with old + // transactions. For non-protected transactions, the homestead signer signer is used + // because the return value of ChainId is zero for those transactions. + var signer types.Signer if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) + signer = types.LatestSignerForChainID(tx.ChainId()) + } else { + signer = types.HomesteadSigner{} } + from, _ := types.Sender(signer, tx) v, r, s := tx.RawSignatureValues() - result := &RPCTransaction{ + Type: hexutil.Uint64(tx.Type()), From: from, Gas: hexutil.Uint64(tx.Gas()), GasPrice: (*hexutil.Big)(tx.GasPrice()), @@ -1244,6 +1257,11 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) result.TransactionIndex = (*hexutil.Uint64)(&index) } + if tx.Type() == types.AccessListTxType { + al := tx.AccessList() + result.Accesses = &al + result.ChainID = (*hexutil.Big)(tx.ChainId()) + } return result } @@ -1267,7 +1285,7 @@ func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.By if index >= uint64(len(txs)) { return nil } - blob, _ := rlp.EncodeToBytes(txs[index]) + blob, _ := txs[index].MarshalBinary() return blob } @@ -1285,11 +1303,15 @@ func newRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransa type PublicTransactionPoolAPI struct { b Backend nonceLock *AddrLocker + signer types.Signer } // NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransactionPoolAPI { - return &PublicTransactionPoolAPI{b, nonceLock} + // The signer used by the API should always be the 'latest' known one because we expect + // signers to be backwards-compatible with old transactions. + signer := types.LatestSigner(b.ChainConfig()) + return &PublicTransactionPoolAPI{b, nonceLock, signer} } // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. @@ -1394,7 +1416,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, } } // Serialize to RLP and return - return rlp.EncodeToBytes(tx) + return tx.MarshalBinary() } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. @@ -1412,10 +1434,9 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha } receipt := receipts[index] - var signer types.Signer = types.FrontierSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } + // Derive the sender. + bigblock := new(big.Int).SetUint64(blockNumber) + signer := types.MakeSigner(s.b.ChainConfig(), bigblock) from, _ := types.Sender(signer, tx) fields := map[string]interface{}{ @@ -1430,6 +1451,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha "contractAddress": nil, "logs": receipt.Logs, "logsBloom": receipt.Bloom, + "type": hexutil.Uint(tx.Type()), } // Assign receipt status or post state. @@ -1473,9 +1495,13 @@ type SendTxArgs struct { // newer name and should be preferred by clients. Data *hexutil.Bytes `json:"data"` Input *hexutil.Bytes `json:"input"` + + // For non-legacy transactions + AccessList *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` } -// setDefaults is a helper function that fills in default values for unspecified tx fields. +// setDefaults fills in default values for unspecified tx fields. func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { if args.GasPrice == nil { price, err := b.SuggestPrice(ctx) @@ -1509,6 +1535,7 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { return errors.New(`contract creation without any data provided`) } } + // Estimate the gas usage if necessary. if args.Gas == nil { // For backwards-compatibility reason, we try both input and data @@ -1518,11 +1545,12 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { input = args.Data } callArgs := CallArgs{ - From: &args.From, // From shouldn't be nil - To: args.To, - GasPrice: args.GasPrice, - Value: args.Value, - Data: input, + From: &args.From, // From shouldn't be nil + To: args.To, + GasPrice: args.GasPrice, + Value: args.Value, + Data: input, + AccessList: args.AccessList, } pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) estimated, err := DoEstimateGas(ctx, b, callArgs, pendingBlockNr, b.RPCGasCap()) @@ -1532,9 +1560,15 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { args.Gas = &estimated log.Trace("Estimate gas usage automatically", "gas", args.Gas) } + if args.ChainID == nil { + id := (*hexutil.Big)(b.ChainConfig().ChainID) + args.ChainID = id + } return nil } +// toTransaction converts the arguments to a transaction. +// This assumes that setDefaults has been called. func (args *SendTxArgs) toTransaction() *types.Transaction { var input []byte if args.Input != nil { @@ -1542,10 +1576,30 @@ func (args *SendTxArgs) toTransaction() *types.Transaction { } else if args.Data != nil { input = *args.Data } - if args.To == nil { - return types.NewContractCreation(uint64(*args.Nonce), (*big.Int)(args.Value), uint64(*args.Gas), (*big.Int)(args.GasPrice), input) + + var data types.TxData + if args.AccessList == nil { + data = &types.LegacyTx{ + To: args.To, + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(args.Value), + Data: input, + } + } else { + data = &types.AccessListTx{ + To: args.To, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(args.Value), + Data: input, + AccessList: *args.AccessList, + } } - return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), uint64(*args.Gas), (*big.Int)(args.GasPrice), input) + return types.NewTx(data) } // SubmitTransaction is a helper function that submits tx to txPool and logs a message. @@ -1619,7 +1673,7 @@ func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args Sen } // Assemble the transaction and obtain rlp tx := args.toTransaction() - data, err := rlp.EncodeToBytes(tx) + data, err := tx.MarshalBinary() if err != nil { return nil, err } @@ -1628,9 +1682,9 @@ func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args Sen // SendRawTransaction will add the signed transaction to the transaction pool. // The sender is responsible for signing the transaction and using the correct nonce. -func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encodedTx hexutil.Bytes) (common.Hash, error) { +func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input hexutil.Bytes) (common.Hash, error) { tx := new(types.Transaction) - if err := rlp.DecodeBytes(encodedTx, tx); err != nil { + if err := tx.UnmarshalBinary(input); err != nil { return common.Hash{}, err } return SubmitTransaction(ctx, s.b, tx) @@ -1691,7 +1745,7 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Sen if err != nil { return nil, err } - data, err := rlp.EncodeToBytes(tx) + data, err := tx.MarshalBinary() if err != nil { return nil, err } @@ -1713,11 +1767,7 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err } transactions := make([]*RPCTransaction, 0, len(pending)) for _, tx := range pending { - var signer types.Signer = types.HomesteadSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } - from, _ := types.Sender(signer, tx) + from, _ := types.Sender(s.signer, tx) if _, exists := accounts[from]; exists { transactions = append(transactions, newRPCPendingTransaction(tx)) } @@ -1754,13 +1804,9 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr return common.Hash{}, err } for _, p := range pending { - var signer types.Signer = types.HomesteadSigner{} - if p.Protected() { - signer = types.NewEIP155Signer(p.ChainId()) - } - wantSigHash := signer.Hash(matchTx) - - if pFrom, err := types.Sender(signer, p); err == nil && pFrom == sendArgs.From && signer.Hash(p) == wantSigHash { + wantSigHash := s.signer.Hash(matchTx) + pFrom, err := types.Sender(s.signer, p) + if err == nil && pFrom == sendArgs.From && s.signer.Hash(p) == wantSigHash { // Match. Re-sign and send the transaction. if gasPrice != nil && (*big.Int)(gasPrice).Sign() != 0 { sendArgs.GasPrice = gasPrice @@ -1778,7 +1824,6 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr return signedTx.Hash(), nil } } - return common.Hash{}, fmt.Errorf("transaction %#x not found", matchTx.Hash()) } diff --git a/internal/guide/guide_test.go b/internal/guide/guide_test.go index 9c7ad16d18..abc48e0e4b 100644 --- a/internal/guide/guide_test.go +++ b/internal/guide/guide_test.go @@ -31,6 +31,7 @@ import ( "time" "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) @@ -75,7 +76,8 @@ func TestAccountManagement(t *testing.T) { if err != nil { t.Fatalf("Failed to create signer account: %v", err) } - tx, chain := new(types.Transaction), big.NewInt(1) + tx := types.NewTransaction(0, common.Address{}, big.NewInt(0), 0, big.NewInt(0), nil) + chain := big.NewInt(1) // Sign a transaction with a single authorization if _, err := ks.SignTxWithPassphrase(signer, "Signer password", tx, chain); err != nil { diff --git a/les/benchmark.go b/les/benchmark.go index 6255c1049e..757822a6b3 100644 --- a/les/benchmark.go +++ b/les/benchmark.go @@ -171,7 +171,7 @@ type benchmarkTxSend struct { func (b *benchmarkTxSend) init(h *serverHandler, count int) error { key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - signer := types.NewEIP155Signer(big.NewInt(18)) + signer := types.LatestSigner(h.server.chainConfig) b.txs = make(types.Transactions, count) for i := range b.txs { diff --git a/les/odr_test.go b/les/odr_test.go index a43382e05e..0c75014d49 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -135,7 +135,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai from := statedb.GetOrNewStateObject(bankAddr) from.SetBalance(math.MaxBig256) - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, false)} + msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, nil, false)} context := core.NewEVMBlockContext(header, bc, nil) txContext := core.NewEVMTxContext(msg) @@ -150,7 +150,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai header := lc.GetHeaderByHash(bhash) state := light.NewState(ctx, header, lc.Odr()) state.SetBalance(bankAddr, math.MaxBig256) - msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, false)} + msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, nil, false)} context := core.NewEVMBlockContext(header, lc, nil) txContext := core.NewEVMTxContext(msg) vmenv := vm.NewEVM(context, txContext, state, config, vm.Config{}) diff --git a/light/odr_test.go b/light/odr_test.go index cb22334fdc..0fc45b8734 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -194,7 +194,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain // Perform read-only call. st.SetBalance(testBankAddress, math.MaxBig256) - msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, new(big.Int), data, false)} + msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, new(big.Int), data, nil, false)} txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, chain, nil) vmenv := vm.NewEVM(context, txContext, st, config, vm.Config{}) diff --git a/light/txpool.go b/light/txpool.go index 2831de5a65..bf5f9ff583 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" ) const ( @@ -69,6 +68,7 @@ type TxPool struct { clearIdx uint64 // earliest block nr that can contain mined tx info istanbul bool // Fork indicator whether we are in the istanbul stage. + eip2718 bool // Fork indicator whether we are in the eip2718 stage. } // TxRelayBackend provides an interface to the mechanism that forwards transacions @@ -90,7 +90,7 @@ type TxRelayBackend interface { func NewTxPool(config *params.ChainConfig, chain *LightChain, relay TxRelayBackend) *TxPool { pool := &TxPool{ config: config, - signer: types.NewEIP155Signer(config.ChainID), + signer: types.LatestSigner(config), nonce: make(map[common.Address]uint64), pending: make(map[common.Hash]*types.Transaction), mined: make(map[common.Hash][]*types.Transaction), @@ -314,6 +314,7 @@ func (pool *TxPool) setNewHead(head *types.Header) { // Update fork indicator by next pending block number next := new(big.Int).Add(head.Number, big.NewInt(1)) pool.istanbul = pool.config.IsIstanbul(next) + pool.eip2718 = pool.config.IsYoloV3(next) } // Stop stops the light transaction pool @@ -381,7 +382,7 @@ func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error } // Should supply enough intrinsic gas - gas, err := core.IntrinsicGas(tx.Data(), tx.To() == nil, true, pool.istanbul) + gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, true, pool.istanbul) if err != nil { return err } @@ -430,8 +431,7 @@ func (pool *TxPool) add(ctx context.Context, tx *types.Transaction) error { func (pool *TxPool) Add(ctx context.Context, tx *types.Transaction) error { pool.mu.Lock() defer pool.mu.Unlock() - - data, err := rlp.EncodeToBytes(tx) + data, err := tx.MarshalBinary() if err != nil { return err } diff --git a/miner/worker.go b/miner/worker.go index e81d50e46e..2cee6af0c3 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -654,7 +654,7 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { state.StartPrefetcher("miner") env := &environment{ - signer: types.NewEIP155Signer(w.chainConfig.ChainID), + signer: types.MakeSigner(w.chainConfig, header.Number), state: state, ancestors: mapset.NewSet(), family: mapset.NewSet(), @@ -829,6 +829,11 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin w.current.tcount++ txs.Shift() + case errors.Is(err, core.ErrTxTypeNotSupported): + // Pop the unsupported transaction without shifting in the next from the account + log.Trace("Skipping unsupported transaction type", "sender", from, "type", tx.Type()) + txs.Pop() + default: // Strange error, discard the transaction and get the next in line (note, the // nonce-too-high clause will prevent us from executing in vain). diff --git a/miner/worker_test.go b/miner/worker_test.go index a5c558ba5f..0fe62316e1 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -81,10 +81,25 @@ func init() { Period: 10, Epoch: 30000, } - tx1, _ := types.SignTx(types.NewTransaction(0, testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + + signer := types.LatestSigner(params.TestChainConfig) + tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ + ChainID: params.TestChainConfig.ChainID, + Nonce: 0, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + }) pendingTxs = append(pendingTxs, tx1) - tx2, _ := types.SignTx(types.NewTransaction(1, testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + + tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ + Nonce: 1, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + }) newTxs = append(newTxs, tx2) + rand.Seed(time.Now().UnixNano()) } diff --git a/params/config.go b/params/config.go index 0dbf592d19..929bdbeb94 100644 --- a/params/config.go +++ b/params/config.go @@ -215,7 +215,7 @@ var ( // YoloV3ChainConfig contains the chain parameters to run a node on the YOLOv3 test network. YoloV3ChainConfig = &ChainConfig{ - ChainID: big.NewInt(133519467574834), + ChainID: new(big.Int).SetBytes([]byte("yolov3x")), HomesteadBlock: big.NewInt(0), DAOForkBlock: nil, DAOForkSupport: true, @@ -246,9 +246,9 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) diff --git a/params/protocol_params.go b/params/protocol_params.go index ffb901ec3d..88f1a06e12 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -60,20 +60,23 @@ const ( JumpdestGas uint64 = 1 // Once per JUMPDEST operation. EpochDuration uint64 = 30000 // Duration between proof-of-work epochs. - CreateDataGas uint64 = 200 // - CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. - ExpGas uint64 = 10 // Once per EXP instruction - LogGas uint64 = 375 // Per LOG* operation. - CopyGas uint64 = 3 // - StackLimit uint64 = 1024 // Maximum size of VM stack allowed. - TierStepGas uint64 = 0 // Once per operation, for a selection of them. - LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. - CreateGas uint64 = 32000 // Once per CREATE operation & contract-creation transaction. - Create2Gas uint64 = 32000 // Once per CREATE2 operation - SelfdestructRefundGas uint64 = 24000 // Refunded following a selfdestruct operation. - MemoryGas uint64 = 3 // Times the address of the (highest referenced byte in memory + 1). NOTE: referencing happens on read, write and in instructions such as RETURN and CALL. - TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. - TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + CreateDataGas uint64 = 200 // + CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. + ExpGas uint64 = 10 // Once per EXP instruction + LogGas uint64 = 375 // Per LOG* operation. + CopyGas uint64 = 3 // + StackLimit uint64 = 1024 // Maximum size of VM stack allowed. + TierStepGas uint64 = 0 // Once per operation, for a selection of them. + LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. + CreateGas uint64 = 32000 // Once per CREATE operation & contract-creation transaction. + Create2Gas uint64 = 32000 // Once per CREATE2 operation + SelfdestructRefundGas uint64 = 24000 // Refunded following a selfdestruct operation. + MemoryGas uint64 = 3 // Times the address of the (highest referenced byte in memory + 1). NOTE: referencing happens on read, write and in instructions such as RETURN and CALL. + + TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. + TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list + TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list // These have been changed during the course of the chain CallGasFrontier uint64 = 40 // Once per CALL operation & message call transaction. diff --git a/signer/core/api.go b/signer/core/api.go index 07e206a74c..968dcfb2ed 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -33,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/signer/storage" ) @@ -574,11 +573,11 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth return nil, err } - rlpdata, err := rlp.EncodeToBytes(signedTx) + data, err := signedTx.MarshalBinary() if err != nil { return nil, err } - response := ethapi.SignTransactionResult{Raw: rlpdata, Tx: signedTx} + response := ethapi.SignTransactionResult{Raw: data, Tx: signedTx} // Finally, send the signed tx to the UI api.UI.OnApprovedTx(response) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 53c7d955be..03dc01459c 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -182,27 +182,21 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh if err != nil { return nil, nil, common.Hash{}, err } + + // Prepare the EVM. txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) - if config.IsYoloV3(context.BlockNumber) { - statedb.AddAddressToAccessList(msg.From()) - if dst := msg.To(); dst != nil { - statedb.AddAddressToAccessList(*dst) - // If it's a create-tx, the destination will be added inside evm.create - } - for _, addr := range evm.ActivePrecompiles() { - statedb.AddAddressToAccessList(addr) - } - } + // Execute the message. + snapshot := statedb.Snapshot() gaspool := new(core.GasPool) gaspool.AddGas(block.GasLimit()) - snapshot := statedb.Snapshot() if _, err := core.ApplyMessage(evm, msg, gaspool); err != nil { statedb.RevertToSnapshot(snapshot) } + // Commit block statedb.Commit(config.IsEIP158(block.Number())) // Add 0-value mining reward. This only makes a difference in the cases @@ -300,7 +294,7 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { return nil, fmt.Errorf("invalid tx data %q", dataHex) } - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, true) + msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, nil, true) return msg, nil } diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index aea90535c3..82ee01de15 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -55,7 +55,7 @@ func (tt *TransactionTest) Run(config *params.ChainConfig) error { return nil, nil, err } // Intrinsic gas - requiredGas, err := core.IntrinsicGas(tx.Data(), tx.To() == nil, isHomestead, isIstanbul) + requiredGas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, isHomestead, isIstanbul) if err != nil { return nil, nil, err } diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index d4488b4029..29706f2e9d 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -1,16 +1,9 @@ package trie import ( - "bytes" - "fmt" - "math/big" - mrand "math/rand" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb/memorydb" ) @@ -78,169 +71,6 @@ func TestValLength56(t *testing.T) { } } -func genTxs(num uint64) (types.Transactions, error) { - key, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") - if err != nil { - return nil, err - } - var addr = crypto.PubkeyToAddress(key.PublicKey) - newTx := func(i uint64) (*types.Transaction, error) { - signer := types.NewEIP155Signer(big.NewInt(18)) - tx, err := types.SignTx(types.NewTransaction(i, addr, new(big.Int), 0, new(big.Int).SetUint64(10000000), nil), signer, key) - return tx, err - } - var txs types.Transactions - for i := uint64(0); i < num; i++ { - tx, err := newTx(i) - if err != nil { - return nil, err - } - txs = append(txs, tx) - } - return txs, nil -} - -func TestDeriveSha(t *testing.T) { - txs, err := genTxs(0) - if err != nil { - t.Fatal(err) - } - for len(txs) < 1000 { - exp := types.DeriveSha(txs, newEmpty()) - got := types.DeriveSha(txs, NewStackTrie(nil)) - if !bytes.Equal(got[:], exp[:]) { - t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) - } - newTxs, err := genTxs(uint64(len(txs) + 1)) - if err != nil { - t.Fatal(err) - } - txs = append(txs, newTxs...) - } -} - -func BenchmarkDeriveSha200(b *testing.B) { - txs, err := genTxs(200) - if err != nil { - b.Fatal(err) - } - var exp common.Hash - var got common.Hash - b.Run("std_trie", func(b *testing.B) { - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - exp = types.DeriveSha(txs, newEmpty()) - } - }) - - b.Run("stack_trie", func(b *testing.B) { - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - got = types.DeriveSha(txs, NewStackTrie(nil)) - } - }) - if got != exp { - b.Errorf("got %x exp %x", got, exp) - } -} - -type dummyDerivableList struct { - len int - seed int -} - -func newDummy(seed int) *dummyDerivableList { - d := &dummyDerivableList{} - src := mrand.NewSource(int64(seed)) - // don't use lists longer than 4K items - d.len = int(src.Int63() & 0x0FFF) - d.seed = seed - return d -} - -func (d *dummyDerivableList) Len() int { - return d.len -} - -func (d *dummyDerivableList) GetRlp(i int) []byte { - src := mrand.NewSource(int64(d.seed + i)) - // max item size 256, at least 1 byte per item - size := 1 + src.Int63()&0x00FF - data := make([]byte, size) - _, err := mrand.New(src).Read(data) - if err != nil { - panic(err) - } - return data -} - -func printList(l types.DerivableList) { - fmt.Printf("list length: %d\n", l.Len()) - fmt.Printf("{\n") - for i := 0; i < l.Len(); i++ { - v := l.GetRlp(i) - fmt.Printf("\"0x%x\",\n", v) - } - fmt.Printf("},\n") -} - -func TestFuzzDeriveSha(t *testing.T) { - // increase this for longer runs -- it's set to quite low for travis - rndSeed := mrand.Int() - for i := 0; i < 10; i++ { - seed := rndSeed + i - exp := types.DeriveSha(newDummy(i), newEmpty()) - got := types.DeriveSha(newDummy(i), NewStackTrie(nil)) - if !bytes.Equal(got[:], exp[:]) { - printList(newDummy(seed)) - t.Fatalf("seed %d: got %x exp %x", seed, got, exp) - } - } -} - -type flatList struct { - rlpvals []string -} - -func newFlatList(rlpvals []string) *flatList { - return &flatList{rlpvals} -} -func (f *flatList) Len() int { - return len(f.rlpvals) -} -func (f *flatList) GetRlp(i int) []byte { - return hexutil.MustDecode(f.rlpvals[i]) -} - -// TestDerivableList contains testcases found via fuzzing -func TestDerivableList(t *testing.T) { - type tcase []string - tcs := []tcase{ - { - "0xc041", - }, - { - "0xf04cf757812428b0763112efb33b6f4fad7deb445e", - "0xf04cf757812428b0763112efb33b6f4fad7deb445e", - }, - { - "0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d", - "0x6cd850eca0a7ac46bb1748d7b9cb88aa3bd21c57d852c28198ad8fa422c4595032e88a4494b4778b36b944fe47a52b8c5cd312910139dfcb4147ab8e972cc456bcb063f25dd78f54c4d34679e03142c42c662af52947d45bdb6e555751334ace76a5080ab5a0256a1d259855dfc5c0b8023b25befbb13fd3684f9f755cbd3d63544c78ee2001452dd54633a7593ade0b183891a0a4e9c7844e1254005fbe592b1b89149a502c24b6e1dca44c158aebedf01beae9c30cabe16a", - "0x14abd5c47c0be87b0454596baad2", - "0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d", - }, - } - for i, tc := range tcs[1:] { - exp := types.DeriveSha(newFlatList(tc), newEmpty()) - got := types.DeriveSha(newFlatList(tc), NewStackTrie(nil)) - if !bytes.Equal(got[:], exp[:]) { - t.Fatalf("case %d: got %x exp %x", i, got, exp) - } - } -} - // TestUpdateSmallNodes tests a case where the leaves are small (both key and value), // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { From de9465f991916e183a504ce79988c6cef544f7f1 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:36:01 +0100 Subject: [PATCH 163/709] cmd/devp2p: add eth66 test suite (#22363) Co-authored-by: Martin Holst Swende --- cmd/devp2p/README.md | 12 +- cmd/devp2p/internal/ethtest/eth66_suite.go | 382 ++++++++++++++++++ .../internal/ethtest/eth66_suiteHelpers.go | 270 +++++++++++++ cmd/devp2p/internal/ethtest/suite.go | 39 +- cmd/devp2p/internal/ethtest/transaction.go | 29 +- cmd/devp2p/internal/ethtest/types.go | 8 +- cmd/devp2p/rlpxcmd.go | 7 +- go.mod | 1 + 8 files changed, 721 insertions(+), 27 deletions(-) create mode 100644 cmd/devp2p/internal/ethtest/eth66_suite.go create mode 100644 cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go diff --git a/cmd/devp2p/README.md b/cmd/devp2p/README.md index ca0b852fda..e934ee25c9 100644 --- a/cmd/devp2p/README.md +++ b/cmd/devp2p/README.md @@ -100,7 +100,17 @@ Then, run the following command, replacing `` with the enode of the geth ``` Repeat the above process (re-initialising the node) in order to run the Eth Protocol test suite again. - + +#### Eth66 Test Suite + +The Eth66 test suite is also a conformance test suite for the eth 66 protocol version specifically. +To run the eth66 protocol test suite, initialize a geth node as described above and run the following command, +replacing `` with the enode of the geth node: + + ``` + devp2p rlpx eth66-test cmd/devp2p/internal/ethtest/testdata/chain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json +``` + [eth]: https://github.com/ethereum/devp2p/blob/master/caps/eth.md [dns-tutorial]: https://geth.ethereum.org/docs/developers/dns-discovery-setup [discv4]: https://github.com/ethereum/devp2p/tree/master/discv4.md diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go new file mode 100644 index 0000000000..644fed61eb --- /dev/null +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -0,0 +1,382 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethtest + +import ( + "time" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/p2p" +) + +// TestStatus_66 attempts to connect to the given node and exchange +// a status message with it on the eth66 protocol, and then check to +// make sure the chain head is correct. +func (s *Suite) TestStatus_66(t *utesting.T) { + conn := s.dial66(t) + // get protoHandshake + conn.handshake(t) + // get status + switch msg := conn.statusExchange66(t, s.chain).(type) { + case *Status: + status := *msg + if status.ProtocolVersion != uint32(66) { + t.Fatalf("mismatch in version: wanted 66, got %d", status.ProtocolVersion) + } + t.Logf("got status message: %s", pretty.Sdump(msg)) + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } +} + +// TestGetBlockHeaders_66 tests whether the given node can respond to +// an eth66 `GetBlockHeaders` request and that the response is accurate. +func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) { + conn := s.setupConnection66(t) + // get block headers + req := ð.GetBlockHeadersPacket66{ + RequestId: 3, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Hash: s.chain.blocks[1].Hash(), + }, + Amount: 2, + Skip: 1, + Reverse: false, + }, + } + // write message + headers := s.getBlockHeaders66(t, conn, req, req.RequestId) + // check for correct headers + headersMatch(t, s.chain, headers) +} + +// TestSimultaneousRequests_66 sends two simultaneous `GetBlockHeader` requests +// with different request IDs and checks to make sure the node responds with the correct +// headers per request. +func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { + // create two connections + conn1, conn2 := s.setupConnection66(t), s.setupConnection66(t) + // create two requests + req1 := ð.GetBlockHeadersPacket66{ + RequestId: 111, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Hash: s.chain.blocks[1].Hash(), + }, + Amount: 2, + Skip: 1, + Reverse: false, + }, + } + req2 := ð.GetBlockHeadersPacket66{ + RequestId: 222, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Hash: s.chain.blocks[1].Hash(), + }, + Amount: 4, + Skip: 1, + Reverse: false, + }, + } + // wait for headers for first request + headerChan := make(chan BlockHeaders, 1) + go func(headers chan BlockHeaders) { + headers <- s.getBlockHeaders66(t, conn1, req1, req1.RequestId) + }(headerChan) + // check headers of second request + headersMatch(t, s.chain, s.getBlockHeaders66(t, conn2, req2, req2.RequestId)) + // check headers of first request + headersMatch(t, s.chain, <-headerChan) +} + +// TestBroadcast_66 tests whether a block announcement is correctly +// propagated to the given node's peer(s) on the eth66 protocol. +func (s *Suite) TestBroadcast_66(t *utesting.T) { + sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) + nextBlock := len(s.chain.blocks) + blockAnnouncement := &NewBlock{ + Block: s.fullChain.blocks[nextBlock], + TD: s.fullChain.TD(nextBlock + 1), + } + s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement) + // update test suite chain + s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) + // wait for client to update its chain + if err := receiveConn.waitForBlock66(s.chain.Head()); err != nil { + t.Fatal(err) + } +} + +// TestGetBlockBodies_66 tests whether the given node can respond to +// a `GetBlockBodies` request and that the response is accurate over +// the eth66 protocol. +func (s *Suite) TestGetBlockBodies_66(t *utesting.T) { + conn := s.setupConnection66(t) + // create block bodies request + id := uint64(55) + req := ð.GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ + s.chain.blocks[54].Hash(), + s.chain.blocks[75].Hash(), + }, + } + if err := conn.write66(req, GetBlockBodies{}.Code()); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + + reqID, msg := conn.readAndServe66(s.chain, timeout) + switch msg := msg.(type) { + case BlockBodies: + if reqID != req.RequestId { + t.Fatalf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID) + } + t.Logf("received %d block bodies", len(msg)) + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } +} + +// TestLargeAnnounce_66 tests the announcement mechanism with a large block. +func (s *Suite) TestLargeAnnounce_66(t *utesting.T) { + nextBlock := len(s.chain.blocks) + blocks := []*NewBlock{ + { + Block: largeBlock(), + TD: s.fullChain.TD(nextBlock + 1), + }, + { + Block: s.fullChain.blocks[nextBlock], + TD: largeNumber(2), + }, + { + Block: largeBlock(), + TD: largeNumber(2), + }, + { + Block: s.fullChain.blocks[nextBlock], + TD: s.fullChain.TD(nextBlock + 1), + }, + } + + for i, blockAnnouncement := range blocks[0:3] { + t.Logf("Testing malicious announcement: %v\n", i) + sendConn := s.setupConnection66(t) + if err := sendConn.Write(blockAnnouncement); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + // Invalid announcement, check that peer disconnected + switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) { + case *Disconnect: + case *Error: + break + default: + t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) + } + } + // Test the last block as a valid block + sendConn := s.setupConnection66(t) + receiveConn := s.setupConnection66(t) + s.testAnnounce66(t, sendConn, receiveConn, blocks[3]) + // update test suite chain + s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) + // wait for client to update its chain + if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil { + t.Fatal(err) + } +} + +// TestMaliciousHandshake_66 tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) { + conn := s.dial66(t) + // write hello to client + pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] + handshakes := []*Hello{ + { + Version: 5, + Caps: []p2p.Cap{ + {Name: largeString(2), Version: 66}, + }, + ID: pub0, + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + {Name: "eth", Version: 66}, + }, + ID: append(pub0, byte(0)), + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + {Name: "eth", Version: 66}, + }, + ID: append(pub0, pub0...), + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + {Name: "eth", Version: 66}, + }, + ID: largeBuffer(2), + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: largeString(2), Version: 66}, + }, + ID: largeBuffer(2), + }, + } + for i, handshake := range handshakes { + t.Logf("Testing malicious handshake %v\n", i) + // Init the handshake + if err := conn.Write(handshake); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + // check that the peer disconnected + timeout := 20 * time.Second + // Discard one hello + for i := 0; i < 2; i++ { + switch msg := conn.ReadAndServe(s.chain, timeout).(type) { + case *Disconnect: + case *Error: + case *Hello: + // Hello's are sent concurrently, so ignore them + continue + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } + } + // Dial for the next round + conn = s.dial66(t) + } +} + +// TestMaliciousStatus_66 sends a status package with a large total difficulty. +func (s *Suite) TestMaliciousStatus_66(t *utesting.T) { + conn := s.dial66(t) + // get protoHandshake + conn.handshake(t) + status := &Status{ + ProtocolVersion: uint32(66), + NetworkID: s.chain.chainConfig.ChainID.Uint64(), + TD: largeNumber(2), + Head: s.chain.blocks[s.chain.Len()-1].Hash(), + Genesis: s.chain.blocks[0].Hash(), + ForkID: s.chain.ForkID(), + } + // get status + switch msg := conn.statusExchange(t, s.chain, status).(type) { + case *Status: + t.Logf("%+v\n", msg) + default: + t.Fatalf("expected status, got: %#v ", msg) + } + // wait for disconnect + switch msg := conn.ReadAndServe(s.chain, timeout).(type) { + case *Disconnect: + case *Error: + return + default: + t.Fatalf("expected disconnect, got: %s", pretty.Sdump(msg)) + } +} + +func (s *Suite) TestTransaction_66(t *utesting.T) { + tests := []*types.Transaction{ + getNextTxFromChain(t, s), + unknownTx(t, s), + } + for i, tx := range tests { + t.Logf("Testing tx propagation: %v\n", i) + sendSuccessfulTx66(t, s, tx) + } +} + +func (s *Suite) TestMaliciousTx_66(t *utesting.T) { + tests := []*types.Transaction{ + getOldTxFromChain(t, s), + invalidNonceTx(t, s), + hugeAmount(t, s), + hugeGasPrice(t, s), + hugeData(t, s), + } + for i, tx := range tests { + t.Logf("Testing malicious tx propagation: %v\n", i) + sendFailingTx66(t, s, tx) + } +} + +// TestZeroRequestID_66 checks that a request ID of zero is still handled +// by the node. +func (s *Suite) TestZeroRequestID_66(t *utesting.T) { + conn := s.setupConnection66(t) + req := ð.GetBlockHeadersPacket66{ + RequestId: 0, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Number: 0, + }, + Amount: 2, + }, + } + headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req, req.RequestId)) +} + +// TestSameRequestID_66 sends two requests with the same request ID +// concurrently to a single node. +func (s *Suite) TestSameRequestID_66(t *utesting.T) { + conn := s.setupConnection66(t) + // create two separate requests with same ID + reqID := uint64(1234) + req1 := ð.GetBlockHeadersPacket66{ + RequestId: reqID, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Number: 0, + }, + Amount: 2, + }, + } + req2 := ð.GetBlockHeadersPacket66{ + RequestId: reqID, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Number: 33, + }, + Amount: 2, + }, + } + // send requests concurrently + go func() { + headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req2, reqID)) + }() + // check response from first request + headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req1, reqID)) +} diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go new file mode 100644 index 0000000000..b7fa1dce26 --- /dev/null +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -0,0 +1,270 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethtest + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" + "github.com/stretchr/testify/assert" +) + +func (c *Conn) statusExchange66(t *utesting.T, chain *Chain) Message { + status := &Status{ + ProtocolVersion: uint32(66), + NetworkID: chain.chainConfig.ChainID.Uint64(), + TD: chain.TD(chain.Len()), + Head: chain.blocks[chain.Len()-1].Hash(), + Genesis: chain.blocks[0].Hash(), + ForkID: chain.ForkID(), + } + return c.statusExchange(t, chain, status) +} + +func (s *Suite) dial66(t *utesting.T) *Conn { + conn, err := s.dial() + if err != nil { + t.Fatalf("could not dial: %v", err) + } + conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) + return conn +} + +func (c *Conn) write66(req eth.Packet, code int) error { + payload, err := rlp.EncodeToBytes(req) + if err != nil { + return err + } + _, err = c.Conn.Write(uint64(code), payload) + return err +} + +func (c *Conn) read66() (uint64, Message) { + code, rawData, _, err := c.Conn.Read() + if err != nil { + return 0, errorf("could not read from connection: %v", err) + } + + var msg Message + + switch int(code) { + case (Hello{}).Code(): + msg = new(Hello) + + case (Ping{}).Code(): + msg = new(Ping) + case (Pong{}).Code(): + msg = new(Pong) + case (Disconnect{}).Code(): + msg = new(Disconnect) + case (Status{}).Code(): + msg = new(Status) + case (GetBlockHeaders{}).Code(): + ethMsg := new(eth.GetBlockHeadersPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) + case (BlockHeaders{}).Code(): + ethMsg := new(eth.BlockHeadersPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) + case (GetBlockBodies{}).Code(): + ethMsg := new(eth.GetBlockBodiesPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) + case (BlockBodies{}).Code(): + ethMsg := new(eth.BlockBodiesPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) + case (NewBlock{}).Code(): + msg = new(NewBlock) + case (NewBlockHashes{}).Code(): + msg = new(NewBlockHashes) + case (Transactions{}).Code(): + msg = new(Transactions) + case (NewPooledTransactionHashes{}).Code(): + msg = new(NewPooledTransactionHashes) + default: + msg = errorf("invalid message code: %d", code) + } + + if msg != nil { + if err := rlp.DecodeBytes(rawData, msg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return 0, msg + } + return 0, errorf("invalid message: %s", string(rawData)) +} + +// ReadAndServe serves GetBlockHeaders requests while waiting +// on another message from the node. +func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { + start := time.Now() + for time.Since(start) < timeout { + timeout := time.Now().Add(10 * time.Second) + c.SetReadDeadline(timeout) + + reqID, msg := c.read66() + + switch msg := msg.(type) { + case *Ping: + c.Write(&Pong{}) + case *GetBlockHeaders: + headers, err := chain.GetHeaders(*msg) + if err != nil { + return 0, errorf("could not get headers for inbound header request: %v", err) + } + + if err := c.Write(headers); err != nil { + return 0, errorf("could not write to connection: %v", err) + } + default: + return reqID, msg + } + } + return 0, errorf("no message received within %v", timeout) +} + +func (s *Suite) setupConnection66(t *utesting.T) *Conn { + // create conn + sendConn := s.dial66(t) + sendConn.handshake(t) + sendConn.statusExchange66(t, s.chain) + return sendConn +} + +func (s *Suite) testAnnounce66(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) { + // Announce the block. + if err := sendConn.Write(blockAnnouncement); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + s.waitAnnounce66(t, receiveConn, blockAnnouncement) +} + +func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) { + timeout := 20 * time.Second + _, msg := conn.readAndServe66(s.chain, timeout) + switch msg := msg.(type) { + case *NewBlock: + t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block)) + assert.Equal(t, + blockAnnouncement.Block.Header(), msg.Block.Header(), + "wrong block header in announcement", + ) + assert.Equal(t, + blockAnnouncement.TD, msg.TD, + "wrong TD in announcement", + ) + case *NewBlockHashes: + blockHashes := *msg + t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes)) + assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash, + "wrong block hash in announcement", + ) + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } +} + +// waitForBlock66 waits for confirmation from the client that it has +// imported the given block. +func (c *Conn) waitForBlock66(block *types.Block) error { + defer c.SetReadDeadline(time.Time{}) + + timeout := time.Now().Add(20 * time.Second) + c.SetReadDeadline(timeout) + for { + req := eth.GetBlockHeadersPacket66{ + RequestId: 54, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Hash: block.Hash(), + }, + Amount: 1, + }, + } + if err := c.write66(req, GetBlockHeaders{}.Code()); err != nil { + return err + } + + reqID, msg := c.read66() + // check message + switch msg := msg.(type) { + case BlockHeaders: + // check request ID + if reqID != req.RequestId { + return fmt.Errorf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID) + } + if len(msg) > 0 { + return nil + } + time.Sleep(100 * time.Millisecond) + default: + return fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) + } + } +} + +func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) { + sendConn := s.setupConnection66(t) + sendSuccessfulTxWithConn(t, s, tx, sendConn) +} + +func sendFailingTx66(t *utesting.T, s *Suite, tx *types.Transaction) { + sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t) + sendFailingTxWithConns(t, s, tx, sendConn, recvConn) +} + +func (s *Suite) getBlockHeaders66(t *utesting.T, conn *Conn, req eth.Packet, expectedID uint64) BlockHeaders { + if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + // check block headers response + reqID, msg := conn.readAndServe66(s.chain, timeout) + + switch msg := msg.(type) { + case BlockHeaders: + if reqID != expectedID { + t.Fatalf("request ID mismatch: wanted %d, got %d", expectedID, reqID) + } + return msg + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + return nil + } +} + +func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) { + for _, header := range headers { + num := header.Number.Uint64() + t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash())) + assert.Equal(t, chain.blocks[int(num)].Header(), header) + } +} diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index edf7bb7e31..48010b90dd 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -53,29 +53,47 @@ type Suite struct { // NewSuite creates and returns a new eth-test suite that can // be used to test the given node against the given blockchain // data. -func NewSuite(dest *enode.Node, chainfile string, genesisfile string) *Suite { +func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, error) { chain, err := loadChain(chainfile, genesisfile) if err != nil { - panic(err) + return nil, err } return &Suite{ Dest: dest, chain: chain.Shorten(1000), fullChain: chain, - } + }, nil } -func (s *Suite) AllTests() []utesting.Test { +func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ + // status {Name: "Status", Fn: s.TestStatus}, + {Name: "Status_66", Fn: s.TestStatus_66}, + // get block headers {Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders}, - {Name: "Broadcast", Fn: s.TestBroadcast}, + {Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, + {Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66}, + {Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66}, + {Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66}, + // get block bodies {Name: "GetBlockBodies", Fn: s.TestGetBlockBodies}, + {Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, + // broadcast + {Name: "Broadcast", Fn: s.TestBroadcast}, + {Name: "Broadcast_66", Fn: s.TestBroadcast_66}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, + // malicious handshakes + status {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, + {Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66}, + {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus}, + // test transactions {Name: "TestTransactions", Fn: s.TestTransaction}, + {Name: "TestTransactions_66", Fn: s.TestTransaction_66}, {Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx}, + {Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66}, } } @@ -161,7 +179,7 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { headers := *msg for _, header := range headers { num := header.Number.Uint64() - t.Logf("received header (%d): %s", num, pretty.Sdump(header)) + t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash())) assert.Equal(t, s.chain.blocks[int(num)].Header(), header) } default: @@ -386,20 +404,23 @@ func (s *Suite) setupConnection(t *utesting.T) *Conn { // returning the created Conn if successful. func (s *Suite) dial() (*Conn, error) { var conn Conn - + // dial fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", s.Dest.IP(), s.Dest.TCP())) if err != nil { return nil, err } conn.Conn = rlpx.NewConn(fd, s.Dest.Pubkey()) - // do encHandshake conn.ourKey, _ = crypto.GenerateKey() _, err = conn.Handshake(conn.ourKey) if err != nil { return nil, err } - + // set default p2p capabilities + conn.caps = []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + } return &conn, nil } diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index effcc3af29..21aa221e8b 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -30,6 +30,10 @@ var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c666 func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) { sendConn := s.setupConnection(t) + sendSuccessfulTxWithConn(t, s, tx, sendConn) +} + +func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, sendConn *Conn) { t.Logf("sending tx: %v %v %v\n", tx.Hash().String(), tx.GasPrice(), tx.Gas()) // Send the transaction if err := sendConn.Write(&Transactions{tx}); err != nil { @@ -41,20 +45,21 @@ func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) { switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { case *Transactions: recTxs := *msg - if len(recTxs) < 1 { - t.Fatalf("received transactions do not match send: %v", recTxs) - } - if tx.Hash() != recTxs[len(recTxs)-1].Hash() { - t.Fatalf("received transactions do not match send: got %v want %v", recTxs, tx) + for _, gotTx := range recTxs { + if gotTx.Hash() == tx.Hash() { + // Ok + return + } } + t.Fatalf("missing transaction: got %v missing %v", recTxs, tx.Hash()) case *NewPooledTransactionHashes: txHashes := *msg - if len(txHashes) < 1 { - t.Fatalf("received transactions do not match send: %v", txHashes) - } - if tx.Hash() != txHashes[len(txHashes)-1] { - t.Fatalf("wrong announcement received, wanted %v got %v", tx, txHashes) + for _, gotHash := range txHashes { + if gotHash == tx.Hash() { + return + } } + t.Fatalf("missing transaction announcement: got %v missing %v", txHashes, tx.Hash()) default: t.Fatalf("unexpected message in sendSuccessfulTx: %s", pretty.Sdump(msg)) } @@ -62,6 +67,10 @@ func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) { func sendFailingTx(t *utesting.T, s *Suite, tx *types.Transaction) { sendConn, recvConn := s.setupConnection(t), s.setupConnection(t) + sendFailingTxWithConns(t, s, tx, sendConn, recvConn) +} + +func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, sendConn, recvConn *Conn) { // Wait for a transaction announcement switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { case *NewPooledTransactionHashes: diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index b901f50700..734adff366 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -125,6 +125,7 @@ type Conn struct { *rlpx.Conn ourKey *ecdsa.PrivateKey ethProtocolVersion uint + caps []p2p.Cap } func (c *Conn) Read() Message { @@ -221,11 +222,8 @@ func (c *Conn) handshake(t *utesting.T) Message { pub0 := crypto.FromECDSAPub(&c.ourKey.PublicKey)[1:] ourHandshake := &Hello{ Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - }, - ID: pub0, + Caps: c.caps, + ID: pub0, } if err := c.Write(ourHandshake); err != nil { t.Fatalf("could not write to connection: %v", err) diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index d90eb4687c..ac92818aa4 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -94,6 +94,9 @@ func rlpxEthTest(ctx *cli.Context) error { if ctx.NArg() < 3 { exit("missing path to chain.rlp as command-line argument") } - suite := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2]) - return runTests(ctx, suite.AllTests()) + suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2]) + if err != nil { + exit(err) + } + return runTests(ctx, suite.EthTests()) } diff --git a/go.mod b/go.mod index cddce4e0d8..a1099c52cc 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 From dc109cce26da8a93f74a998f9dd7fc2ac0ab98d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Thu, 25 Feb 2021 21:08:34 +0100 Subject: [PATCH 164/709] les: move server pool to les/vflux/client (#22377) * les: move serverPool to les/vflux/client * les: add metrics * les: moved ValueTracker inside ServerPool * les: protect against node registration before server pool is started * les/vflux/client: fixed tests * les: make peer registration safe --- internal/web3ext/web3ext.go | 14 +- les/client.go | 38 +---- les/client_handler.go | 14 ++ les/peer.go | 9 +- les/vflux/client/queueiterator_test.go | 11 -- les/{ => vflux/client}/serverpool.go | 173 +++++++++++++--------- les/{ => vflux/client}/serverpool_test.go | 64 ++++---- les/vflux/client/valuetracker.go | 77 +++++----- les/vflux/client/valuetracker_test.go | 4 +- 9 files changed, 206 insertions(+), 198 deletions(-) rename les/{ => vflux/client}/serverpool.go (76%) rename les/{ => vflux/client}/serverpool_test.go (86%) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 77954bbbf0..6fcf4b8380 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -33,7 +33,7 @@ var Modules = map[string]string{ "swarmfs": SwarmfsJs, "txpool": TxpoolJs, "les": LESJs, - "lespay": LESPayJs, + "vflux": VfluxJs, } const ChequebookJs = ` @@ -877,24 +877,24 @@ web3._extend({ }); ` -const LESPayJs = ` +const VfluxJs = ` web3._extend({ - property: 'lespay', + property: 'vflux', methods: [ new web3._extend.Method({ name: 'distribution', - call: 'lespay_distribution', + call: 'vflux_distribution', params: 2 }), new web3._extend.Method({ name: 'timeout', - call: 'lespay_timeout', + call: 'vflux_timeout', params: 2 }), new web3._extend.Method({ name: 'value', - call: 'lespay_value', + call: 'vflux_value', params: 2 }), ], @@ -902,7 +902,7 @@ web3._extend({ [ new web3._extend.Property({ name: 'requestStats', - getter: 'lespay_requestStats' + getter: 'vflux_requestStats' }), ] }); diff --git a/les/client.go b/les/client.go index 053118df5a..e20519fd91 100644 --- a/les/client.go +++ b/les/client.go @@ -57,8 +57,7 @@ type LightEthereum struct { handler *clientHandler txPool *light.TxPool blockchain *light.LightChain - serverPool *serverPool - valueTracker *vfc.ValueTracker + serverPool *vfc.ServerPool dialCandidates enode.Iterator pruner *pruner @@ -109,17 +108,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), - valueTracker: vfc.NewValueTracker(lesDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), p2pServer: stack.Server(), p2pConfig: &stack.Config().P2P, } - peers.subscribe((*vtSubscription)(leth.valueTracker)) - leth.serverPool = newServerPool(lesDb, []byte("serverpool:"), leth.valueTracker, time.Second, nil, &mclock.System{}, config.UltraLightServers) - peers.subscribe(leth.serverPool) - leth.dialCandidates = leth.serverPool.dialIterator + leth.serverPool, leth.dialCandidates = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, nil, &mclock.System{}, config.UltraLightServers, requestList) + leth.serverPool.AddMetrics(suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge, sessionValueMeter, serverDialedMeter) - leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.getTimeout) + leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.GetTimeout) leth.relay = newLesTxRelay(peers, leth.retriever) leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.peers, leth.retriever) @@ -193,23 +189,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { return leth, nil } -// vtSubscription implements serverPeerSubscriber -type vtSubscription vfc.ValueTracker - -// registerPeer implements serverPeerSubscriber -func (v *vtSubscription) registerPeer(p *serverPeer) { - vt := (*vfc.ValueTracker)(v) - p.setValueTracker(vt, vt.Register(p.ID())) - p.updateVtParams() -} - -// unregisterPeer implements serverPeerSubscriber -func (v *vtSubscription) unregisterPeer(p *serverPeer) { - vt := (*vfc.ValueTracker)(v) - vt.Unregister(p.ID()) - p.setValueTracker(nil, nil) -} - type LightDummyAPI struct{} // Etherbase is the address that mining rewards will be send to @@ -266,7 +245,7 @@ func (s *LightEthereum) APIs() []rpc.API { }, { Namespace: "vflux", Version: "1.0", - Service: vfc.NewPrivateClientAPI(s.valueTracker), + Service: s.serverPool.API(), Public: false, }, }...) @@ -302,8 +281,8 @@ func (s *LightEthereum) Start() error { if err != nil { return err } - s.serverPool.addSource(discovery) - s.serverPool.start() + s.serverPool.AddSource(discovery) + s.serverPool.Start() // Start bloom request workers. s.wg.Add(bloomServiceThreads) s.startBloomHandlers(params.BloomBitsBlocksClient) @@ -316,8 +295,7 @@ func (s *LightEthereum) Start() error { // Ethereum protocol. func (s *LightEthereum) Stop() error { close(s.closeCh) - s.serverPool.stop() - s.valueTracker.Stop() + s.serverPool.Stop() s.peers.close() s.reqDist.close() s.odr.Stop() diff --git a/les/client_handler.go b/les/client_handler.go index 6cd786cda0..f8e9edc9fe 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -114,11 +114,25 @@ func (h *clientHandler) handle(p *serverPeer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } + // Register peer with the server pool + if h.backend.serverPool != nil { + if nvt, err := h.backend.serverPool.RegisterNode(p.Node()); err == nil { + p.setValueTracker(nvt) + p.updateVtParams() + defer func() { + p.setValueTracker(nil) + h.backend.serverPool.UnregisterNode(p.Node()) + }() + } else { + return err + } + } // Register the peer locally if err := h.backend.peers.register(p); err != nil { p.Log().Error("Light Ethereum peer registration failed", "err", err) return err } + serverConnectionGauge.Update(int64(h.backend.peers.len())) connectedAt := mclock.Now() diff --git a/les/peer.go b/les/peer.go index 0361167ee1..78019b1d87 100644 --- a/les/peer.go +++ b/les/peer.go @@ -349,7 +349,6 @@ type serverPeer struct { fcServer *flowcontrol.ServerNode // Client side mirror token bucket. vtLock sync.Mutex - valueTracker *vfc.ValueTracker nodeValueTracker *vfc.NodeValueTracker sentReqs map[uint64]sentReqEntry @@ -676,9 +675,8 @@ func (p *serverPeer) Handshake(genesis common.Hash, forkid forkid.ID, forkFilter // setValueTracker sets the value tracker references for connected servers. Note that the // references should be removed upon disconnection by setValueTracker(nil, nil). -func (p *serverPeer) setValueTracker(vt *vfc.ValueTracker, nvt *vfc.NodeValueTracker) { +func (p *serverPeer) setValueTracker(nvt *vfc.NodeValueTracker) { p.vtLock.Lock() - p.valueTracker = vt p.nodeValueTracker = nvt if nvt != nil { p.sentReqs = make(map[uint64]sentReqEntry) @@ -705,7 +703,7 @@ func (p *serverPeer) updateVtParams() { } } } - p.valueTracker.UpdateCosts(p.nodeValueTracker, reqCosts) + p.nodeValueTracker.UpdateCosts(reqCosts) } // sentReqEntry remembers sent requests and their sending times @@ -732,7 +730,6 @@ func (p *serverPeer) answeredRequest(id uint64) { } e, ok := p.sentReqs[id] delete(p.sentReqs, id) - vt := p.valueTracker nvt := p.nodeValueTracker p.vtLock.Unlock() if !ok { @@ -752,7 +749,7 @@ func (p *serverPeer) answeredRequest(id uint64) { vtReqs[1] = vfc.ServedRequest{ReqType: uint32(m.rest), Amount: e.amount - 1} } dt := time.Duration(mclock.Now() - e.at) - vt.Served(nvt, vtReqs[:reqCount], dt) + nvt.Served(vtReqs[:reqCount], dt) } // clientPeer represents each node to which the les server is connected. diff --git a/les/vflux/client/queueiterator_test.go b/les/vflux/client/queueiterator_test.go index a74301c7d3..400d978e19 100644 --- a/les/vflux/client/queueiterator_test.go +++ b/les/vflux/client/queueiterator_test.go @@ -26,17 +26,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/nodestate" ) -func testNodeID(i int) enode.ID { - return enode.ID{42, byte(i % 256), byte(i / 256)} -} - -func testNodeIndex(id enode.ID) int { - if id[0] != 42 { - return -1 - } - return int(id[1]) + int(id[2])*256 -} - func testNode(i int) *enode.Node { return enode.SignNull(new(enr.Record), testNodeID(i)) } diff --git a/les/serverpool.go b/les/vflux/client/serverpool.go similarity index 76% rename from les/serverpool.go rename to les/vflux/client/serverpool.go index 977579988e..95f7246091 100644 --- a/les/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package les +package client import ( "errors" @@ -27,8 +27,8 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/utils" - vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nodestate" @@ -50,31 +50,34 @@ const ( maxQueryFails = 100 // number of consecutive UDP query failures before we print a warning ) -// serverPool provides a node iterator for dial candidates. The output is a mix of newly discovered +// ServerPool provides a node iterator for dial candidates. The output is a mix of newly discovered // nodes, a weighted random selection of known (previously valuable) nodes and trusted/paid nodes. -type serverPool struct { +type ServerPool struct { clock mclock.Clock unixTime func() int64 db ethdb.KeyValueStore - ns *nodestate.NodeStateMachine - vt *vfc.ValueTracker - mixer *enode.FairMix - mixSources []enode.Iterator - dialIterator enode.Iterator - validSchemes enr.IdentityScheme - trustedURLs []string - fillSet *vfc.FillSet - queryFails uint32 + ns *nodestate.NodeStateMachine + vt *ValueTracker + mixer *enode.FairMix + mixSources []enode.Iterator + dialIterator enode.Iterator + validSchemes enr.IdentityScheme + trustedURLs []string + fillSet *FillSet + started, queryFails uint32 timeoutLock sync.RWMutex timeout time.Duration - timeWeights vfc.ResponseTimeWeights + timeWeights ResponseTimeWeights timeoutRefreshed mclock.AbsTime + + suggestedTimeoutGauge, totalValueGauge metrics.Gauge + sessionValueMeter metrics.Meter } // nodeHistory keeps track of dial costs which determine node weight together with the -// service value calculated by vfc.ValueTracker. +// service value calculated by ValueTracker. type nodeHistory struct { dialCost utils.ExpiredValue redialWaitStart, redialWaitEnd int64 // unix time (seconds) @@ -91,18 +94,18 @@ type nodeHistoryEnc struct { type queryFunc func(*enode.Node) int var ( - serverPoolSetup = &nodestate.Setup{Version: 1} - sfHasValue = serverPoolSetup.NewPersistentFlag("hasValue") - sfQueried = serverPoolSetup.NewFlag("queried") - sfCanDial = serverPoolSetup.NewFlag("canDial") - sfDialing = serverPoolSetup.NewFlag("dialed") - sfWaitDialTimeout = serverPoolSetup.NewFlag("dialTimeout") - sfConnected = serverPoolSetup.NewFlag("connected") - sfRedialWait = serverPoolSetup.NewFlag("redialWait") - sfAlwaysConnect = serverPoolSetup.NewFlag("alwaysConnect") + clientSetup = &nodestate.Setup{Version: 1} + sfHasValue = clientSetup.NewPersistentFlag("hasValue") + sfQueried = clientSetup.NewFlag("queried") + sfCanDial = clientSetup.NewFlag("canDial") + sfDialing = clientSetup.NewFlag("dialed") + sfWaitDialTimeout = clientSetup.NewFlag("dialTimeout") + sfConnected = clientSetup.NewFlag("connected") + sfRedialWait = clientSetup.NewFlag("redialWait") + sfAlwaysConnect = clientSetup.NewFlag("alwaysConnect") sfDisableSelection = nodestate.MergeFlags(sfQueried, sfCanDial, sfDialing, sfConnected, sfRedialWait) - sfiNodeHistory = serverPoolSetup.NewPersistentField("nodeHistory", reflect.TypeOf(nodeHistory{}), + sfiNodeHistory = clientSetup.NewPersistentField("nodeHistory", reflect.TypeOf(nodeHistory{}), func(field interface{}) ([]byte, error) { if n, ok := field.(nodeHistory); ok { ne := nodeHistoryEnc{ @@ -126,25 +129,25 @@ var ( return n, err }, ) - sfiNodeWeight = serverPoolSetup.NewField("nodeWeight", reflect.TypeOf(uint64(0))) - sfiConnectedStats = serverPoolSetup.NewField("connectedStats", reflect.TypeOf(vfc.ResponseTimeStats{})) + sfiNodeWeight = clientSetup.NewField("nodeWeight", reflect.TypeOf(uint64(0))) + sfiConnectedStats = clientSetup.NewField("connectedStats", reflect.TypeOf(ResponseTimeStats{})) ) // newServerPool creates a new server pool -func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *vfc.ValueTracker, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string) *serverPool { - s := &serverPool{ +func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string, requestList []RequestInfo) (*ServerPool, enode.Iterator) { + s := &ServerPool{ db: db, clock: clock, unixTime: func() int64 { return time.Now().Unix() }, validSchemes: enode.ValidSchemes, trustedURLs: trustedURLs, - vt: vt, - ns: nodestate.NewNodeStateMachine(db, []byte(string(dbKey)+"ns:"), clock, serverPoolSetup), + vt: NewValueTracker(db, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), + ns: nodestate.NewNodeStateMachine(db, []byte(string(dbKey)+"ns:"), clock, clientSetup), } s.recalTimeout() s.mixer = enode.NewFairMix(mixTimeout) - knownSelector := vfc.NewWrsIterator(s.ns, sfHasValue, sfDisableSelection, sfiNodeWeight) - alwaysConnect := vfc.NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) + knownSelector := NewWrsIterator(s.ns, sfHasValue, sfDisableSelection, sfiNodeWeight) + alwaysConnect := NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) s.mixSources = append(s.mixSources, knownSelector) s.mixSources = append(s.mixSources, alwaysConnect) @@ -166,14 +169,30 @@ func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *vfc.ValueTracker, m } }) - s.ns.AddLogMetrics(sfHasValue, sfDisableSelection, "selectable", nil, nil, serverSelectableGauge) - s.ns.AddLogMetrics(sfDialing, nodestate.Flags{}, "dialed", serverDialedMeter, nil, nil) - s.ns.AddLogMetrics(sfConnected, nodestate.Flags{}, "connected", nil, nil, serverConnectedGauge) - return s + return s, s.dialIterator +} + +// AddMetrics adds metrics to the server pool. Should be called before Start(). +func (s *ServerPool) AddMetrics( + suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge metrics.Gauge, + sessionValueMeter, serverDialedMeter metrics.Meter) { + + s.suggestedTimeoutGauge = suggestedTimeoutGauge + s.totalValueGauge = totalValueGauge + s.sessionValueMeter = sessionValueMeter + if serverSelectableGauge != nil { + s.ns.AddLogMetrics(sfHasValue, sfDisableSelection, "selectable", nil, nil, serverSelectableGauge) + } + if serverDialedMeter != nil { + s.ns.AddLogMetrics(sfDialing, nodestate.Flags{}, "dialed", serverDialedMeter, nil, nil) + } + if serverConnectedGauge != nil { + s.ns.AddLogMetrics(sfConnected, nodestate.Flags{}, "connected", nil, nil, serverConnectedGauge) + } } -// addSource adds a node discovery source to the server pool (should be called before start) -func (s *serverPool) addSource(source enode.Iterator) { +// AddSource adds a node discovery source to the server pool (should be called before start) +func (s *ServerPool) AddSource(source enode.Iterator) { if source != nil { s.mixSources = append(s.mixSources, source) } @@ -182,8 +201,8 @@ func (s *serverPool) addSource(source enode.Iterator) { // addPreNegFilter installs a node filter mechanism that performs a pre-negotiation query. // Nodes that are filtered out and does not appear on the output iterator are put back // into redialWait state. -func (s *serverPool) addPreNegFilter(input enode.Iterator, query queryFunc) enode.Iterator { - s.fillSet = vfc.NewFillSet(s.ns, input, sfQueried) +func (s *ServerPool) addPreNegFilter(input enode.Iterator, query queryFunc) enode.Iterator { + s.fillSet = NewFillSet(s.ns, input, sfQueried) s.ns.SubscribeState(sfQueried, func(n *enode.Node, oldState, newState nodestate.Flags) { if newState.Equals(sfQueried) { fails := atomic.LoadUint32(&s.queryFails) @@ -221,7 +240,7 @@ func (s *serverPool) addPreNegFilter(input enode.Iterator, query queryFunc) enod }() } }) - return vfc.NewQueueIterator(s.ns, sfCanDial, nodestate.Flags{}, false, func(waiting bool) { + return NewQueueIterator(s.ns, sfCanDial, nodestate.Flags{}, false, func(waiting bool) { if waiting { s.fillSet.SetTarget(preNegLimit) } else { @@ -231,7 +250,7 @@ func (s *serverPool) addPreNegFilter(input enode.Iterator, query queryFunc) enod } // start starts the server pool. Note that NodeStateMachine should be started first. -func (s *serverPool) start() { +func (s *ServerPool) Start() { s.ns.Start() for _, iter := range s.mixSources { // add sources to mixer at startup because the mixer instantly tries to read them @@ -261,10 +280,11 @@ func (s *serverPool) start() { } }) }) + atomic.StoreUint32(&s.started, 1) } // stop stops the server pool -func (s *serverPool) stop() { +func (s *ServerPool) Stop() { s.dialIterator.Close() if s.fillSet != nil { s.fillSet.Close() @@ -276,32 +296,34 @@ func (s *serverPool) stop() { }) }) s.ns.Stop() + s.vt.Stop() } // registerPeer implements serverPeerSubscriber -func (s *serverPool) registerPeer(p *serverPeer) { - s.ns.SetState(p.Node(), sfConnected, sfDialing.Or(sfWaitDialTimeout), 0) - nvt := s.vt.Register(p.ID()) - s.ns.SetField(p.Node(), sfiConnectedStats, nvt.RtStats()) - p.setValueTracker(s.vt, nvt) - p.updateVtParams() +func (s *ServerPool) RegisterNode(node *enode.Node) (*NodeValueTracker, error) { + if atomic.LoadUint32(&s.started) == 0 { + return nil, errors.New("server pool not started yet") + } + s.ns.SetState(node, sfConnected, sfDialing.Or(sfWaitDialTimeout), 0) + nvt := s.vt.Register(node.ID()) + s.ns.SetField(node, sfiConnectedStats, nvt.RtStats()) + return nvt, nil } // unregisterPeer implements serverPeerSubscriber -func (s *serverPool) unregisterPeer(p *serverPeer) { +func (s *ServerPool) UnregisterNode(node *enode.Node) { s.ns.Operation(func() { - s.setRedialWait(p.Node(), dialCost, dialWaitStep) - s.ns.SetStateSub(p.Node(), nodestate.Flags{}, sfConnected, 0) - s.ns.SetFieldSub(p.Node(), sfiConnectedStats, nil) + s.setRedialWait(node, dialCost, dialWaitStep) + s.ns.SetStateSub(node, nodestate.Flags{}, sfConnected, 0) + s.ns.SetFieldSub(node, sfiConnectedStats, nil) }) - s.vt.Unregister(p.ID()) - p.setValueTracker(nil, nil) + s.vt.Unregister(node.ID()) } // recalTimeout calculates the current recommended timeout. This value is used by // the client as a "soft timeout" value. It also affects the service value calculation // of individual nodes. -func (s *serverPool) recalTimeout() { +func (s *ServerPool) recalTimeout() { // Use cached result if possible, avoid recalculating too frequently. s.timeoutLock.RLock() refreshed := s.timeoutRefreshed @@ -330,17 +352,21 @@ func (s *serverPool) recalTimeout() { s.timeoutLock.Lock() if s.timeout != timeout { s.timeout = timeout - s.timeWeights = vfc.TimeoutWeights(s.timeout) + s.timeWeights = TimeoutWeights(s.timeout) - suggestedTimeoutGauge.Update(int64(s.timeout / time.Millisecond)) - totalValueGauge.Update(int64(rts.Value(s.timeWeights, s.vt.StatsExpFactor()))) + if s.suggestedTimeoutGauge != nil { + s.suggestedTimeoutGauge.Update(int64(s.timeout / time.Millisecond)) + } + if s.totalValueGauge != nil { + s.totalValueGauge.Update(int64(rts.Value(s.timeWeights, s.vt.StatsExpFactor()))) + } } s.timeoutRefreshed = now s.timeoutLock.Unlock() } -// getTimeout returns the recommended request timeout. -func (s *serverPool) getTimeout() time.Duration { +// GetTimeout returns the recommended request timeout. +func (s *ServerPool) GetTimeout() time.Duration { s.recalTimeout() s.timeoutLock.RLock() defer s.timeoutLock.RUnlock() @@ -349,7 +375,7 @@ func (s *serverPool) getTimeout() time.Duration { // getTimeoutAndWeight returns the recommended request timeout as well as the // response time weight which is necessary to calculate service value. -func (s *serverPool) getTimeoutAndWeight() (time.Duration, vfc.ResponseTimeWeights) { +func (s *ServerPool) getTimeoutAndWeight() (time.Duration, ResponseTimeWeights) { s.recalTimeout() s.timeoutLock.RLock() defer s.timeoutLock.RUnlock() @@ -358,7 +384,7 @@ func (s *serverPool) getTimeoutAndWeight() (time.Duration, vfc.ResponseTimeWeigh // addDialCost adds the given amount of dial cost to the node history and returns the current // amount of total dial cost -func (s *serverPool) addDialCost(n *nodeHistory, amount int64) uint64 { +func (s *ServerPool) addDialCost(n *nodeHistory, amount int64) uint64 { logOffset := s.vt.StatsExpirer().LogOffset(s.clock.Now()) if amount > 0 { n.dialCost.Add(amount, logOffset) @@ -371,7 +397,7 @@ func (s *serverPool) addDialCost(n *nodeHistory, amount int64) uint64 { } // serviceValue returns the service value accumulated in this session and in total -func (s *serverPool) serviceValue(node *enode.Node) (sessionValue, totalValue float64) { +func (s *ServerPool) serviceValue(node *enode.Node) (sessionValue, totalValue float64) { nvt := s.vt.GetNode(node.ID()) if nvt == nil { return 0, 0 @@ -381,11 +407,13 @@ func (s *serverPool) serviceValue(node *enode.Node) (sessionValue, totalValue fl expFactor := s.vt.StatsExpFactor() totalValue = currentStats.Value(timeWeights, expFactor) - if connStats, ok := s.ns.GetField(node, sfiConnectedStats).(vfc.ResponseTimeStats); ok { + if connStats, ok := s.ns.GetField(node, sfiConnectedStats).(ResponseTimeStats); ok { diff := currentStats diff.SubStats(&connStats) sessionValue = diff.Value(timeWeights, expFactor) - sessionValueMeter.Mark(int64(sessionValue)) + if s.sessionValueMeter != nil { + s.sessionValueMeter.Mark(int64(sessionValue)) + } } return } @@ -393,7 +421,7 @@ func (s *serverPool) serviceValue(node *enode.Node) (sessionValue, totalValue fl // updateWeight calculates the node weight and updates the nodeWeight field and the // hasValue flag. It also saves the node state if necessary. // Note: this function should run inside a NodeStateMachine operation -func (s *serverPool) updateWeight(node *enode.Node, totalValue float64, totalDialCost uint64) { +func (s *ServerPool) updateWeight(node *enode.Node, totalValue float64, totalDialCost uint64) { weight := uint64(totalValue * nodeWeightMul / float64(totalDialCost)) if weight >= nodeWeightThreshold { s.ns.SetStateSub(node, sfHasValue, nodestate.Flags{}, 0) @@ -415,7 +443,7 @@ func (s *serverPool) updateWeight(node *enode.Node, totalValue float64, totalDia // to the minimum. // Note: node weight is also recalculated and updated by this function. // Note 2: this function should run inside a NodeStateMachine operation -func (s *serverPool) setRedialWait(node *enode.Node, addDialCost int64, waitStep float64) { +func (s *ServerPool) setRedialWait(node *enode.Node, addDialCost int64, waitStep float64) { n, _ := s.ns.GetField(node, sfiNodeHistory).(nodeHistory) sessionValue, totalValue := s.serviceValue(node) totalDialCost := s.addDialCost(&n, addDialCost) @@ -481,9 +509,14 @@ func (s *serverPool) setRedialWait(node *enode.Node, addDialCost int64, waitStep // This function should be called during startup and shutdown only, otherwise setRedialWait // will keep the weights updated as the underlying statistics are adjusted. // Note: this function should run inside a NodeStateMachine operation -func (s *serverPool) calculateWeight(node *enode.Node) { +func (s *ServerPool) calculateWeight(node *enode.Node) { n, _ := s.ns.GetField(node, sfiNodeHistory).(nodeHistory) _, totalValue := s.serviceValue(node) totalDialCost := s.addDialCost(&n, 0) s.updateWeight(node, totalValue, totalDialCost) } + +// API returns the vflux client API +func (s *ServerPool) API() *PrivateClientAPI { + return NewPrivateClientAPI(s.vt) +} diff --git a/les/serverpool_test.go b/les/vflux/client/serverpool_test.go similarity index 86% rename from les/serverpool_test.go rename to les/vflux/client/serverpool_test.go index 5c8ae56f6c..3af3db95bc 100644 --- a/les/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -14,10 +14,11 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package les +package client import ( "math/rand" + "strconv" "sync/atomic" "testing" "time" @@ -25,8 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" - vfc "github.com/ethereum/go-ethereum/les/vflux/client" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" ) @@ -50,13 +49,13 @@ func testNodeIndex(id enode.ID) int { return int(id[1]) + int(id[2])*256 } -type serverPoolTest struct { +type ServerPoolTest struct { db ethdb.KeyValueStore clock *mclock.Simulated quit chan struct{} preNeg, preNegFail bool - vt *vfc.ValueTracker - sp *serverPool + vt *ValueTracker + sp *ServerPool input enode.Iterator testNodes []spTestNode trusted []string @@ -71,15 +70,15 @@ type spTestNode struct { connectCycles, waitCycles int nextConnCycle, totalConn int connected, service bool - peer *serverPeer + node *enode.Node } -func newServerPoolTest(preNeg, preNegFail bool) *serverPoolTest { +func newServerPoolTest(preNeg, preNegFail bool) *ServerPoolTest { nodes := make([]*enode.Node, spTestNodes) for i := range nodes { nodes[i] = enode.SignNull(&enr.Record{}, testNodeID(i)) } - return &serverPoolTest{ + return &ServerPoolTest{ clock: &mclock.Simulated{}, db: memorydb.New(), input: enode.CycleNodes(nodes), @@ -89,7 +88,7 @@ func newServerPoolTest(preNeg, preNegFail bool) *serverPoolTest { } } -func (s *serverPoolTest) beginWait() { +func (s *ServerPoolTest) beginWait() { // ensure that dialIterator and the maximal number of pre-neg queries are not all stuck in a waiting state for atomic.AddInt32(&s.waitCount, 1) > preNegLimit { atomic.AddInt32(&s.waitCount, -1) @@ -97,16 +96,16 @@ func (s *serverPoolTest) beginWait() { } } -func (s *serverPoolTest) endWait() { +func (s *ServerPoolTest) endWait() { atomic.AddInt32(&s.waitCount, -1) atomic.AddInt32(&s.waitEnded, 1) } -func (s *serverPoolTest) addTrusted(i int) { +func (s *ServerPoolTest) addTrusted(i int) { s.trusted = append(s.trusted, enode.SignNull(&enr.Record{}, testNodeID(i)).String()) } -func (s *serverPoolTest) start() { +func (s *ServerPoolTest) start() { var testQuery queryFunc if s.preNeg { testQuery = func(node *enode.Node) int { @@ -144,13 +143,17 @@ func (s *serverPoolTest) start() { } } - s.vt = vfc.NewValueTracker(s.db, s.clock, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)) - s.sp = newServerPool(s.db, []byte("serverpool:"), s.vt, 0, testQuery, s.clock, s.trusted) - s.sp.addSource(s.input) + requestList := make([]RequestInfo, testReqTypes) + for i := range requestList { + requestList[i] = RequestInfo{Name: "testreq" + strconv.Itoa(i), InitAmount: 1, InitValue: 1} + } + + s.sp, _ = NewServerPool(s.db, []byte("sp:"), 0, testQuery, s.clock, s.trusted, requestList) + s.sp.AddSource(s.input) s.sp.validSchemes = enode.ValidSchemesForTesting s.sp.unixTime = func() int64 { return int64(s.clock.Now()) / int64(time.Second) } s.disconnect = make(map[int][]int) - s.sp.start() + s.sp.Start() s.quit = make(chan struct{}) go func() { last := int32(-1) @@ -170,31 +173,30 @@ func (s *serverPoolTest) start() { }() } -func (s *serverPoolTest) stop() { +func (s *ServerPoolTest) stop() { close(s.quit) - s.sp.stop() - s.vt.Stop() + s.sp.Stop() for i := range s.testNodes { n := &s.testNodes[i] if n.connected { n.totalConn += s.cycle } n.connected = false - n.peer = nil + n.node = nil n.nextConnCycle = 0 } s.conn, s.servedConn = 0, 0 } -func (s *serverPoolTest) run() { +func (s *ServerPoolTest) run() { for count := spTestLength; count > 0; count-- { if dcList := s.disconnect[s.cycle]; dcList != nil { for _, idx := range dcList { n := &s.testNodes[idx] - s.sp.unregisterPeer(n.peer) + s.sp.UnregisterNode(n.node) n.totalConn += s.cycle n.connected = false - n.peer = nil + n.node = nil s.conn-- if n.service { s.servedConn-- @@ -221,10 +223,10 @@ func (s *serverPoolTest) run() { n.connected = true dc := s.cycle + n.connectCycles s.disconnect[dc] = append(s.disconnect[dc], idx) - n.peer = &serverPeer{peerCommons: peerCommons{Peer: p2p.NewPeer(id, "", nil)}} - s.sp.registerPeer(n.peer) + n.node = dial + nv, _ := s.sp.RegisterNode(n.node) if n.service { - s.vt.Served(s.vt.GetNode(id), []vfc.ServedRequest{{ReqType: 0, Amount: 100}}, 0) + nv.Served([]ServedRequest{{ReqType: 0, Amount: 100}}, 0) } } } @@ -234,7 +236,7 @@ func (s *serverPoolTest) run() { } } -func (s *serverPoolTest) setNodes(count, conn, wait int, service, trusted bool) (res []int) { +func (s *ServerPoolTest) setNodes(count, conn, wait int, service, trusted bool) (res []int) { for ; count > 0; count-- { idx := rand.Intn(spTestNodes) for s.testNodes[idx].connectCycles != 0 || s.testNodes[idx].connected { @@ -253,11 +255,11 @@ func (s *serverPoolTest) setNodes(count, conn, wait int, service, trusted bool) return } -func (s *serverPoolTest) resetNodes() { +func (s *ServerPoolTest) resetNodes() { for i, n := range s.testNodes { if n.connected { n.totalConn += s.cycle - s.sp.unregisterPeer(n.peer) + s.sp.UnregisterNode(n.node) } s.testNodes[i] = spTestNode{totalConn: n.totalConn} } @@ -266,7 +268,7 @@ func (s *serverPoolTest) resetNodes() { s.trusted = nil } -func (s *serverPoolTest) checkNodes(t *testing.T, nodes []int) { +func (s *ServerPoolTest) checkNodes(t *testing.T, nodes []int) { var sum int for _, idx := range nodes { n := &s.testNodes[idx] diff --git a/les/vflux/client/valuetracker.go b/les/vflux/client/valuetracker.go index 4e67b31d96..f5390d0920 100644 --- a/les/vflux/client/valuetracker.go +++ b/les/vflux/client/valuetracker.go @@ -45,6 +45,7 @@ var ( type NodeValueTracker struct { lock sync.Mutex + vt *ValueTracker rtStats, lastRtStats ResponseTimeStats lastTransfer mclock.AbsTime basket serverBasket @@ -52,15 +53,12 @@ type NodeValueTracker struct { reqValues *[]float64 } -// init initializes a NodeValueTracker. -// Note that the contents of the referenced reqValues slice will not change; a new -// reference is passed if the values are updated by ValueTracker. -func (nv *NodeValueTracker) init(now mclock.AbsTime, reqValues *[]float64) { - reqTypeCount := len(*reqValues) - nv.reqCosts = make([]uint64, reqTypeCount) - nv.lastTransfer = now - nv.reqValues = reqValues - nv.basket.init(reqTypeCount) +// UpdateCosts updates the node value tracker's request cost table +func (nv *NodeValueTracker) UpdateCosts(reqCosts []uint64) { + nv.vt.lock.Lock() + defer nv.vt.lock.Unlock() + + nv.updateCosts(reqCosts, &nv.vt.refBasket.reqValues, nv.vt.refBasket.reqValueFactor(reqCosts)) } // updateCosts updates the request cost table of the server. The request value factor @@ -97,6 +95,28 @@ func (nv *NodeValueTracker) transferStats(now mclock.AbsTime, transferRate float return nv.basket.transfer(-math.Expm1(-transferRate * float64(dt))), recentRtStats } +type ServedRequest struct { + ReqType, Amount uint32 +} + +// Served adds a served request to the node's statistics. An actual request may be composed +// of one or more request types (service vector indices). +func (nv *NodeValueTracker) Served(reqs []ServedRequest, respTime time.Duration) { + nv.vt.statsExpLock.RLock() + expFactor := nv.vt.statsExpFactor + nv.vt.statsExpLock.RUnlock() + + nv.lock.Lock() + defer nv.lock.Unlock() + + var value float64 + for _, r := range reqs { + nv.basket.add(r.ReqType, r.Amount, nv.reqCosts[r.ReqType]*uint64(r.Amount), expFactor) + value += (*nv.reqValues)[r.ReqType] * float64(r.Amount) + } + nv.rtStats.Add(respTime, value, expFactor) +} + // RtStats returns the node's own response time distribution statistics func (nv *NodeValueTracker) RtStats() ResponseTimeStats { nv.lock.Lock() @@ -333,7 +353,12 @@ func (vt *ValueTracker) Register(id enode.ID) *NodeValueTracker { return nil } nv := vt.loadOrNewNode(id) - nv.init(vt.clock.Now(), &vt.refBasket.reqValues) + reqTypeCount := len(vt.refBasket.reqValues) + nv.reqCosts = make([]uint64, reqTypeCount) + nv.lastTransfer = vt.clock.Now() + nv.reqValues = &vt.refBasket.reqValues + nv.basket.init(reqTypeCount) + vt.connected[id] = nv return nv } @@ -364,7 +389,7 @@ func (vt *ValueTracker) loadOrNewNode(id enode.ID) *NodeValueTracker { if nv, ok := vt.connected[id]; ok { return nv } - nv := &NodeValueTracker{lastTransfer: vt.clock.Now()} + nv := &NodeValueTracker{vt: vt, lastTransfer: vt.clock.Now()} enc, err := vt.db.Get(append(vtNodeKey, id[:]...)) if err != nil { return nv @@ -425,14 +450,6 @@ func (vt *ValueTracker) saveNode(id enode.ID, nv *NodeValueTracker) { } } -// UpdateCosts updates the node value tracker's request cost table -func (vt *ValueTracker) UpdateCosts(nv *NodeValueTracker, reqCosts []uint64) { - vt.lock.Lock() - defer vt.lock.Unlock() - - nv.updateCosts(reqCosts, &vt.refBasket.reqValues, vt.refBasket.reqValueFactor(reqCosts)) -} - // RtStats returns the global response time distribution statistics func (vt *ValueTracker) RtStats() ResponseTimeStats { vt.lock.Lock() @@ -464,28 +481,6 @@ func (vt *ValueTracker) periodicUpdate() { vt.saveToDb() } -type ServedRequest struct { - ReqType, Amount uint32 -} - -// Served adds a served request to the node's statistics. An actual request may be composed -// of one or more request types (service vector indices). -func (vt *ValueTracker) Served(nv *NodeValueTracker, reqs []ServedRequest, respTime time.Duration) { - vt.statsExpLock.RLock() - expFactor := vt.statsExpFactor - vt.statsExpLock.RUnlock() - - nv.lock.Lock() - defer nv.lock.Unlock() - - var value float64 - for _, r := range reqs { - nv.basket.add(r.ReqType, r.Amount, nv.reqCosts[r.ReqType]*uint64(r.Amount), expFactor) - value += (*nv.reqValues)[r.ReqType] * float64(r.Amount) - } - nv.rtStats.Add(respTime, value, vt.statsExpFactor) -} - type RequestStatsItem struct { Name string ReqAmount, ReqValue float64 diff --git a/les/vflux/client/valuetracker_test.go b/les/vflux/client/valuetracker_test.go index ad398749e9..87a337be8d 100644 --- a/les/vflux/client/valuetracker_test.go +++ b/les/vflux/client/valuetracker_test.go @@ -64,7 +64,7 @@ func TestValueTracker(t *testing.T) { for j := range costList { costList[j] = uint64(baseCost * relPrices[j]) } - vt.UpdateCosts(nodes[i], costList) + nodes[i].UpdateCosts(costList) } for i := range nodes { nodes[i] = vt.Register(enode.ID{byte(i)}) @@ -77,7 +77,7 @@ func TestValueTracker(t *testing.T) { node := rand.Intn(testNodeCount) respTime := time.Duration((rand.Float64() + 1) * float64(time.Second) * float64(node+1) / testNodeCount) totalAmount[reqType] += uint64(reqAmount) - vt.Served(nodes[node], []ServedRequest{{uint32(reqType), uint32(reqAmount)}}, respTime) + nodes[node].Served([]ServedRequest{{uint32(reqType), uint32(reqAmount)}}, respTime) clock.Run(time.Second) } } else { From 092856267067dd78b527a773f5b240d5c9f5693a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 25 Feb 2021 09:10:30 +0200 Subject: [PATCH 165/709] all: define Berlin hard fork spec --- cmd/geth/config.go | 5 ++- cmd/geth/main.go | 1 + cmd/puppeth/wizard_genesis.go | 4 ++ cmd/utils/flags.go | 4 ++ core/forkid/forkid_test.go | 60 ++++++++++++++------------ core/genesis.go | 7 ++++ core/state_transition.go | 2 +- core/tx_pool.go | 2 +- core/types/transaction_signing.go | 4 +- core/vm/contracts.go | 12 +++--- core/vm/evm.go | 10 ++--- core/vm/interpreter.go | 4 +- core/vm/jump_table.go | 15 ++++--- core/vm/runtime/runtime.go | 7 ++-- eth/backend.go | 2 +- eth/ethconfig/config.go | 3 ++ eth/tracers/api.go | 4 +- les/client.go | 2 +- light/txpool.go | 2 +- params/config.go | 70 ++++++++++++++++++------------- tests/init.go | 2 +- tests/state_test_util.go | 3 +- 22 files changed, 132 insertions(+), 93 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index c71d5eb653..c77c04c252 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -20,6 +20,7 @@ import ( "bufio" "errors" "fmt" + "math/big" "os" "reflect" "unicode" @@ -165,7 +166,9 @@ func checkWhisper(ctx *cli.Context) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - + if ctx.GlobalIsSet(utils.OverrideBerlinFlag.Name) { + cfg.Eth.OverrideBerlin = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideBerlinFlag.Name)) + } backend := utils.RegisterEthService(stack, &cfg.Eth) checkWhisper(ctx) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a3af44f6c4..9afa031584 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -69,6 +69,7 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, + utils.OverrideBerlinFlag, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 78f63e1c7a..4f701fa1c3 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -235,6 +235,10 @@ func (w *wizard) manageGenesis() { fmt.Printf("Which block should Istanbul come into effect? (default = %v)\n", w.conf.Genesis.Config.IstanbulBlock) w.conf.Genesis.Config.IstanbulBlock = w.readDefaultBigInt(w.conf.Genesis.Config.IstanbulBlock) + fmt.Println() + fmt.Printf("Which block should Berlin come into effect? (default = %v)\n", w.conf.Genesis.Config.BerlinBlock) + w.conf.Genesis.Config.BerlinBlock = w.readDefaultBigInt(w.conf.Genesis.Config.BerlinBlock) + fmt.Println() fmt.Printf("Which block should YOLOv3 come into effect? (default = %v)\n", w.conf.Genesis.Config.YoloV3Block) w.conf.Genesis.Config.YoloV3Block = w.readDefaultBigInt(w.conf.Genesis.Config.YoloV3Block) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 57547d0c49..324a6d6a47 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -230,6 +230,10 @@ var ( Usage: "Megabytes of memory allocated to bloom-filter for pruning", Value: 2048, } + OverrideBerlinFlag = cli.Uint64Flag{ + Name: "override.berlin", + Usage: "Manually specify Berlin fork-block, overriding the bundled setting", + } // Light server and client settings LightServeFlag = cli.IntFlag{ Name: "light.serve", diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 888b553475..a20598fa9d 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -43,24 +43,26 @@ func TestCreation(t *testing.T) { params.MainnetChainConfig, params.MainnetGenesisHash, []testcase{ - {0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Unsynced - {1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Last Frontier block - {1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // First Homestead block - {1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // Last Homestead block - {1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // First DAO block - {2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // Last DAO block - {2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // First Tangerine block - {2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // Last Tangerine block - {2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // First Spurious block - {4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // Last Spurious block - {4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // First Byzantium block - {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block - {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block - {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block - {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block - {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block - {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // First Muir Glacier block - {10000000, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // Future Muir Glacier block + {0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Unsynced + {1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Last Frontier block + {1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // First Homestead block + {1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // Last Homestead block + {1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // First DAO block + {2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // Last DAO block + {2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // First Tangerine block + {2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // Last Tangerine block + {2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // First Spurious block + {4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // Last Spurious block + {4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // First Byzantium block + {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block + {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block + {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block + {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block + {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block + {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // First Muir Glacier block + {12243999, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // Last Muir Glacier block + {12244000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 0}}, // First Berlin block + {20000000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 0}}, // Future Berlin block }, }, // Ropsten test cases @@ -80,8 +82,10 @@ func TestCreation(t *testing.T) { {6485845, ID{Hash: checksumToBytes(0xd6e2149b), Next: 6485846}}, // Last Petersburg block {6485846, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // First Istanbul block {7117116, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // Last Istanbul block - {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // First Muir Glacier block - {7500000, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // Future + {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 9812189}}, // First Muir Glacier block + {9812188, ID{Hash: checksumToBytes(0x6727ef90), Next: 9812189}}, // Last Muir Glacier block + {9812189, ID{Hash: checksumToBytes(0xa157d377), Next: 0}}, // First Berlin block + {10000000, ID{Hash: checksumToBytes(0xa157d377), Next: 0}}, // Future Berlin block }, }, // Rinkeby test cases @@ -100,8 +104,10 @@ func TestCreation(t *testing.T) { {4321233, ID{Hash: checksumToBytes(0xe49cab14), Next: 4321234}}, // Last Constantinople block {4321234, ID{Hash: checksumToBytes(0xafec6b27), Next: 5435345}}, // First Petersburg block {5435344, ID{Hash: checksumToBytes(0xafec6b27), Next: 5435345}}, // Last Petersburg block - {5435345, ID{Hash: checksumToBytes(0xcbdb8838), Next: 0}}, // First Istanbul block - {6000000, ID{Hash: checksumToBytes(0xcbdb8838), Next: 0}}, // Future Istanbul block + {5435345, ID{Hash: checksumToBytes(0xcbdb8838), Next: 8290928}}, // First Istanbul block + {8290927, ID{Hash: checksumToBytes(0xcbdb8838), Next: 8290928}}, // Last Istanbul block + {8290928, ID{Hash: checksumToBytes(0x6910c8bd), Next: 0}}, // First Berlin block + {10000000, ID{Hash: checksumToBytes(0x6910c8bd), Next: 0}}, // Future Berlin block }, }, // Goerli test cases @@ -111,8 +117,10 @@ func TestCreation(t *testing.T) { []testcase{ {0, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople and first Petersburg block {1561650, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}}, // Last Petersburg block - {1561651, ID{Hash: checksumToBytes(0xc25efa5c), Next: 0}}, // First Istanbul block - {2000000, ID{Hash: checksumToBytes(0xc25efa5c), Next: 0}}, // Future Istanbul block + {1561651, ID{Hash: checksumToBytes(0xc25efa5c), Next: 4460644}}, // First Istanbul block + {4460643, ID{Hash: checksumToBytes(0xc25efa5c), Next: 4460644}}, // Last Istanbul block + {4460644, ID{Hash: checksumToBytes(0x757a1c47), Next: 0}}, // First Berlin block + {5000000, ID{Hash: checksumToBytes(0x757a1c47), Next: 0}}, // Future Berlin block }, }, } @@ -185,11 +193,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Muir Glacier, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Berlin, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {88888888, ID{Hash: checksumToBytes(0xe029e991), Next: 88888888}, ErrLocalIncompatibleOrStale}, + {88888888, ID{Hash: checksumToBytes(0x0eb440f6), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. diff --git a/core/genesis.go b/core/genesis.go index 6bcc50b050..e05e27fe17 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -153,6 +153,10 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { + return SetupGenesisBlockWithOverride(db, genesis, nil) +} + +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideBerlin *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -198,6 +202,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) + if overrideBerlin != nil { + newcfg.BerlinBlock = overrideBerlin + } if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } diff --git a/core/state_transition.go b/core/state_transition.go index 0d589a6119..d511e40bd6 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -259,7 +259,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { } // Set up the initial access list. - if st.evm.ChainConfig().IsYoloV3(st.evm.Context.BlockNumber) { + if st.evm.ChainConfig().IsBerlin(st.evm.Context.BlockNumber) { st.state.PrepareAccessList(msg.From(), msg.To(), st.evm.ActivePrecompiles(), msg.AccessList()) } diff --git a/core/tx_pool.go b/core/tx_pool.go index 28ac822131..4c1bd809fd 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -1204,7 +1204,7 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { // Update all fork indicator by next pending block number. next := new(big.Int).Add(newHead.Number, big.NewInt(1)) pool.istanbul = pool.chainconfig.IsIstanbul(next) - pool.eip2718 = pool.chainconfig.IsYoloV3(next) + pool.eip2718 = pool.chainconfig.IsBerlin(next) } // promoteExecutables moves transactions that have become processable from the diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 1645369b4f..b4594cb90b 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -40,7 +40,7 @@ type sigCache struct { func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { var signer Signer switch { - case config.IsYoloV3(blockNumber): + case config.IsBerlin(blockNumber): signer = NewEIP2930Signer(config.ChainID) case config.IsEIP155(blockNumber): signer = NewEIP155Signer(config.ChainID) @@ -61,7 +61,7 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { // have the current block number available, use MakeSigner instead. func LatestSigner(config *params.ChainConfig) Signer { if config.ChainID != nil { - if config.YoloV3Block != nil { + if config.BerlinBlock != nil || config.YoloV3Block != nil { return NewEIP2930Signer(config.ChainID) } if config.EIP155Block != nil { diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 9ea19d38a7..4e99a51618 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -78,9 +78,9 @@ var PrecompiledContractsIstanbul = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{9}): &blake2F{}, } -// PrecompiledContractsYoloV3 contains the default set of pre-compiled Ethereum -// contracts used in the Yolo v3 test release. -var PrecompiledContractsYoloV3 = map[common.Address]PrecompiledContract{ +// PrecompiledContractsBerlin contains the default set of pre-compiled Ethereum +// contracts used in the Berlin release. +var PrecompiledContractsBerlin = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{1}): &ecrecover{}, common.BytesToAddress([]byte{2}): &sha256hash{}, common.BytesToAddress([]byte{3}): &ripemd160hash{}, @@ -107,7 +107,7 @@ var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{ } var ( - PrecompiledAddressesYoloV3 []common.Address + PrecompiledAddressesBerlin []common.Address PrecompiledAddressesIstanbul []common.Address PrecompiledAddressesByzantium []common.Address PrecompiledAddressesHomestead []common.Address @@ -123,8 +123,8 @@ func init() { for k := range PrecompiledContractsIstanbul { PrecompiledAddressesIstanbul = append(PrecompiledAddressesIstanbul, k) } - for k := range PrecompiledContractsYoloV3 { - PrecompiledAddressesYoloV3 = append(PrecompiledAddressesYoloV3, k) + for k := range PrecompiledContractsBerlin { + PrecompiledAddressesBerlin = append(PrecompiledAddressesBerlin, k) } } diff --git a/core/vm/evm.go b/core/vm/evm.go index 3617d77b12..7346d76e5b 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -46,8 +46,8 @@ type ( // configuration func (evm *EVM) ActivePrecompiles() []common.Address { switch { - case evm.chainRules.IsYoloV3: - return PrecompiledAddressesYoloV3 + case evm.chainRules.IsBerlin: + return PrecompiledAddressesBerlin case evm.chainRules.IsIstanbul: return PrecompiledAddressesIstanbul case evm.chainRules.IsByzantium: @@ -60,8 +60,8 @@ func (evm *EVM) ActivePrecompiles() []common.Address { func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) { var precompiles map[common.Address]PrecompiledContract switch { - case evm.chainRules.IsYoloV3: - precompiles = PrecompiledContractsYoloV3 + case evm.chainRules.IsBerlin: + precompiles = PrecompiledContractsBerlin case evm.chainRules.IsIstanbul: precompiles = PrecompiledContractsIstanbul case evm.chainRules.IsByzantium: @@ -446,7 +446,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, evm.StateDB.SetNonce(caller.Address(), nonce+1) // We add this to the access list _before_ taking a snapshot. Even if the creation fails, // the access-list change should not be rolled back - if evm.chainRules.IsYoloV3 { + if evm.chainRules.IsBerlin { evm.StateDB.AddAddressToAccessList(address) } // Ensure there's no existing contract already at the designated address diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 967db32780..0084b7d071 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -99,8 +99,8 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { if cfg.JumpTable[STOP] == nil { var jt JumpTable switch { - case evm.chainRules.IsYoloV3: - jt = yoloV3InstructionSet + case evm.chainRules.IsBerlin: + jt = berlinInstructionSet case evm.chainRules.IsIstanbul: jt = istanbulInstructionSet case evm.chainRules.IsConstantinople: diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 954f657a94..d831f9300f 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -56,24 +56,23 @@ var ( byzantiumInstructionSet = newByzantiumInstructionSet() constantinopleInstructionSet = newConstantinopleInstructionSet() istanbulInstructionSet = newIstanbulInstructionSet() - yoloV3InstructionSet = newYoloV3InstructionSet() + berlinInstructionSet = newBerlinInstructionSet() ) // JumpTable contains the EVM opcodes supported at a given fork. type JumpTable [256]*operation -// newYoloV3InstructionSet creates an instructionset containing -// - "EIP-2315: Simple Subroutines" -// - "EIP-2929: Gas cost increases for state access opcodes" -func newYoloV3InstructionSet() JumpTable { +// newBerlinInstructionSet returns the frontier, homestead, byzantium, +// contantinople, istanbul, petersburg and berlin instructions. +func newBerlinInstructionSet() JumpTable { instructionSet := newIstanbulInstructionSet() enable2315(&instructionSet) // Subroutines - https://eips.ethereum.org/EIPS/eip-2315 enable2929(&instructionSet) // Access lists for trie accesses https://eips.ethereum.org/EIPS/eip-2929 return instructionSet } -// newIstanbulInstructionSet returns the frontier, homestead -// byzantium, contantinople and petersburg instructions. +// newIstanbulInstructionSet returns the frontier, homestead, byzantium, +// contantinople, istanbul and petersburg instructions. func newIstanbulInstructionSet() JumpTable { instructionSet := newConstantinopleInstructionSet() @@ -84,7 +83,7 @@ func newIstanbulInstructionSet() JumpTable { return instructionSet } -// newConstantinopleInstructionSet returns the frontier, homestead +// newConstantinopleInstructionSet returns the frontier, homestead, // byzantium and contantinople instructions. func newConstantinopleInstructionSet() JumpTable { instructionSet := newByzantiumInstructionSet() diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index c97bdc420c..9cb69e1c76 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -65,6 +65,7 @@ func setDefaults(cfg *Config) { PetersburgBlock: new(big.Int), IstanbulBlock: new(big.Int), MuirGlacierBlock: new(big.Int), + BerlinBlock: new(big.Int), YoloV3Block: nil, } } @@ -113,7 +114,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { vmenv = NewEnv(cfg) sender = vm.AccountRef(cfg.Origin) ) - if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { + if cfg.ChainConfig.IsBerlin(vmenv.Context.BlockNumber) { cfg.State.PrepareAccessList(cfg.Origin, &address, vmenv.ActivePrecompiles(), nil) } cfg.State.CreateAccount(address) @@ -145,7 +146,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { vmenv = NewEnv(cfg) sender = vm.AccountRef(cfg.Origin) ) - if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { + if cfg.ChainConfig.IsBerlin(vmenv.Context.BlockNumber) { cfg.State.PrepareAccessList(cfg.Origin, nil, vmenv.ActivePrecompiles(), nil) } @@ -171,7 +172,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er sender := cfg.State.GetOrNewStateObject(cfg.Origin) statedb := cfg.State - if cfg.ChainConfig.IsYoloV3(vmenv.Context.BlockNumber) { + if cfg.ChainConfig.IsBerlin(vmenv.Context.BlockNumber) { statedb.PrepareAccessList(cfg.Origin, &address, vmenv.ActivePrecompiles(), nil) } diff --git a/eth/backend.go b/eth/backend.go index 044422763b..76ce5137f4 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -126,7 +126,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideBerlin) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 841dc5e9e1..5d0eece067 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -198,6 +198,9 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + + // Berlin block override (TODO: remove after the fork) + OverrideBerlin *big.Int `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/eth/tracers/api.go b/eth/tracers/api.go index bd995d08a9..61fe055689 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -581,8 +581,8 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block chainConfigCopy := new(params.ChainConfig) *chainConfigCopy = *chainConfig chainConfig = chainConfigCopy - if yolov3 := config.LogConfig.Overrides.YoloV3Block; yolov3 != nil { - chainConfig.YoloV3Block = yolov3 + if berlin := config.LogConfig.Overrides.BerlinBlock; berlin != nil { + chainConfig.BerlinBlock = berlin canon = false } } diff --git a/les/client.go b/les/client.go index e20519fd91..4d07f844f8 100644 --- a/les/client.go +++ b/les/client.go @@ -84,7 +84,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideBerlin) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } diff --git a/light/txpool.go b/light/txpool.go index bf5f9ff583..1296389e3b 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -314,7 +314,7 @@ func (pool *TxPool) setNewHead(head *types.Header) { // Update fork indicator by next pending block number next := new(big.Int).Add(head.Number, big.NewInt(1)) pool.istanbul = pool.config.IsIstanbul(next) - pool.eip2718 = pool.config.IsYoloV3(next) + pool.eip2718 = pool.config.IsBerlin(next) } // Stop stops the light transaction pool diff --git a/params/config.go b/params/config.go index 929bdbeb94..8cbfffc03c 100644 --- a/params/config.go +++ b/params/config.go @@ -56,18 +56,19 @@ var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), + EIP150Block: big.NewInt(2_463_000), EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - ByzantiumBlock: big.NewInt(4370000), - ConstantinopleBlock: big.NewInt(7280000), - PetersburgBlock: big.NewInt(7280000), - IstanbulBlock: big.NewInt(9069000), - MuirGlacierBlock: big.NewInt(9200000), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), Ethash: new(EthashConfig), } @@ -102,11 +103,12 @@ var ( EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), EIP155Block: big.NewInt(10), EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1700000), - ConstantinopleBlock: big.NewInt(4230000), - PetersburgBlock: big.NewInt(4939394), - IstanbulBlock: big.NewInt(6485846), - MuirGlacierBlock: big.NewInt(7117117), + ByzantiumBlock: big.NewInt(1_700_000), + ConstantinopleBlock: big.NewInt(4_230_000), + PetersburgBlock: big.NewInt(4_939_394), + IstanbulBlock: big.NewInt(6_485_846), + MuirGlacierBlock: big.NewInt(7_117_117), + BerlinBlock: big.NewInt(9_812_189), Ethash: new(EthashConfig), } @@ -141,11 +143,12 @@ var ( EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(1035301), - ConstantinopleBlock: big.NewInt(3660663), - PetersburgBlock: big.NewInt(4321234), - IstanbulBlock: big.NewInt(5435345), + ByzantiumBlock: big.NewInt(1_035_301), + ConstantinopleBlock: big.NewInt(3_660_663), + PetersburgBlock: big.NewInt(4_321_234), + IstanbulBlock: big.NewInt(5_435_345), MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(8_290_928), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -184,8 +187,9 @@ var ( ByzantiumBlock: big.NewInt(0), ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(1561651), + IstanbulBlock: big.NewInt(1_561_651), MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(4_460_644), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -227,6 +231,7 @@ var ( PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), MuirGlacierBlock: nil, + BerlinBlock: nil, // Don't enable Berlin directly, we're YOLOing it YoloV3Block: big.NewInt(0), Clique: &CliqueConfig{ Period: 15, @@ -239,16 +244,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -319,6 +324,7 @@ type ChainConfig struct { PetersburgBlock *big.Int `json:"petersburgBlock,omitempty"` // Petersburg switch block (nil = same as Constantinople) IstanbulBlock *big.Int `json:"istanbulBlock,omitempty"` // Istanbul switch block (nil = no fork, 0 = already on istanbul) MuirGlacierBlock *big.Int `json:"muirGlacierBlock,omitempty"` // Eip-2384 (bomb delay) switch block (nil = no fork, 0 = already activated) + BerlinBlock *big.Int `json:"berlinBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) YoloV3Block *big.Int `json:"yoloV3Block,omitempty"` // YOLO v3: Gas repricings TODO @holiman add EIP references EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) @@ -358,7 +364,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, YOLO v3: %v, Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, YOLO v3: %v, Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -371,6 +377,7 @@ func (c *ChainConfig) String() string { c.PetersburgBlock, c.IstanbulBlock, c.MuirGlacierBlock, + c.BerlinBlock, c.YoloV3Block, engine, ) @@ -428,9 +435,9 @@ func (c *ChainConfig) IsIstanbul(num *big.Int) bool { return isForked(c.IstanbulBlock, num) } -// IsYoloV3 returns whether num is either equal to the YoloV3 fork block or greater. -func (c *ChainConfig) IsYoloV3(num *big.Int) bool { - return isForked(c.YoloV3Block, num) +// IsBerlin returns whether num is either equal to the Berlin fork block or greater. +func (c *ChainConfig) IsBerlin(num *big.Int) bool { + return isForked(c.BerlinBlock, num) || isForked(c.YoloV3Block, num) } // IsEWASM returns whether num represents a block number after the EWASM fork @@ -476,7 +483,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "petersburgBlock", block: c.PetersburgBlock}, {name: "istanbulBlock", block: c.IstanbulBlock}, {name: "muirGlacierBlock", block: c.MuirGlacierBlock, optional: true}, - {name: "yoloV3Block", block: c.YoloV3Block}, + {name: "berlinBlock", block: c.BerlinBlock}, } { if lastFork.name != "" { // Next one must be higher number @@ -540,6 +547,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.MuirGlacierBlock, newcfg.MuirGlacierBlock, head) { return newCompatError("Muir Glacier fork block", c.MuirGlacierBlock, newcfg.MuirGlacierBlock) } + if isForkIncompatible(c.BerlinBlock, newcfg.BerlinBlock, head) { + return newCompatError("Berlin fork block", c.BerlinBlock, newcfg.BerlinBlock) + } if isForkIncompatible(c.YoloV3Block, newcfg.YoloV3Block, head) { return newCompatError("YOLOv3 fork block", c.YoloV3Block, newcfg.YoloV3Block) } @@ -613,7 +623,7 @@ type Rules struct { ChainID *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool - IsYoloV3 bool + IsBerlin bool } // Rules ensures c's ChainID is not nil. @@ -632,6 +642,6 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { IsConstantinople: c.IsConstantinople(num), IsPetersburg: c.IsPetersburg(num), IsIstanbul: c.IsIstanbul(num), - IsYoloV3: c.IsYoloV3(num), + IsBerlin: c.IsBerlin(num), } } diff --git a/tests/init.go b/tests/init.go index a2ef040786..67f706eb50 100644 --- a/tests/init.go +++ b/tests/init.go @@ -165,7 +165,7 @@ var Forks = map[string]*params.ChainConfig{ ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), - YoloV3Block: big.NewInt(0), + BerlinBlock: big.NewInt(0), }, } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 03dc01459c..99681baaf1 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -24,14 +24,13 @@ import ( "strconv" "strings" - "github.com/ethereum/go-ethereum/core/state/snapshot" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" From 27b31371d46bc932853cce36078b28a53088b2b2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 26 Feb 2021 13:40:35 +0100 Subject: [PATCH 166/709] rpc: add separate size limit for websocket (#22385) This makes the WebSocket message size limit independent of the limit used for HTTP requests. The new limit for WebSocket messages is 15MB. --- rpc/http_test.go | 25 +++++++++++++++++++++++++ rpc/testservice_test.go | 10 ++++++++++ rpc/websocket.go | 3 ++- rpc/websocket_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/rpc/http_test.go b/rpc/http_test.go index fc939ae48f..b75af67c52 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -98,3 +98,28 @@ func confirmHTTPRequestYieldsStatusCode(t *testing.T, method, contentType, body func TestHTTPResponseWithEmptyGet(t *testing.T) { confirmHTTPRequestYieldsStatusCode(t, http.MethodGet, "", "", http.StatusOK) } + +// This checks that maxRequestContentLength is not applied to the response of a request. +func TestHTTPRespBodyUnlimited(t *testing.T) { + const respLength = maxRequestContentLength * 3 + + s := NewServer() + defer s.Stop() + s.RegisterName("test", largeRespService{respLength}) + ts := httptest.NewServer(s) + defer ts.Close() + + c, err := DialHTTP(ts.URL) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + var r string + if err := c.Call(&r, "test_largeResp"); err != nil { + t.Fatal(err) + } + if len(r) != respLength { + t.Fatalf("response has wrong length %d, want %d", len(r), respLength) + } +} diff --git a/rpc/testservice_test.go b/rpc/testservice_test.go index 6f948a1bac..62afc1df44 100644 --- a/rpc/testservice_test.go +++ b/rpc/testservice_test.go @@ -20,6 +20,7 @@ import ( "context" "encoding/binary" "errors" + "strings" "sync" "time" ) @@ -194,3 +195,12 @@ func (s *notificationTestService) HangSubscription(ctx context.Context, val int) }() return subscription, nil } + +// largeRespService generates arbitrary-size JSON responses. +type largeRespService struct { + length int +} + +func (x largeRespService) LargeResp() string { + return strings.Repeat("x", x.length) +} diff --git a/rpc/websocket.go b/rpc/websocket.go index cd60eeb613..ab55ae69c1 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -37,6 +37,7 @@ const ( wsWriteBuffer = 1024 wsPingInterval = 60 * time.Second wsPingWriteTimeout = 5 * time.Second + wsMessageSizeLimit = 15 * 1024 * 1024 ) var wsBufferPool = new(sync.Pool) @@ -239,7 +240,7 @@ type websocketCodec struct { } func newWebsocketCodec(conn *websocket.Conn) ServerCodec { - conn.SetReadLimit(maxRequestContentLength) + conn.SetReadLimit(wsMessageSizeLimit) wc := &websocketCodec{ jsonCodec: NewFuncCodec(conn, conn.WriteJSON, conn.ReadJSON).(*jsonCodec), conn: conn, diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index f54fc3cd54..37ed19476f 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -157,6 +157,33 @@ func TestClientWebsocketPing(t *testing.T) { } } +// This checks that the websocket transport can deal with large messages. +func TestClientWebsocketLargeMessage(t *testing.T) { + var ( + srv = NewServer() + httpsrv = httptest.NewServer(srv.WebsocketHandler(nil)) + wsURL = "ws:" + strings.TrimPrefix(httpsrv.URL, "http:") + ) + defer srv.Stop() + defer httpsrv.Close() + + respLength := wsMessageSizeLimit - 50 + srv.RegisterName("test", largeRespService{respLength}) + + c, err := DialWebsocket(context.Background(), wsURL, "") + if err != nil { + t.Fatal(err) + } + + var r string + if err := c.Call(&r, "test_largeResp"); err != nil { + t.Fatal("call failed:", err) + } + if len(r) != respLength { + t.Fatalf("response has wrong length %d, want %d", len(r), respLength) + } +} + // wsPingTestServer runs a WebSocket server which accepts a single subscription request. // When a value arrives on sendPing, the server sends a ping frame, waits for a matching // pong and finally delivers a single subscription result. From 3822b09904edcd92bc203b5739115208daa38765 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 26 Feb 2021 15:28:34 +0100 Subject: [PATCH 167/709] accounts/keystore: use github.com/google/uuid (#22217) This replaces the github.com/pborman/uuid dependency with github.com/google/uuid because the former is only a wrapper for the latter (since v1.0.0). Co-authored-by: Felix Lange --- accounts/keystore/key.go | 12 +++++++++--- accounts/keystore/passphrase.go | 21 ++++++++++++++++----- accounts/keystore/presale.go | 9 ++++++--- cmd/ethkey/generate.go | 9 ++++++--- go.mod | 3 +-- go.sum | 20 ++------------------ 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/accounts/keystore/key.go b/accounts/keystore/key.go index 84d8df0c5a..2b815ce0f9 100644 --- a/accounts/keystore/key.go +++ b/accounts/keystore/key.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/pborman/uuid" + "github.com/google/uuid" ) const ( @@ -110,7 +110,10 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) { } u := new(uuid.UUID) - *u = uuid.Parse(keyJSON.Id) + *u, err = uuid.Parse(keyJSON.Id) + if err != nil { + return err + } k.Id = *u addr, err := hex.DecodeString(keyJSON.Address) if err != nil { @@ -128,7 +131,10 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) { } func newKeyFromECDSA(privateKeyECDSA *ecdsa.PrivateKey) *Key { - id := uuid.NewRandom() + id, err := uuid.NewRandom() + if err != nil { + panic(fmt.Sprintf("Could not create random uuid: %v", err)) + } key := &Key{ Id: id, Address: crypto.PubkeyToAddress(privateKeyECDSA.PublicKey), diff --git a/accounts/keystore/passphrase.go b/accounts/keystore/passphrase.go index dd4d7764e4..3b3e631888 100644 --- a/accounts/keystore/passphrase.go +++ b/accounts/keystore/passphrase.go @@ -42,7 +42,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" - "github.com/pborman/uuid" + "github.com/google/uuid" "golang.org/x/crypto/pbkdf2" "golang.org/x/crypto/scrypt" ) @@ -228,9 +228,12 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) { return nil, err } key := crypto.ToECDSAUnsafe(keyBytes) - + id, err := uuid.FromBytes(keyId) + if err != nil { + return nil, err + } return &Key{ - Id: keyId, + Id: id, Address: crypto.PubkeyToAddress(key.PublicKey), PrivateKey: key, }, nil @@ -276,7 +279,11 @@ func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byt if keyProtected.Version != version { return nil, nil, fmt.Errorf("version not supported: %v", keyProtected.Version) } - keyId = uuid.Parse(keyProtected.Id) + keyUUID, err := uuid.Parse(keyProtected.Id) + if err != nil { + return nil, nil, err + } + keyId = keyUUID[:] plainText, err := DecryptDataV3(keyProtected.Crypto, auth) if err != nil { return nil, nil, err @@ -285,7 +292,11 @@ func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byt } func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byte, keyId []byte, err error) { - keyId = uuid.Parse(keyProtected.Id) + keyUUID, err := uuid.Parse(keyProtected.Id) + if err != nil { + return nil, nil, err + } + keyId = keyUUID[:] mac, err := hex.DecodeString(keyProtected.Crypto.MAC) if err != nil { return nil, nil, err diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index 03055245f5..0664dc2cdd 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -27,7 +27,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/crypto" - "github.com/pborman/uuid" + "github.com/google/uuid" "golang.org/x/crypto/pbkdf2" ) @@ -37,7 +37,10 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou if err != nil { return accounts.Account{}, nil, err } - key.Id = uuid.NewRandom() + key.Id, err = uuid.NewRandom() + if err != nil { + return accounts.Account{}, nil, err + } a := accounts.Account{ Address: key.Address, URL: accounts.URL{ @@ -86,7 +89,7 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error ecKey := crypto.ToECDSAUnsafe(ethPriv) key = &Key{ - Id: nil, + Id: uuid.UUID{}, Address: crypto.PubkeyToAddress(ecKey.PublicKey), PrivateKey: ecKey, } diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index c2aa1c6fb4..629d23da5b 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" - "github.com/pborman/uuid" + "github.com/google/uuid" "gopkg.in/urfave/cli.v1" ) @@ -86,9 +86,12 @@ If you want to encrypt an existing private key, it can be specified by setting } // Create the keyfile object with a random UUID. - id := uuid.NewRandom() + UUID, err := uuid.NewRandom() + if err != nil { + utils.Fatalf("Failed to generate random uuid: %v", err) + } key := &keystore.Key{ - Id: id, + Id: UUID, Address: crypto.PubkeyToAddress(privateKey.PublicKey), PrivateKey: privateKey, } diff --git a/go.mod b/go.mod index a1099c52cc..96a217ed06 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa + github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d @@ -37,7 +38,6 @@ require ( github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c - github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 @@ -48,7 +48,6 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 diff --git a/go.sum b/go.sum index 4b788824c7..af76759c51 100644 --- a/go.sum +++ b/go.sum @@ -48,7 +48,6 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= -github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -72,19 +71,12 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -109,7 +101,6 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -137,7 +128,6 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= -github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -145,7 +135,6 @@ github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -158,7 +147,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -190,12 +178,10 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -204,7 +190,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -215,6 +200,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -255,7 +242,6 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 h1:HarGZ5h9HD9LgEg1yRVMXyfiw4wlXiLiYM2oMjeA/SE= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= @@ -264,7 +250,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= @@ -372,7 +357,6 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= From 498458b4102c0d32d7453035a115e6b9df5e485d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 26 Feb 2021 16:33:37 +0100 Subject: [PATCH 168/709] core/state: fix eta calculation on pruning (#22386) --- core/state/pruner/pruner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 1fbfa55b6a..530a348540 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -155,7 +155,7 @@ func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[c if done := binary.BigEndian.Uint64(key[:8]); done > 0 { var ( left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) - speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero ) eta = time.Duration(left/speed) * time.Millisecond } From d96870428f116494d5190a8e595189e283dd144b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 1 Mar 2021 10:24:20 +0100 Subject: [PATCH 169/709] les: UDP pre-negotiation of available server capacity (#22183) This PR implements the first one of the "lespay" UDP queries which is already useful in itself: the capacity query. The server pool is making use of this query by doing a cheap UDP query to determine whether it is worth starting the more expensive TCP connection process. --- common/prque/lazyqueue.go | 5 +- common/prque/lazyqueue_test.go | 2 +- les/client.go | 85 +++++++++-- les/clientpool.go | 55 +++++++ les/clientpool_test.go | 6 +- les/enr_entry.go | 3 +- les/server.go | 29 +++- les/vflux/client/serverpool.go | 105 +++++++++++-- les/vflux/client/serverpool_test.go | 8 +- les/vflux/requests.go | 180 +++++++++++++++++++++++ les/vflux/server/balance.go | 58 +++++--- les/vflux/server/balance_test.go | 4 +- les/vflux/server/prioritypool.go | 202 +++++++++++++++++++++++--- les/vflux/server/prioritypool_test.go | 126 +++++++++++++++- les/vflux/server/service.go | 122 ++++++++++++++++ p2p/discover/v5_udp.go | 11 +- p2p/discover/v5_udp_test.go | 2 +- p2p/nodestate/nodestate.go | 1 + 18 files changed, 915 insertions(+), 89 deletions(-) create mode 100644 les/vflux/requests.go create mode 100644 les/vflux/server/service.go diff --git a/common/prque/lazyqueue.go b/common/prque/lazyqueue.go index 52403df464..c74faab7e6 100644 --- a/common/prque/lazyqueue.go +++ b/common/prque/lazyqueue.go @@ -48,7 +48,7 @@ type LazyQueue struct { } type ( - PriorityCallback func(data interface{}, now mclock.AbsTime) int64 // actual priority callback + PriorityCallback func(data interface{}) int64 // actual priority callback MaxPriorityCallback func(data interface{}, until mclock.AbsTime) int64 // estimated maximum priority callback ) @@ -139,11 +139,10 @@ func (q *LazyQueue) peekIndex() int { // Pop multiple times. Popped items are passed to the callback. MultiPop returns // when the callback returns false or there are no more items to pop. func (q *LazyQueue) MultiPop(callback func(data interface{}, priority int64) bool) { - now := q.clock.Now() nextIndex := q.peekIndex() for nextIndex != -1 { data := heap.Pop(q.queue[nextIndex]).(*item).value - heap.Push(q.popQueue, &item{data, q.priority(data, now)}) + heap.Push(q.popQueue, &item{data, q.priority(data)}) nextIndex = q.peekIndex() for q.popQueue.Len() != 0 && (nextIndex == -1 || q.queue[nextIndex].blocks[0][0].priority < q.popQueue.blocks[0][0].priority) { i := heap.Pop(q.popQueue).(*item) diff --git a/common/prque/lazyqueue_test.go b/common/prque/lazyqueue_test.go index be9491e24e..9a831d628b 100644 --- a/common/prque/lazyqueue_test.go +++ b/common/prque/lazyqueue_test.go @@ -40,7 +40,7 @@ type lazyItem struct { index int } -func testPriority(a interface{}, now mclock.AbsTime) int64 { +func testPriority(a interface{}) int64 { return a.(*lazyItem).p } diff --git a/les/client.go b/les/client.go index 4d07f844f8..ecabfdf503 100644 --- a/les/client.go +++ b/les/client.go @@ -36,30 +36,33 @@ import ( "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/les/vflux" vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) type LightEthereum struct { lesCommons - peers *serverPeerSet - reqDist *requestDistributor - retriever *retrieveManager - odr *LesOdr - relay *lesTxRelay - handler *clientHandler - txPool *light.TxPool - blockchain *light.LightChain - serverPool *vfc.ServerPool - dialCandidates enode.Iterator - pruner *pruner + peers *serverPeerSet + reqDist *requestDistributor + retriever *retrieveManager + odr *LesOdr + relay *lesTxRelay + handler *clientHandler + txPool *light.TxPool + blockchain *light.LightChain + serverPool *vfc.ServerPool + serverPoolIterator enode.Iterator + pruner *pruner bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports @@ -112,7 +115,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { p2pConfig: &stack.Config().P2P, } - leth.serverPool, leth.dialCandidates = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, nil, &mclock.System{}, config.UltraLightServers, requestList) + leth.serverPool, leth.serverPoolIterator = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, leth.prenegQuery, &mclock.System{}, config.UltraLightServers, requestList) leth.serverPool.AddMetrics(suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge, sessionValueMeter, serverDialedMeter) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.GetTimeout) @@ -189,6 +192,62 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { return leth, nil } +// VfluxRequest sends a batch of requests to the given node through discv5 UDP TalkRequest and returns the responses +func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.Replies { + reqsEnc, _ := rlp.EncodeToBytes(&reqs) + repliesEnc, _ := s.p2pServer.DiscV5.TalkRequest(s.serverPool.DialNode(n), "vfx", reqsEnc) + var replies vflux.Replies + if len(repliesEnc) == 0 || rlp.DecodeBytes(repliesEnc, &replies) != nil { + return nil + } + return replies +} + +// vfxVersion returns the version number of the "les" service subdomain of the vflux UDP +// service, as advertised in the ENR record +func (s *LightEthereum) vfxVersion(n *enode.Node) uint { + if n.Seq() == 0 { + var err error + if n, err = s.p2pServer.DiscV5.RequestENR(n); n != nil && err == nil && n.Seq() != 0 { + s.serverPool.Persist(n) + } else { + return 0 + } + } + + var les []rlp.RawValue + if err := n.Load(enr.WithEntry("les", &les)); err != nil || len(les) < 1 { + return 0 + } + var version uint + rlp.DecodeBytes(les[0], &version) // Ignore additional fields (for forward compatibility). + return version +} + +// prenegQuery sends a capacity query to the given server node to determine whether +// a connection slot is immediately available +func (s *LightEthereum) prenegQuery(n *enode.Node) int { + if s.vfxVersion(n) < 1 { + // UDP query not supported, always try TCP connection + return 1 + } + + var requests vflux.Requests + requests.Add("les", vflux.CapacityQueryName, vflux.CapacityQueryReq{ + Bias: 180, + AddTokens: []vflux.IntOrInf{{}}, + }) + replies := s.VfluxRequest(n, requests) + var cqr vflux.CapacityQueryReply + if replies.Get(0, &cqr) != nil || len(cqr) != 1 { // Note: Get returns an error if replies is nil + return -1 + } + if cqr[0] > 0 { + return 1 + } + return 0 +} + type LightDummyAPI struct{} // Etherbase is the address that mining rewards will be send to @@ -269,7 +328,7 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { return p.Info() } return nil - }, s.dialCandidates) + }, s.serverPoolIterator) } // Start implements node.Lifecycle, starting all internal goroutines needed by the diff --git a/les/clientpool.go b/les/clientpool.go index 4e1499bf5d..1aa63a281e 100644 --- a/les/clientpool.go +++ b/les/clientpool.go @@ -24,11 +24,13 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/utils" + "github.com/ethereum/go-ethereum/les/vflux" vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nodestate" + "github.com/ethereum/go-ethereum/rlp" ) const ( @@ -382,3 +384,56 @@ func (f *clientPool) forClients(ids []enode.ID, cb func(client *clientInfo)) { } } } + +// serveCapQuery serves a vflux capacity query. It receives multiple token amount values +// and a bias time value. For each given token amount it calculates the maximum achievable +// capacity in case the amount is added to the balance. +func (f *clientPool) serveCapQuery(id enode.ID, freeID string, data []byte) []byte { + var req vflux.CapacityQueryReq + if rlp.DecodeBytes(data, &req) != nil { + return nil + } + if l := len(req.AddTokens); l == 0 || l > vflux.CapacityQueryMaxLen { + return nil + } + node := f.ns.GetNode(id) + if node == nil { + node = enode.SignNull(&enr.Record{}, id) + } + c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo) + if c == nil { + c = &clientInfo{node: node} + f.ns.SetField(node, clientInfoField, c) + f.ns.SetField(node, connAddressField, freeID) + defer func() { + f.ns.SetField(node, connAddressField, nil) + f.ns.SetField(node, clientInfoField, nil) + }() + if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance == nil { + log.Error("BalanceField is missing", "node", node.ID()) + return nil + } + } + // use vfs.CapacityCurve to answer request for multiple newly bought token amounts + curve := f.pp.GetCapacityCurve().Exclude(id) + result := make(vflux.CapacityQueryReply, len(req.AddTokens)) + bias := time.Second * time.Duration(req.Bias) + if f.connectedBias > bias { + bias = f.connectedBias + } + pb, _ := c.balance.GetBalance() + for i, addTokens := range req.AddTokens { + add := addTokens.Int64() + result[i] = curve.MaxCapacity(func(capacity uint64) int64 { + return c.balance.EstimatePriority(capacity, add, 0, bias, false) / int64(capacity) + }) + if add <= 0 && uint64(-add) >= pb && result[i] > f.minCap { + result[i] = f.minCap + } + if result[i] < f.minCap { + result[i] = 0 + } + } + reply, _ := rlp.EncodeToBytes(&result) + return reply +} diff --git a/les/clientpool_test.go b/les/clientpool_test.go index 5cff010409..345b373b0f 100644 --- a/les/clientpool_test.go +++ b/les/clientpool_test.go @@ -508,8 +508,10 @@ func TestNegativeBalanceCalculation(t *testing.T) { for i := 0; i < 10; i++ { pool.disconnect(newPoolTestPeer(i, nil)) _, nb := getBalance(pool, newPoolTestPeer(i, nil)) - if checkDiff(nb, uint64(time.Minute)/1000) { - t.Fatalf("Negative balance mismatch, want %v, got %v", uint64(time.Minute)/1000, nb) + exp := uint64(time.Minute) / 1000 + exp -= exp / 120 // correct for negative balance expiration + if checkDiff(nb, exp) { + t.Fatalf("Negative balance mismatch, want %v, got %v", exp, nb) } } } diff --git a/les/enr_entry.go b/les/enr_entry.go index 1e56c1f175..8be4a7a00e 100644 --- a/les/enr_entry.go +++ b/les/enr_entry.go @@ -27,7 +27,8 @@ import ( // lesEntry is the "les" ENR entry. This is set for LES servers only. type lesEntry struct { // Ignore additional fields (for forward compatibility). - _ []rlp.RawValue `rlp:"tail"` + VfxVersion uint + Rest []rlp.RawValue `rlp:"tail"` } func (lesEntry) ENRKey() string { return "les" } diff --git a/les/server.go b/les/server.go index 359784cf7d..63feaf892c 100644 --- a/les/server.go +++ b/les/server.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/les/vflux" vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" @@ -68,6 +69,7 @@ type LesServer struct { archiveMode bool // Flag whether the ethereum node runs in archive mode. handler *serverHandler broadcaster *broadcaster + vfluxServer *vfs.Server privateKey *ecdsa.PrivateKey // Flow control and capacity management @@ -112,12 +114,14 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les ns: ns, archiveMode: e.ArchiveMode(), broadcaster: newBroadcaster(ns), + vfluxServer: vfs.NewServer(time.Millisecond * 10), fcManager: flowcontrol.NewClientManager(nil, &mclock.System{}), servingQueue: newServingQueue(int64(time.Millisecond*10), float64(config.LightServ)/100), threadsBusy: config.LightServ/100 + 1, threadsIdle: threads, p2pSrv: node.Server(), } + srv.vfluxServer.Register(srv) issync := e.Synced if config.LightNoSyncServe { issync = func() bool { return true } @@ -201,7 +205,9 @@ func (s *LesServer) Protocols() []p2p.Protocol { }, nil) // Add "les" ENR entries. for i := range ps { - ps[i].Attributes = []enr.Entry{&lesEntry{}} + ps[i].Attributes = []enr.Entry{&lesEntry{ + VfxVersion: 1, + }} } return ps } @@ -211,10 +217,11 @@ func (s *LesServer) Start() error { s.privateKey = s.p2pSrv.PrivateKey s.broadcaster.setSignerKey(s.privateKey) s.handler.start() - s.wg.Add(1) go s.capacityManagement() - + if s.p2pSrv.DiscV5 != nil { + s.p2pSrv.DiscV5.RegisterTalkHandler("vfx", s.vfluxServer.ServeEncoded) + } return nil } @@ -228,6 +235,7 @@ func (s *LesServer) Stop() error { s.costTracker.stop() s.handler.stop() s.servingQueue.stop() + s.vfluxServer.Stop() // Note, bloom trie indexer is closed by parent bloombits indexer. s.chtIndexer.Close() @@ -311,3 +319,18 @@ func (s *LesServer) dropClient(id enode.ID) { p.Peer.Disconnect(p2p.DiscRequested) } } + +// ServiceInfo implements vfs.Service +func (s *LesServer) ServiceInfo() (string, string) { + return "les", "Ethereum light client service" +} + +// Handle implements vfs.Service +func (s *LesServer) Handle(id enode.ID, address string, name string, data []byte) []byte { + switch name { + case vflux.CapacityQueryName: + return s.clientPool.serveCapQuery(id, address, data) + default: + return nil + } +} diff --git a/les/vflux/client/serverpool.go b/les/vflux/client/serverpool.go index 95f7246091..47ec4fee74 100644 --- a/les/vflux/client/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -94,7 +94,7 @@ type nodeHistoryEnc struct { type queryFunc func(*enode.Node) int var ( - clientSetup = &nodestate.Setup{Version: 1} + clientSetup = &nodestate.Setup{Version: 2} sfHasValue = clientSetup.NewPersistentFlag("hasValue") sfQueried = clientSetup.NewFlag("queried") sfCanDial = clientSetup.NewFlag("canDial") @@ -131,9 +131,25 @@ var ( ) sfiNodeWeight = clientSetup.NewField("nodeWeight", reflect.TypeOf(uint64(0))) sfiConnectedStats = clientSetup.NewField("connectedStats", reflect.TypeOf(ResponseTimeStats{})) + sfiLocalAddress = clientSetup.NewPersistentField("localAddress", reflect.TypeOf(&enr.Record{}), + func(field interface{}) ([]byte, error) { + if enr, ok := field.(*enr.Record); ok { + enc, err := rlp.EncodeToBytes(enr) + return enc, err + } + return nil, errors.New("invalid field type") + }, + func(enc []byte) (interface{}, error) { + var enr enr.Record + if err := rlp.DecodeBytes(enc, &enr); err != nil { + return nil, err + } + return &enr, nil + }, + ) ) -// newServerPool creates a new server pool +// NewServerPool creates a new server pool func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string, requestList []RequestInfo) (*ServerPool, enode.Iterator) { s := &ServerPool{ db: db, @@ -151,15 +167,10 @@ func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duratio s.mixSources = append(s.mixSources, knownSelector) s.mixSources = append(s.mixSources, alwaysConnect) - iter := enode.Iterator(s.mixer) + s.dialIterator = s.mixer if query != nil { - iter = s.addPreNegFilter(iter, query) + s.dialIterator = s.addPreNegFilter(s.dialIterator, query) } - s.dialIterator = enode.Filter(iter, func(node *enode.Node) bool { - s.ns.SetState(node, sfDialing, sfCanDial, 0) - s.ns.SetState(node, sfWaitDialTimeout, nodestate.Flags{}, time.Second*10) - return true - }) s.ns.SubscribeState(nodestate.MergeFlags(sfWaitDialTimeout, sfConnected), func(n *enode.Node, oldState, newState nodestate.Flags) { if oldState.Equals(sfWaitDialTimeout) && newState.IsEmpty() { @@ -169,7 +180,41 @@ func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duratio } }) - return s, s.dialIterator + return s, &serverPoolIterator{ + dialIterator: s.dialIterator, + nextFn: func(node *enode.Node) { + s.ns.Operation(func() { + s.ns.SetStateSub(node, sfDialing, sfCanDial, 0) + s.ns.SetStateSub(node, sfWaitDialTimeout, nodestate.Flags{}, time.Second*10) + }) + }, + nodeFn: s.DialNode, + } +} + +type serverPoolIterator struct { + dialIterator enode.Iterator + nextFn func(*enode.Node) + nodeFn func(*enode.Node) *enode.Node +} + +// Next implements enode.Iterator +func (s *serverPoolIterator) Next() bool { + if s.dialIterator.Next() { + s.nextFn(s.dialIterator.Node()) + return true + } + return false +} + +// Node implements enode.Iterator +func (s *serverPoolIterator) Node() *enode.Node { + return s.nodeFn(s.dialIterator.Node()) +} + +// Close implements enode.Iterator +func (s *serverPoolIterator) Close() { + s.dialIterator.Close() } // AddMetrics adds metrics to the server pool. Should be called before Start(). @@ -285,7 +330,6 @@ func (s *ServerPool) Start() { // stop stops the server pool func (s *ServerPool) Stop() { - s.dialIterator.Close() if s.fillSet != nil { s.fillSet.Close() } @@ -299,18 +343,23 @@ func (s *ServerPool) Stop() { s.vt.Stop() } -// registerPeer implements serverPeerSubscriber +// RegisterNode implements serverPeerSubscriber func (s *ServerPool) RegisterNode(node *enode.Node) (*NodeValueTracker, error) { if atomic.LoadUint32(&s.started) == 0 { return nil, errors.New("server pool not started yet") } - s.ns.SetState(node, sfConnected, sfDialing.Or(sfWaitDialTimeout), 0) nvt := s.vt.Register(node.ID()) - s.ns.SetField(node, sfiConnectedStats, nvt.RtStats()) + s.ns.Operation(func() { + s.ns.SetStateSub(node, sfConnected, sfDialing.Or(sfWaitDialTimeout), 0) + s.ns.SetFieldSub(node, sfiConnectedStats, nvt.RtStats()) + if node.IP().IsLoopback() { + s.ns.SetFieldSub(node, sfiLocalAddress, node.Record()) + } + }) return nvt, nil } -// unregisterPeer implements serverPeerSubscriber +// UnregisterNode implements serverPeerSubscriber func (s *ServerPool) UnregisterNode(node *enode.Node) { s.ns.Operation(func() { s.setRedialWait(node, dialCost, dialWaitStep) @@ -430,6 +479,7 @@ func (s *ServerPool) updateWeight(node *enode.Node, totalValue float64, totalDia s.ns.SetStateSub(node, nodestate.Flags{}, sfHasValue, 0) s.ns.SetFieldSub(node, sfiNodeWeight, nil) s.ns.SetFieldSub(node, sfiNodeHistory, nil) + s.ns.SetFieldSub(node, sfiLocalAddress, nil) } s.ns.Persist(node) // saved if node history or hasValue changed } @@ -520,3 +570,28 @@ func (s *ServerPool) calculateWeight(node *enode.Node) { func (s *ServerPool) API() *PrivateClientAPI { return NewPrivateClientAPI(s.vt) } + +type dummyIdentity enode.ID + +func (id dummyIdentity) Verify(r *enr.Record, sig []byte) error { return nil } +func (id dummyIdentity) NodeAddr(r *enr.Record) []byte { return id[:] } + +// DialNode replaces the given enode with a locally generated one containing the ENR +// stored in the sfiLocalAddress field if present. This workaround ensures that nodes +// on the local network can be dialed at the local address if a connection has been +// successfully established previously. +// Note that NodeStateMachine always remembers the enode with the latest version of +// the remote signed ENR. ENR filtering should be performed on that version while +// dialNode should be used for dialing the node over TCP or UDP. +func (s *ServerPool) DialNode(n *enode.Node) *enode.Node { + if enr, ok := s.ns.GetField(n, sfiLocalAddress).(*enr.Record); ok { + n, _ := enode.New(dummyIdentity(n.ID()), enr) + return n + } + return n +} + +// Persist immediately stores the state of a node in the node database +func (s *ServerPool) Persist(n *enode.Node) { + s.ns.Persist(n) +} diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index 3af3db95bc..ee299618c6 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -56,6 +56,7 @@ type ServerPoolTest struct { preNeg, preNegFail bool vt *ValueTracker sp *ServerPool + spi enode.Iterator input enode.Iterator testNodes []spTestNode trusted []string @@ -148,7 +149,7 @@ func (s *ServerPoolTest) start() { requestList[i] = RequestInfo{Name: "testreq" + strconv.Itoa(i), InitAmount: 1, InitValue: 1} } - s.sp, _ = NewServerPool(s.db, []byte("sp:"), 0, testQuery, s.clock, s.trusted, requestList) + s.sp, s.spi = NewServerPool(s.db, []byte("sp:"), 0, testQuery, s.clock, s.trusted, requestList) s.sp.AddSource(s.input) s.sp.validSchemes = enode.ValidSchemesForTesting s.sp.unixTime = func() int64 { return int64(s.clock.Now()) / int64(time.Second) } @@ -176,6 +177,7 @@ func (s *ServerPoolTest) start() { func (s *ServerPoolTest) stop() { close(s.quit) s.sp.Stop() + s.spi.Close() for i := range s.testNodes { n := &s.testNodes[i] if n.connected { @@ -208,9 +210,9 @@ func (s *ServerPoolTest) run() { if s.conn < spTestTarget { s.dialCount++ s.beginWait() - s.sp.dialIterator.Next() + s.spi.Next() s.endWait() - dial := s.sp.dialIterator.Node() + dial := s.spi.Node() id := dial.ID() idx := testNodeIndex(id) n := &s.testNodes[idx] diff --git a/les/vflux/requests.go b/les/vflux/requests.go new file mode 100644 index 0000000000..11255607e8 --- /dev/null +++ b/les/vflux/requests.go @@ -0,0 +1,180 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package vflux + +import ( + "errors" + "math" + "math/big" + + "github.com/ethereum/go-ethereum/rlp" +) + +var ErrNoReply = errors.New("no reply for given request") + +const ( + MaxRequestLength = 16 // max number of individual requests in a batch + CapacityQueryName = "cq" + CapacityQueryMaxLen = 16 +) + +type ( + // Request describes a single vflux request inside a batch. Service and request + // type are identified by strings, parameters are RLP encoded. + Request struct { + Service, Name string + Params []byte + } + // Requests are a batch of vflux requests + Requests []Request + + // Replies are the replies to a batch of requests + Replies [][]byte + + // CapacityQueryReq is the encoding format of the capacity query + CapacityQueryReq struct { + Bias uint64 // seconds + AddTokens []IntOrInf + } + // CapacityQueryReq is the encoding format of the response to the capacity query + CapacityQueryReply []uint64 +) + +// Add encodes and adds a new request to the batch +func (r *Requests) Add(service, name string, val interface{}) (int, error) { + enc, err := rlp.EncodeToBytes(val) + if err != nil { + return -1, err + } + *r = append(*r, Request{ + Service: service, + Name: name, + Params: enc, + }) + return len(*r) - 1, nil +} + +// Get decodes the reply to the i-th request in the batch +func (r Replies) Get(i int, val interface{}) error { + if i < 0 || i >= len(r) { + return ErrNoReply + } + return rlp.DecodeBytes(r[i], val) +} + +const ( + IntNonNegative = iota + IntNegative + IntPlusInf + IntMinusInf +) + +// IntOrInf is the encoding format for arbitrary length signed integers that can also +// hold the values of +Inf or -Inf +type IntOrInf struct { + Type uint8 + Value big.Int +} + +// BigInt returns the value as a big.Int or panics if the value is infinity +func (i *IntOrInf) BigInt() *big.Int { + switch i.Type { + case IntNonNegative: + return new(big.Int).Set(&i.Value) + case IntNegative: + return new(big.Int).Neg(&i.Value) + case IntPlusInf: + panic(nil) // caller should check Inf() before trying to convert to big.Int + case IntMinusInf: + panic(nil) + } + return &big.Int{} // invalid type decodes to 0 value +} + +// Inf returns 1 if the value is +Inf, -1 if it is -Inf, 0 otherwise +func (i *IntOrInf) Inf() int { + switch i.Type { + case IntPlusInf: + return 1 + case IntMinusInf: + return -1 + } + return 0 // invalid type decodes to 0 value +} + +// Int64 limits the value between MinInt64 and MaxInt64 (even if it is +-Inf) and returns an int64 type +func (i *IntOrInf) Int64() int64 { + switch i.Type { + case IntNonNegative: + if i.Value.IsInt64() { + return i.Value.Int64() + } else { + return math.MaxInt64 + } + case IntNegative: + if i.Value.IsInt64() { + return -i.Value.Int64() + } else { + return math.MinInt64 + } + case IntPlusInf: + return math.MaxInt64 + case IntMinusInf: + return math.MinInt64 + } + return 0 // invalid type decodes to 0 value +} + +// SetBigInt sets the value to the given big.Int +func (i *IntOrInf) SetBigInt(v *big.Int) { + if v.Sign() >= 0 { + i.Type = IntNonNegative + i.Value.Set(v) + } else { + i.Type = IntNegative + i.Value.Neg(v) + } +} + +// SetInt64 sets the value to the given int64. Note that MaxInt64 translates to +Inf +// while MinInt64 translates to -Inf. +func (i *IntOrInf) SetInt64(v int64) { + if v >= 0 { + if v == math.MaxInt64 { + i.Type = IntPlusInf + } else { + i.Type = IntNonNegative + i.Value.SetInt64(v) + } + } else { + if v == math.MinInt64 { + i.Type = IntMinusInf + } else { + i.Type = IntNegative + i.Value.SetInt64(-v) + } + } +} + +// SetInf sets the value to +Inf or -Inf +func (i *IntOrInf) SetInf(sign int) { + if sign == 1 { + i.Type = IntPlusInf + } else { + i.Type = IntMinusInf + } +} diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index f5073d0db1..db12a5c573 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -243,11 +243,11 @@ func (n *NodeBalance) RequestServed(cost uint64) uint64 { } // Priority returns the actual priority based on the current balance -func (n *NodeBalance) Priority(now mclock.AbsTime, capacity uint64) int64 { +func (n *NodeBalance) Priority(capacity uint64) int64 { n.lock.Lock() defer n.lock.Unlock() - n.updateBalance(now) + n.updateBalance(n.bt.clock.Now()) return n.balanceToPriority(n.balance, capacity) } @@ -256,16 +256,35 @@ func (n *NodeBalance) Priority(now mclock.AbsTime, capacity uint64) int64 { // in the current session. // If update is true then a priority callback is added that turns UpdateFlag on and off // in case the priority goes below the estimated minimum. -func (n *NodeBalance) EstMinPriority(at mclock.AbsTime, capacity uint64, update bool) int64 { +func (n *NodeBalance) EstimatePriority(capacity uint64, addBalance int64, future, bias time.Duration, update bool) int64 { n.lock.Lock() defer n.lock.Unlock() - var avgReqCost float64 - dt := time.Duration(n.lastUpdate - n.initTime) - if dt > time.Second { - avgReqCost = float64(n.sumReqCost) * 2 / float64(dt) + now := n.bt.clock.Now() + n.updateBalance(now) + b := n.balance + if addBalance != 0 { + offset := n.bt.posExp.LogOffset(now) + old := n.balance.pos.Value(offset) + if addBalance > 0 && (addBalance > maxBalance || old > maxBalance-uint64(addBalance)) { + b.pos = utils.ExpiredValue{} + b.pos.Add(maxBalance, offset) + } else { + b.pos.Add(addBalance, offset) + } } - pri := n.balanceToPriority(n.reducedBalance(at, capacity, avgReqCost), capacity) + if future > 0 { + var avgReqCost float64 + dt := time.Duration(n.lastUpdate - n.initTime) + if dt > time.Second { + avgReqCost = float64(n.sumReqCost) * 2 / float64(dt) + } + b = n.reducedBalance(b, now, future, capacity, avgReqCost) + } + if bias > 0 { + b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) + } + pri := n.balanceToPriority(b, capacity) if update { n.addCallback(balanceCallbackUpdate, pri, n.signalPriorityUpdate) } @@ -366,7 +385,7 @@ func (n *NodeBalance) deactivate() { // updateBalance updates balance based on the time factor func (n *NodeBalance) updateBalance(now mclock.AbsTime) { if n.active && now > n.lastUpdate { - n.balance = n.reducedBalance(now, n.capacity, 0) + n.balance = n.reducedBalance(n.balance, n.lastUpdate, time.Duration(now-n.lastUpdate), n.capacity, 0) n.lastUpdate = now } } @@ -546,23 +565,25 @@ func (n *NodeBalance) balanceToPriority(b balance, capacity uint64) int64 { } // reducedBalance estimates the reduced balance at a given time in the fututre based -// on the current balance, the time factor and an estimated average request cost per time ratio -func (n *NodeBalance) reducedBalance(at mclock.AbsTime, capacity uint64, avgReqCost float64) balance { - dt := float64(at - n.lastUpdate) - b := n.balance +// on the given balance, the time factor and an estimated average request cost per time ratio +func (n *NodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { + // since the costs are applied continuously during the dt time period we calculate + // the expiration offset at the middle of the period + at := start + mclock.AbsTime(dt/2) + dtf := float64(dt) if !b.pos.IsZero() { factor := n.posFactor.timePrice(capacity) + n.posFactor.RequestFactor*avgReqCost - diff := -int64(dt * factor) + diff := -int64(dtf * factor) dd := b.pos.Add(diff, n.bt.posExp.LogOffset(at)) if dd == diff { - dt = 0 + dtf = 0 } else { - dt += float64(dd) / factor + dtf += float64(dd) / factor } } if dt > 0 { factor := n.negFactor.timePrice(capacity) + n.negFactor.RequestFactor*avgReqCost - b.neg.Add(int64(dt*factor), n.bt.negExp.LogOffset(at)) + b.neg.Add(int64(dtf*factor), n.bt.negExp.LogOffset(at)) } return b } @@ -588,8 +609,9 @@ func (n *NodeBalance) timeUntil(priority int64) (time.Duration, bool) { } dt = float64(posBalance-newBalance) / timePrice return time.Duration(dt), true + } else { + dt = float64(posBalance) / timePrice } - dt = float64(posBalance) / timePrice } else { if priority > 0 { return 0, false diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 6c817aa26c..e22074db2d 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -231,7 +231,7 @@ func TestBalanceToPriority(t *testing.T) { } for _, i := range inputs { node.SetBalance(i.pos, i.neg) - priority := node.Priority(b.clock.Now(), 1000) + priority := node.Priority(1000) if priority != i.priority { t.Fatalf("Priority mismatch, want %v, got %v", i.priority, priority) } @@ -272,7 +272,7 @@ func TestEstimatedPriority(t *testing.T) { for _, i := range inputs { b.clock.Run(i.runTime) node.RequestServed(i.reqCost) - priority := node.EstMinPriority(b.clock.Now()+mclock.AbsTime(i.futureTime), 1000000000, false) + priority := node.EstimatePriority(1000000000, 0, i.futureTime, 0, false) if priority != i.priority { t.Fatalf("Estimated priority mismatch, want %v, got %v", i.priority, priority) } diff --git a/les/vflux/server/prioritypool.go b/les/vflux/server/prioritypool.go index e3327aba75..e940ac7c65 100644 --- a/les/vflux/server/prioritypool.go +++ b/les/vflux/server/prioritypool.go @@ -101,17 +101,21 @@ type PriorityPool struct { minCap uint64 activeBias time.Duration capacityStepDiv uint64 + + cachedCurve *CapacityCurve + ccUpdatedAt mclock.AbsTime + ccUpdateForced bool } // nodePriority interface provides current and estimated future priorities on demand type nodePriority interface { // Priority should return the current priority of the node (higher is better) - Priority(now mclock.AbsTime, cap uint64) int64 + Priority(cap uint64) int64 // EstMinPriority should return a lower estimate for the minimum of the node priority // value starting from the current moment until the given time. If the priority goes // under the returned estimate before the specified moment then it is the caller's // responsibility to signal with updateFlag. - EstMinPriority(until mclock.AbsTime, cap uint64, update bool) int64 + EstimatePriority(cap uint64, addBalance int64, future, bias time.Duration, update bool) int64 } // ppNodeInfo is the internal node descriptor of PriorityPool @@ -131,12 +135,12 @@ func NewPriorityPool(ns *nodestate.NodeStateMachine, setup PriorityPoolSetup, cl ns: ns, PriorityPoolSetup: setup, clock: clock, - activeQueue: prque.NewLazyQueue(activeSetIndex, activePriority, activeMaxPriority, clock, lazyQueueRefresh), inactiveQueue: prque.New(inactiveSetIndex), minCap: minCap, activeBias: activeBias, capacityStepDiv: capacityStepDiv, } + pp.activeQueue = prque.NewLazyQueue(activeSetIndex, activePriority, pp.activeMaxPriority, clock, lazyQueueRefresh) ns.SubscribeField(pp.priorityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { if newValue != nil { @@ -197,6 +201,9 @@ func (pp *PriorityPool) RequestCapacity(node *enode.Node, targetCap uint64, bias if targetCap < pp.minCap { targetCap = pp.minCap } + if bias < pp.activeBias { + bias = pp.activeBias + } c, _ := pp.ns.GetField(node, pp.ppNodeInfoField).(*ppNodeInfo) if c == nil { log.Error("RequestCapacity called for unknown node", "id", node.ID()) @@ -204,9 +211,9 @@ func (pp *PriorityPool) RequestCapacity(node *enode.Node, targetCap uint64, bias } var priority int64 if targetCap > c.capacity { - priority = c.nodePriority.EstMinPriority(pp.clock.Now()+mclock.AbsTime(bias), targetCap, false) + priority = c.nodePriority.EstimatePriority(targetCap, 0, 0, bias, false) } else { - priority = c.nodePriority.Priority(pp.clock.Now(), targetCap) + priority = c.nodePriority.Priority(targetCap) } pp.markForChange(c) pp.setCapacity(c, targetCap) @@ -214,7 +221,7 @@ func (pp *PriorityPool) RequestCapacity(node *enode.Node, targetCap uint64, bias pp.activeQueue.Remove(c.activeIndex) pp.inactiveQueue.Remove(c.inactiveIndex) pp.activeQueue.Push(c) - minPriority = pp.enforceLimits() + _, minPriority = pp.enforceLimits() // if capacity update is possible now then minPriority == math.MinInt64 // if it is not possible at all then minPriority == math.MaxInt64 allowed = priority > minPriority @@ -281,29 +288,34 @@ func invertPriority(p int64) int64 { } // activePriority callback returns actual priority of ppNodeInfo item in activeQueue -func activePriority(a interface{}, now mclock.AbsTime) int64 { +func activePriority(a interface{}) int64 { c := a.(*ppNodeInfo) if c.forced { return math.MinInt64 } if c.bias == 0 { - return invertPriority(c.nodePriority.Priority(now, c.capacity)) + return invertPriority(c.nodePriority.Priority(c.capacity)) + } else { + return invertPriority(c.nodePriority.EstimatePriority(c.capacity, 0, 0, c.bias, true)) } - return invertPriority(c.nodePriority.EstMinPriority(now+mclock.AbsTime(c.bias), c.capacity, true)) } // activeMaxPriority callback returns estimated maximum priority of ppNodeInfo item in activeQueue -func activeMaxPriority(a interface{}, until mclock.AbsTime) int64 { +func (pp *PriorityPool) activeMaxPriority(a interface{}, until mclock.AbsTime) int64 { c := a.(*ppNodeInfo) if c.forced { return math.MinInt64 } - return invertPriority(c.nodePriority.EstMinPriority(until+mclock.AbsTime(c.bias), c.capacity, false)) + future := time.Duration(until - pp.clock.Now()) + if future < 0 { + future = 0 + } + return invertPriority(c.nodePriority.EstimatePriority(c.capacity, 0, future, c.bias, false)) } // inactivePriority callback returns actual priority of ppNodeInfo item in inactiveQueue func (pp *PriorityPool) inactivePriority(p *ppNodeInfo) int64 { - return p.nodePriority.Priority(pp.clock.Now(), pp.minCap) + return p.nodePriority.Priority(pp.minCap) } // connectedNode is called when a new node has been added to the pool (InactiveFlag set) @@ -379,16 +391,19 @@ func (pp *PriorityPool) setCapacity(n *ppNodeInfo, cap uint64) { // enforceLimits enforces active node count and total capacity limits. It returns the // lowest active node priority. Note that this function is performed on the temporary // internal state. -func (pp *PriorityPool) enforceLimits() int64 { +func (pp *PriorityPool) enforceLimits() (*ppNodeInfo, int64) { if pp.activeCap <= pp.maxCap && pp.activeCount <= pp.maxCount { - return math.MinInt64 + return nil, math.MinInt64 } - var maxActivePriority int64 + var ( + c *ppNodeInfo + maxActivePriority int64 + ) pp.activeQueue.MultiPop(func(data interface{}, priority int64) bool { - c := data.(*ppNodeInfo) + c = data.(*ppNodeInfo) pp.markForChange(c) maxActivePriority = priority - if c.capacity == pp.minCap { + if c.capacity == pp.minCap || pp.activeCount > pp.maxCount { pp.setCapacity(c, 0) } else { sub := c.capacity / pp.capacityStepDiv @@ -400,7 +415,7 @@ func (pp *PriorityPool) enforceLimits() int64 { } return pp.activeCap > pp.maxCap || pp.activeCount > pp.maxCount }) - return invertPriority(maxActivePriority) + return c, invertPriority(maxActivePriority) } // finalizeChanges either commits or reverts temporary changes. The necessary capacity @@ -430,6 +445,9 @@ func (pp *PriorityPool) finalizeChanges(commit bool) (updates []capUpdate) { c.origCap = 0 } pp.changed = nil + if commit { + pp.ccUpdateForced = true + } return } @@ -472,6 +490,7 @@ func (pp *PriorityPool) tryActivate() []capUpdate { break } } + pp.ccUpdateForced = true return pp.finalizeChanges(commit) } @@ -500,3 +519,150 @@ func (pp *PriorityPool) updatePriority(node *enode.Node) { } updates = pp.tryActivate() } + +// CapacityCurve is a snapshot of the priority pool contents in a format that can efficiently +// estimate how much capacity could be granted to a given node at a given priority level. +type CapacityCurve struct { + points []curvePoint // curve points sorted in descending order of priority + index map[enode.ID][]int // curve point indexes belonging to each node + exclude []int // curve point indexes of excluded node + excludeFirst bool // true if activeCount == maxCount +} + +type curvePoint struct { + freeCap uint64 // available capacity and node count at the current priority level + nextPri int64 // next priority level where more capacity will be available +} + +// GetCapacityCurve returns a new or recently cached CapacityCurve based on the contents of the pool +func (pp *PriorityPool) GetCapacityCurve() *CapacityCurve { + pp.lock.Lock() + defer pp.lock.Unlock() + + now := pp.clock.Now() + dt := time.Duration(now - pp.ccUpdatedAt) + if !pp.ccUpdateForced && pp.cachedCurve != nil && dt < time.Second*10 { + return pp.cachedCurve + } + + pp.ccUpdateForced = false + pp.ccUpdatedAt = now + curve := &CapacityCurve{ + index: make(map[enode.ID][]int), + } + pp.cachedCurve = curve + + var excludeID enode.ID + excludeFirst := pp.maxCount == pp.activeCount + // reduce node capacities or remove nodes until nothing is left in the queue; + // record the available capacity and the necessary priority after each step + for pp.activeCap > 0 { + cp := curvePoint{} + if pp.activeCap > pp.maxCap { + log.Error("Active capacity is greater than allowed maximum", "active", pp.activeCap, "maximum", pp.maxCap) + } else { + cp.freeCap = pp.maxCap - pp.activeCap + } + // temporarily increase activeCap to enforce reducing or removing a node capacity + tempCap := cp.freeCap + 1 + pp.activeCap += tempCap + var next *ppNodeInfo + // enforceLimits removes the lowest priority node if it has minimal capacity, + // otherwise reduces its capacity + next, cp.nextPri = pp.enforceLimits() + pp.activeCap -= tempCap + if next == nil { + log.Error("GetCapacityCurve: cannot remove next element from the priority queue") + break + } + id := next.node.ID() + if excludeFirst { + // if the node count limit is already reached then mark the node with the + // lowest priority for exclusion + curve.excludeFirst = true + excludeID = id + excludeFirst = false + } + // multiple curve points and therefore multiple indexes may belong to a node + // if it was removed in multiple steps (if its capacity was more than the minimum) + curve.index[id] = append(curve.index[id], len(curve.points)) + curve.points = append(curve.points, cp) + } + // restore original state of the queue + pp.finalizeChanges(false) + curve.points = append(curve.points, curvePoint{ + freeCap: pp.maxCap, + nextPri: math.MaxInt64, + }) + if curve.excludeFirst { + curve.exclude = curve.index[excludeID] + } + return curve +} + +// Exclude returns a CapacityCurve with the given node excluded from the original curve +func (cc *CapacityCurve) Exclude(id enode.ID) *CapacityCurve { + if exclude, ok := cc.index[id]; ok { + // return a new version of the curve (only one excluded node can be selected) + // Note: if the first node was excluded by default (excludeFirst == true) then + // we can forget about that and exclude the node with the given id instead. + return &CapacityCurve{ + points: cc.points, + index: cc.index, + exclude: exclude, + } + } + return cc +} + +func (cc *CapacityCurve) getPoint(i int) curvePoint { + cp := cc.points[i] + if i == 0 && cc.excludeFirst { + cp.freeCap = 0 + return cp + } + for ii := len(cc.exclude) - 1; ii >= 0; ii-- { + ei := cc.exclude[ii] + if ei < i { + break + } + e1, e2 := cc.points[ei], cc.points[ei+1] + cp.freeCap += e2.freeCap - e1.freeCap + } + return cp +} + +// MaxCapacity calculates the maximum capacity available for a node with a given +// (monotonically decreasing) priority vs. capacity function. Note that if the requesting +// node is already in the pool then it should be excluded from the curve in order to get +// the correct result. +func (cc *CapacityCurve) MaxCapacity(priority func(cap uint64) int64) uint64 { + min, max := 0, len(cc.points)-1 // the curve always has at least one point + for min < max { + mid := (min + max) / 2 + cp := cc.getPoint(mid) + if cp.freeCap == 0 || priority(cp.freeCap) > cp.nextPri { + min = mid + 1 + } else { + max = mid + } + } + cp2 := cc.getPoint(min) + if cp2.freeCap == 0 || min == 0 { + return cp2.freeCap + } + cp1 := cc.getPoint(min - 1) + if priority(cp2.freeCap) > cp1.nextPri { + return cp2.freeCap + } + minc, maxc := cp1.freeCap, cp2.freeCap-1 + for minc < maxc { + midc := (minc + maxc + 1) / 2 + if midc == 0 || priority(midc) > cp1.nextPri { + minc = midc + } else { + maxc = midc - 1 + } + } + return maxc +} diff --git a/les/vflux/server/prioritypool_test.go b/les/vflux/server/prioritypool_test.go index cbb3f5b372..d83ddc1767 100644 --- a/les/vflux/server/prioritypool_test.go +++ b/les/vflux/server/prioritypool_test.go @@ -20,6 +20,7 @@ import ( "math/rand" "reflect" "testing" + "time" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/p2p/enode" @@ -42,6 +43,7 @@ func init() { const ( testCapacityStepDiv = 100 testCapacityToleranceDiv = 10 + testMinCap = 100 ) type ppTestClient struct { @@ -49,11 +51,11 @@ type ppTestClient struct { balance, cap uint64 } -func (c *ppTestClient) Priority(now mclock.AbsTime, cap uint64) int64 { +func (c *ppTestClient) Priority(cap uint64) int64 { return int64(c.balance / cap) } -func (c *ppTestClient) EstMinPriority(until mclock.AbsTime, cap uint64, update bool) int64 { +func (c *ppTestClient) EstimatePriority(cap uint64, addBalance int64, future, bias time.Duration, update bool) int64 { return int64(c.balance / cap) } @@ -67,7 +69,7 @@ func TestPriorityPool(t *testing.T) { c.cap = newValue.(uint64) } }) - pp := NewPriorityPool(ns, ppTestSetup, clock, 100, 0, testCapacityStepDiv) + pp := NewPriorityPool(ns, ppTestSetup, clock, testMinCap, 0, testCapacityStepDiv) ns.Start() pp.SetLimits(100, 1000000) clients := make([]*ppTestClient, 100) @@ -94,7 +96,7 @@ func TestPriorityPool(t *testing.T) { for i := range clients { c := &ppTestClient{ node: enode.SignNull(&enr.Record{}, enode.ID{byte(i)}), - balance: 1000000000, + balance: 100000000000, cap: 1000, } sumBalance += c.balance @@ -109,7 +111,7 @@ func TestPriorityPool(t *testing.T) { for count := 0; count < 100; count++ { c := clients[rand.Intn(len(clients))] oldBalance := c.balance - c.balance = uint64(rand.Int63n(1000000000) + 1000000000) + c.balance = uint64(rand.Int63n(100000000000) + 100000000000) sumBalance += c.balance - oldBalance pp.ns.SetState(c.node, ppUpdateFlag, nodestate.Flags{}, 0) pp.ns.SetState(c.node, nodestate.Flags{}, ppUpdateFlag, 0) @@ -120,10 +122,124 @@ func TestPriorityPool(t *testing.T) { raise(c) } } + // check whether capacities are proportional to balances for _, c := range clients { check(c) } + if count%10 == 0 { + // test available capacity calculation with capacity curve + c = clients[rand.Intn(len(clients))] + curve := pp.GetCapacityCurve().Exclude(c.node.ID()) + + add := uint64(rand.Int63n(10000000000000)) + c.balance += add + sumBalance += add + expCap := curve.MaxCapacity(func(cap uint64) int64 { + return int64(c.balance / cap) + }) + //fmt.Println(expCap, c.balance, sumBalance) + /*for i, cp := range curve.points { + fmt.Println("cp", i, cp, "ex", curve.getPoint(i)) + }*/ + var ok bool + expFail := expCap + 1 + if expFail < testMinCap { + expFail = testMinCap + } + ns.Operation(func() { + _, ok = pp.RequestCapacity(c.node, expFail, 0, true) + }) + if ok { + t.Errorf("Request for more than expected available capacity succeeded") + } + if expCap >= testMinCap { + ns.Operation(func() { + _, ok = pp.RequestCapacity(c.node, expCap, 0, true) + }) + if !ok { + t.Errorf("Request for expected available capacity failed") + } + } + c.balance -= add + sumBalance -= add + pp.ns.SetState(c.node, ppUpdateFlag, nodestate.Flags{}, 0) + pp.ns.SetState(c.node, nodestate.Flags{}, ppUpdateFlag, 0) + for _, c := range clients { + raise(c) + } + } } ns.Stop() } + +func TestCapacityCurve(t *testing.T) { + clock := &mclock.Simulated{} + ns := nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) + pp := NewPriorityPool(ns, ppTestSetup, clock, 400000, 0, 2) + ns.Start() + pp.SetLimits(10, 10000000) + clients := make([]*ppTestClient, 10) + + for i := range clients { + c := &ppTestClient{ + node: enode.SignNull(&enr.Record{}, enode.ID{byte(i)}), + balance: 100000000000 * uint64(i+1), + cap: 1000000, + } + clients[i] = c + ns.SetState(c.node, ppTestClientFlag, nodestate.Flags{}, 0) + ns.SetField(c.node, ppTestSetup.priorityField, c) + ns.SetState(c.node, ppTestSetup.InactiveFlag, nodestate.Flags{}, 0) + ns.Operation(func() { + pp.RequestCapacity(c.node, c.cap, 0, true) + }) + } + + curve := pp.GetCapacityCurve() + check := func(balance, expCap uint64) { + cap := curve.MaxCapacity(func(cap uint64) int64 { + return int64(balance / cap) + }) + var fail bool + if cap == 0 || expCap == 0 { + fail = cap != expCap + } else { + pri := balance / cap + expPri := balance / expCap + fail = pri != expPri && pri != expPri+1 + } + if fail { + t.Errorf("Incorrect capacity for %d balance (got %d, expected %d)", balance, cap, expCap) + } + } + + check(0, 0) + check(10000000000, 100000) + check(50000000000, 500000) + check(100000000000, 1000000) + check(200000000000, 1000000) + check(300000000000, 1500000) + check(450000000000, 1500000) + check(600000000000, 2000000) + check(800000000000, 2000000) + check(1000000000000, 2500000) + + pp.SetLimits(11, 10000000) + curve = pp.GetCapacityCurve() + + check(0, 0) + check(10000000000, 100000) + check(50000000000, 500000) + check(150000000000, 750000) + check(200000000000, 1000000) + check(220000000000, 1100000) + check(275000000000, 1100000) + check(375000000000, 1500000) + check(450000000000, 1500000) + check(600000000000, 2000000) + check(800000000000, 2000000) + check(1000000000000, 2500000) + + ns.Stop() +} diff --git a/les/vflux/server/service.go b/les/vflux/server/service.go new file mode 100644 index 0000000000..ab759ae441 --- /dev/null +++ b/les/vflux/server/service.go @@ -0,0 +1,122 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package server + +import ( + "net" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/les/utils" + "github.com/ethereum/go-ethereum/les/vflux" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" +) + +type ( + // Server serves vflux requests + Server struct { + limiter *utils.Limiter + lock sync.Mutex + services map[string]*serviceEntry + delayPerRequest time.Duration + } + + // Service is a service registered at the Server and identified by a string id + Service interface { + ServiceInfo() (id, desc string) // only called during registration + Handle(id enode.ID, address string, name string, data []byte) []byte // never called concurrently + } + + serviceEntry struct { + id, desc string + backend Service + } +) + +// NewServer creates a new Server +func NewServer(delayPerRequest time.Duration) *Server { + return &Server{ + limiter: utils.NewLimiter(1000), + delayPerRequest: delayPerRequest, + services: make(map[string]*serviceEntry), + } +} + +// Register registers a Service +func (s *Server) Register(b Service) { + srv := &serviceEntry{backend: b} + srv.id, srv.desc = b.ServiceInfo() + if strings.Contains(srv.id, ":") { + // srv.id + ":" will be used as a service database prefix + log.Error("Service ID contains ':'", "id", srv.id) + return + } + s.lock.Lock() + s.services[srv.id] = srv + s.lock.Unlock() +} + +// Serve serves a vflux request batch +// Note: requests are served by the Handle functions of the registered services. Serve +// may be called concurrently but the Handle functions are called sequentially and +// therefore thread safety is guaranteed. +func (s *Server) Serve(id enode.ID, address string, requests vflux.Requests) vflux.Replies { + reqLen := uint(len(requests)) + if reqLen == 0 || reqLen > vflux.MaxRequestLength { + return nil + } + // Note: the value parameter will be supplied by the token sale module (total amount paid) + ch := <-s.limiter.Add(id, address, 0, reqLen) + if ch == nil { + return nil + } + // Note: the limiter ensures that the following section is not running concurrently, + // the lock only protects against contention caused by new service registration + s.lock.Lock() + results := make(vflux.Replies, len(requests)) + for i, req := range requests { + if service := s.services[req.Service]; service != nil { + results[i] = service.backend.Handle(id, address, req.Name, req.Params) + } + } + s.lock.Unlock() + time.Sleep(s.delayPerRequest * time.Duration(reqLen)) + close(ch) + return results +} + +// ServeEncoded serves an encoded vflux request batch and returns the encoded replies +func (s *Server) ServeEncoded(id enode.ID, addr *net.UDPAddr, req []byte) []byte { + var requests vflux.Requests + if err := rlp.DecodeBytes(req, &requests); err != nil { + return nil + } + results := s.Serve(id, addr.String(), requests) + if results == nil { + return nil + } + res, _ := rlp.EncodeToBytes(&results) + return res +} + +// Stop shuts down the server +func (s *Server) Stop() { + s.limiter.Stop() +} diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 9dd2b31733..eb01d95e93 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -74,7 +74,7 @@ type UDPv5 struct { // talkreq handler registry trlock sync.Mutex - trhandlers map[string]func([]byte) []byte + trhandlers map[string]TalkRequestHandler // channels into dispatch packetInCh chan ReadPacket @@ -96,6 +96,9 @@ type UDPv5 struct { wg sync.WaitGroup } +// TalkRequestHandler callback processes a talk request and optionally returns a reply +type TalkRequestHandler func(enode.ID, *net.UDPAddr, []byte) []byte + // callV5 represents a remote procedure call against another node. type callV5 struct { node *enode.Node @@ -145,7 +148,7 @@ func newUDPv5(conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { log: cfg.Log, validSchemes: cfg.ValidSchemes, clock: cfg.Clock, - trhandlers: make(map[string]func([]byte) []byte), + trhandlers: make(map[string]TalkRequestHandler), // channels into dispatch packetInCh: make(chan ReadPacket, 1), readNextCh: make(chan struct{}, 1), @@ -233,7 +236,7 @@ func (t *UDPv5) LocalNode() *enode.LocalNode { // RegisterTalkHandler adds a handler for 'talk requests'. The handler function is called // whenever a request for the given protocol is received and should return the response // data or nil. -func (t *UDPv5) RegisterTalkHandler(protocol string, handler func([]byte) []byte) { +func (t *UDPv5) RegisterTalkHandler(protocol string, handler TalkRequestHandler) { t.trlock.Lock() defer t.trlock.Unlock() t.trhandlers[protocol] = handler @@ -841,7 +844,7 @@ func (t *UDPv5) handleTalkRequest(p *v5wire.TalkRequest, fromID enode.ID, fromAd var response []byte if handler != nil { - response = handler(p.Message) + response = handler(fromID, fromAddr, p.Message) } resp := &v5wire.TalkResponse{ReqID: p.ReqID, Message: response} t.sendResponse(fromID, fromAddr, resp) diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index d91a2097db..292785bd51 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -435,7 +435,7 @@ func TestUDPv5_talkHandling(t *testing.T) { defer test.close() var recvMessage []byte - test.udp.RegisterTalkHandler("test", func(message []byte) []byte { + test.udp.RegisterTalkHandler("test", func(id enode.ID, addr *net.UDPAddr, message []byte) []byte { recvMessage = message return []byte("test response") }) diff --git a/p2p/nodestate/nodestate.go b/p2p/nodestate/nodestate.go index def93bac43..d3166f1d87 100644 --- a/p2p/nodestate/nodestate.go +++ b/p2p/nodestate/nodestate.go @@ -599,6 +599,7 @@ func (ns *NodeStateMachine) updateEnode(n *enode.Node) (enode.ID, *nodeInfo) { node := ns.nodes[id] if node != nil && n.Seq() > node.node.Seq() { node.node = n + node.dirty = true } return id, node } From 19d7a37abb9f3b9bf1a94baf6bd8c7d5042e54f8 Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 1 Mar 2021 17:26:10 +0800 Subject: [PATCH 170/709] core/rawdb: fix the transaction indexer (#22395) --- core/rawdb/chain_iterator.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 393b72c26c..862a549540 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -243,13 +243,13 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan } } } - // If there exists uncommitted data, flush them. - if batch.ValueSize() > 0 { - WriteTxIndexTail(batch, lastNum) // Also write the tail there - if err := batch.Write(); err != nil { - log.Crit("Failed writing batch to db", "error", err) - return - } + // Flush the new indexing tail and the last committed data. It can also happen + // that the last batch is empty because nothing to index, but the tail has to + // be flushed anyway. + WriteTxIndexTail(batch, lastNum) + if err := batch.Write(); err != nil { + log.Crit("Failed writing batch to db", "error", err) + return } select { case <-interrupt: @@ -334,13 +334,13 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } } } - // Commit the last batch if there exists uncommitted data - if batch.ValueSize() > 0 { - WriteTxIndexTail(batch, nextNum) - if err := batch.Write(); err != nil { - log.Crit("Failed writing batch to db", "error", err) - return - } + // Flush the new indexing tail and the last committed data. It can also happen + // that the last batch is empty because nothing to unindex, but the tail has to + // be flushed anyway. + WriteTxIndexTail(batch, nextNum) + if err := batch.Write(); err != nil { + log.Crit("Failed writing batch to db", "error", err) + return } select { case <-interrupt: From 7834e4a278038e57b741ee826b3a46ff9d809fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 2 Mar 2021 23:40:57 +0200 Subject: [PATCH 171/709] core, eth: unship EIP 2315 --- core/vm/contract.go | 13 ------------- core/vm/eips.go | 29 ----------------------------- core/vm/gen_structlog.go | 13 ------------- core/vm/instructions.go | 32 -------------------------------- core/vm/instructions_test.go | 33 ++++++++++++++++----------------- core/vm/interpreter.go | 16 ++++++---------- core/vm/jump_table.go | 1 - core/vm/logger.go | 35 +++++++---------------------------- core/vm/logger_json.go | 5 ++--- core/vm/logger_test.go | 3 +-- core/vm/opcodes.go | 34 ++++++++++++---------------------- core/vm/stack.go | 31 ------------------------------- eth/tracers/tracer.go | 4 ++-- eth/tracers/tracer_test.go | 4 ++-- 14 files changed, 48 insertions(+), 205 deletions(-) diff --git a/core/vm/contract.go b/core/vm/contract.go index 915193d137..61dbd5007a 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -96,19 +96,6 @@ func (c *Contract) validJumpdest(dest *uint256.Int) bool { return c.isCode(udest) } -func (c *Contract) validJumpSubdest(udest uint64) bool { - // PC cannot go beyond len(code) and certainly can't be bigger than 63 bits. - // Don't bother checking for BEGINSUB in that case. - if int64(udest) < 0 || udest >= uint64(len(c.Code)) { - return false - } - // Only BEGINSUBs allowed for destinations - if OpCode(c.Code[udest]) != BEGINSUB { - return false - } - return c.isCode(udest) -} - // isCode returns true if the provided PC location is an actual opcode, as // opposed to a data-segment following a PUSHN operation. func (c *Contract) isCode(udest uint64) bool { diff --git a/core/vm/eips.go b/core/vm/eips.go index 962c0f14b1..0c8bf1792e 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -29,7 +29,6 @@ var activators = map[int]func(*JumpTable){ 2200: enable2200, 1884: enable1884, 1344: enable1344, - 2315: enable2315, } // EnableEIP enables the given EIP on the config. @@ -108,34 +107,6 @@ func enable2200(jt *JumpTable) { jt[SSTORE].dynamicGas = gasSStoreEIP2200 } -// enable2315 applies EIP-2315 (Simple Subroutines) -// - Adds opcodes that jump to and return from subroutines -func enable2315(jt *JumpTable) { - // New opcode - jt[BEGINSUB] = &operation{ - execute: opBeginSub, - constantGas: GasQuickStep, - minStack: minStack(0, 0), - maxStack: maxStack(0, 0), - } - // New opcode - jt[JUMPSUB] = &operation{ - execute: opJumpSub, - constantGas: GasSlowStep, - minStack: minStack(1, 0), - maxStack: maxStack(1, 0), - jumps: true, - } - // New opcode - jt[RETURNSUB] = &operation{ - execute: opReturnSub, - constantGas: GasFastStep, - minStack: minStack(0, 0), - maxStack: maxStack(0, 0), - jumps: true, - } -} - // enable2929 enables "EIP-2929: Gas cost increases for state access opcodes" // https://eips.ethereum.org/EIPS/eip-2929 func enable2929(jt *JumpTable) { diff --git a/core/vm/gen_structlog.go b/core/vm/gen_structlog.go index 44da014de9..ac04afe8b7 100644 --- a/core/vm/gen_structlog.go +++ b/core/vm/gen_structlog.go @@ -45,12 +45,6 @@ func (s StructLog) MarshalJSON() ([]byte, error) { enc.Stack[k] = (*math.HexOrDecimal256)(v) } } - if s.ReturnStack != nil { - enc.ReturnStack = make([]math.HexOrDecimal64, len(s.ReturnStack)) - for k, v := range s.ReturnStack { - enc.ReturnStack[k] = math.HexOrDecimal64(v) - } - } enc.ReturnData = s.ReturnData enc.Storage = s.Storage enc.Depth = s.Depth @@ -71,7 +65,6 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { Memory *hexutil.Bytes `json:"memory"` MemorySize *int `json:"memSize"` Stack []*math.HexOrDecimal256 `json:"stack"` - ReturnStack []math.HexOrDecimal64 `json:"returnStack"` ReturnData *hexutil.Bytes `json:"returnData"` Storage map[common.Hash]common.Hash `json:"-"` Depth *int `json:"depth"` @@ -106,12 +99,6 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { s.Stack[k] = (*big.Int)(v) } } - if dec.ReturnStack != nil { - s.ReturnStack = make([]uint32, len(dec.ReturnStack)) - for k, v := range dec.ReturnStack { - s.ReturnStack[k] = uint32(v) - } - } if dec.ReturnData != nil { s.ReturnData = *dec.ReturnData } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 1137505292..f4ca0603ed 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -547,38 +547,6 @@ func opJumpdest(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ( return nil, nil } -func opBeginSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - return nil, ErrInvalidSubroutineEntry -} - -func opJumpSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - if len(callContext.rstack.data) >= 1023 { - return nil, ErrReturnStackExceeded - } - pos := callContext.stack.pop() - if !pos.IsUint64() { - return nil, ErrInvalidJump - } - posU64 := pos.Uint64() - if !callContext.contract.validJumpSubdest(posU64) { - return nil, ErrInvalidJump - } - callContext.rstack.push(uint32(*pc)) - *pc = posU64 + 1 - return nil, nil -} - -func opReturnSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - if len(callContext.rstack.data) == 0 { - return nil, ErrInvalidRetsub - } - // Other than the check that the return stack is not empty, there is no - // need to validate the pc from 'returns', since we only ever push valid - //values onto it via jumpsub. - *pc = uint64(callContext.rstack.pop()) + 1 - return nil, nil -} - func opPc(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { callContext.stack.push(new(uint256.Int).SetUint64(*pc)) return nil, nil diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 985d5a5156..55d876581c 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -94,7 +94,6 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) stack = newstack() - rstack = newReturnStack() pc = uint64(0) evmInterpreter = env.interpreter.(*EVMInterpreter) ) @@ -105,7 +104,7 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu expected := new(uint256.Int).SetBytes(common.Hex2Bytes(test.Expected)) stack.push(x) stack.push(y) - opFn(&pc, evmInterpreter, &callCtx{nil, stack, rstack, nil}) + opFn(&pc, evmInterpreter, &callCtx{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", name, len(stack.data)) } @@ -220,7 +219,7 @@ func TestAddMod(t *testing.T) { stack.push(z) stack.push(y) stack.push(x) - opAddmod(&pc, evmInterpreter, &callCtx{nil, stack, nil, nil}) + opAddmod(&pc, evmInterpreter, &callCtx{nil, stack, nil}) actual := stack.pop() if actual.Cmp(expected) != 0 { t.Errorf("Testcase %d, expected %x, got %x", i, expected, actual) @@ -231,10 +230,10 @@ func TestAddMod(t *testing.T) { // getResult is a convenience function to generate the expected values func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack, rstack = newstack(), newReturnStack() - pc = uint64(0) - interpreter = env.interpreter.(*EVMInterpreter) + env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) + interpreter = env.interpreter.(*EVMInterpreter) ) result := make([]TwoOperandTestcase, len(args)) for i, param := range args { @@ -242,7 +241,7 @@ func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcas y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) stack.push(x) stack.push(y) - opFn(&pc, interpreter, &callCtx{nil, stack, rstack, nil}) + opFn(&pc, interpreter, &callCtx{nil, stack, nil}) actual := stack.pop() result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} } @@ -282,7 +281,7 @@ func TestJsonTestcases(t *testing.T) { func opBenchmark(bench *testing.B, op executionFunc, args ...string) { var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack, rstack = newstack(), newReturnStack() + stack = newstack() evmInterpreter = NewEVMInterpreter(env, env.vmConfig) ) @@ -300,7 +299,7 @@ func opBenchmark(bench *testing.B, op executionFunc, args ...string) { a.SetBytes(arg) stack.push(a) } - op(&pc, evmInterpreter, &callCtx{nil, stack, rstack, nil}) + op(&pc, evmInterpreter, &callCtx{nil, stack, nil}) stack.pop() } } @@ -516,7 +515,7 @@ func BenchmarkOpIsZero(b *testing.B) { func TestOpMstore(t *testing.T) { var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack, rstack = newstack(), newReturnStack() + stack = newstack() mem = NewMemory() evmInterpreter = NewEVMInterpreter(env, env.vmConfig) ) @@ -526,12 +525,12 @@ func TestOpMstore(t *testing.T) { pc := uint64(0) v := "abcdef00000000000000abba000000000deaf000000c0de00100000000133700" stack.pushN(*new(uint256.Int).SetBytes(common.Hex2Bytes(v)), *new(uint256.Int)) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) + opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) if got := common.Bytes2Hex(mem.GetCopy(0, 32)); got != v { t.Fatalf("Mstore fail, got %v, expected %v", got, v) } stack.pushN(*new(uint256.Int).SetUint64(0x1), *new(uint256.Int)) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) + opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) if common.Bytes2Hex(mem.GetCopy(0, 32)) != "0000000000000000000000000000000000000000000000000000000000000001" { t.Fatalf("Mstore failed to overwrite previous value") } @@ -540,7 +539,7 @@ func TestOpMstore(t *testing.T) { func BenchmarkOpMstore(bench *testing.B) { var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack, rstack = newstack(), newReturnStack() + stack = newstack() mem = NewMemory() evmInterpreter = NewEVMInterpreter(env, env.vmConfig) ) @@ -554,14 +553,14 @@ func BenchmarkOpMstore(bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { stack.pushN(*value, *memStart) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) + opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) } } func BenchmarkOpSHA3(bench *testing.B) { var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack, rstack = newstack(), newReturnStack() + stack = newstack() mem = NewMemory() evmInterpreter = NewEVMInterpreter(env, env.vmConfig) ) @@ -573,7 +572,7 @@ func BenchmarkOpSHA3(bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { stack.pushN(*uint256.NewInt().SetUint64(32), *start) - opSha3(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) + opSha3(&pc, evmInterpreter, &callCtx{mem, stack, nil}) } } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 0084b7d071..06a3b962b2 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -67,7 +67,6 @@ type Interpreter interface { type callCtx struct { memory *Memory stack *Stack - rstack *ReturnStack contract *Contract } @@ -161,14 +160,12 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } var ( - op OpCode // current opcode - mem = NewMemory() // bound memory - stack = newstack() // local stack - returns = newReturnStack() // local returns stack + op OpCode // current opcode + mem = NewMemory() // bound memory + stack = newstack() // local stack callContext = &callCtx{ memory: mem, stack: stack, - rstack: returns, contract: contract, } // For optimisation reason we're using uint64 as the program counter. @@ -187,7 +184,6 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( // they are returned to the pools defer func() { returnStack(stack) - returnRStack(returns) }() contract.Input = input @@ -195,9 +191,9 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer func() { if err != nil { if !logged { - in.cfg.Tracer.CaptureState(in.evm, pcCopy, op, gasCopy, cost, mem, stack, returns, in.returnData, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pcCopy, op, gasCopy, cost, mem, stack, in.returnData, contract, in.evm.depth, err) } else { - in.cfg.Tracer.CaptureFault(in.evm, pcCopy, op, gasCopy, cost, mem, stack, returns, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureFault(in.evm, pcCopy, op, gasCopy, cost, mem, stack, contract, in.evm.depth, err) } } }() @@ -279,7 +275,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } if in.cfg.Debug { - in.cfg.Tracer.CaptureState(in.evm, pc, op, gasCopy, cost, mem, stack, returns, in.returnData, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pc, op, gasCopy, cost, mem, stack, in.returnData, contract, in.evm.depth, err) logged = true } diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index d831f9300f..7b61762456 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -66,7 +66,6 @@ type JumpTable [256]*operation // contantinople, istanbul, petersburg and berlin instructions. func newBerlinInstructionSet() JumpTable { instructionSet := newIstanbulInstructionSet() - enable2315(&instructionSet) // Subroutines - https://eips.ethereum.org/EIPS/eip-2315 enable2929(&instructionSet) // Access lists for trie accesses https://eips.ethereum.org/EIPS/eip-2929 return instructionSet } diff --git a/core/vm/logger.go b/core/vm/logger.go index 962be6ec8e..41ce00ed01 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -70,7 +70,6 @@ type StructLog struct { Memory []byte `json:"memory"` MemorySize int `json:"memSize"` Stack []*big.Int `json:"stack"` - ReturnStack []uint32 `json:"returnStack"` ReturnData []byte `json:"returnData"` Storage map[common.Hash]common.Hash `json:"-"` Depth int `json:"depth"` @@ -81,7 +80,6 @@ type StructLog struct { // overrides for gencodec type structLogMarshaling struct { Stack []*math.HexOrDecimal256 - ReturnStack []math.HexOrDecimal64 Gas math.HexOrDecimal64 GasCost math.HexOrDecimal64 Memory hexutil.Bytes @@ -110,8 +108,8 @@ func (s *StructLog) ErrorString() string { // if you need to retain them beyond the current call. type Tracer interface { CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error - CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, rData []byte, contract *Contract, depth int, err error) error - CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error + CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error + CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error } @@ -148,7 +146,7 @@ func (l *StructLogger) CaptureStart(from common.Address, to common.Address, crea // CaptureState logs a new structured log message and pushes it out to the environment // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. -func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, rData []byte, contract *Contract, depth int, err error) error { +func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error { // check if already accumulated the specified number of logs if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) { return errTraceLimitReached @@ -167,11 +165,6 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui stck[i] = new(big.Int).Set(item.ToBig()) } } - var rstack []uint32 - if !l.cfg.DisableStack && rStack != nil { - rstck := make([]uint32, len(rStack.data)) - copy(rstck, rStack.data) - } // Copy a snapshot of the current storage to a new container var storage Storage if !l.cfg.DisableStorage { @@ -204,14 +197,14 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui copy(rdata, rData) } // create a new snapshot of the EVM. - log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rstack, rdata, storage, depth, env.StateDB.GetRefund(), err} + log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rdata, storage, depth, env.StateDB.GetRefund(), err} l.logs = append(l.logs, log) return nil } // CaptureFault implements the Tracer interface to trace an execution fault // while running an opcode. -func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { +func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { return nil } @@ -252,12 +245,6 @@ func WriteTrace(writer io.Writer, logs []StructLog) { fmt.Fprintf(writer, "%08d %x\n", len(log.Stack)-i-1, math.PaddedBigBytes(log.Stack[i], 32)) } } - if len(log.ReturnStack) > 0 { - fmt.Fprintln(writer, "ReturnStack:") - for i := len(log.Stack) - 1; i >= 0; i-- { - fmt.Fprintf(writer, "%08d 0x%x (%d)\n", len(log.Stack)-i-1, log.ReturnStack[i], log.ReturnStack[i]) - } - } if len(log.Memory) > 0 { fmt.Fprintln(writer, "Memory:") fmt.Fprint(writer, hex.Dump(log.Memory)) @@ -323,7 +310,7 @@ func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create b return nil } -func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, rData []byte, contract *Contract, depth int, err error) error { +func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error { fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, op, cost) if !t.cfg.DisableStack { @@ -334,14 +321,6 @@ func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64 } b := fmt.Sprintf("[%v]", strings.Join(a, ",")) fmt.Fprintf(t.out, "%10v |", b) - - // format return stack - a = a[:0] - for _, elem := range rStack.data { - a = append(a, fmt.Sprintf("%2d", elem)) - } - b = fmt.Sprintf("[%v]", strings.Join(a, ",")) - fmt.Fprintf(t.out, "%10v |", b) } fmt.Fprintf(t.out, "%10v |", env.StateDB.GetRefund()) fmt.Fprintln(t.out, "") @@ -351,7 +330,7 @@ func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64 return nil } -func (t *mdLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { +func (t *mdLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { fmt.Fprintf(t.out, "\nError: at pc=%d, op=%v: %v\n", pc, op, err) diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index 5f3f2c42f7..a27c261ed8 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -46,7 +46,7 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create } // CaptureState outputs state information on the logger. -func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, rData []byte, contract *Contract, depth int, err error) error { +func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error { log := StructLog{ Pc: pc, Op: op, @@ -68,7 +68,6 @@ func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint logstack[i] = item.ToBig() } log.Stack = logstack - log.ReturnStack = rStack.data } if !l.cfg.DisableReturnData { log.ReturnData = rData @@ -77,7 +76,7 @@ func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint } // CaptureFault outputs state information on the logger. -func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { +func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { return nil } diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index bf7d5358f8..5a5f42fd34 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -55,13 +55,12 @@ func TestStoreCapture(t *testing.T) { logger = NewStructLogger(nil) mem = NewMemory() stack = newstack() - rstack = newReturnStack() contract = NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 0) ) stack.push(uint256.NewInt().SetUint64(1)) stack.push(uint256.NewInt()) var index common.Hash - logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, rstack, nil, contract, 0, nil) + logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, nil, contract, 0, nil) if len(logger.storage[contract.Address()]) == 0 { t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.storage[contract.Address()])) } diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index da7b2ee4aa..b0adf37d0c 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -107,21 +107,18 @@ const ( // 0x50 range - 'storage' and execution. const ( - POP OpCode = 0x50 - MLOAD OpCode = 0x51 - MSTORE OpCode = 0x52 - MSTORE8 OpCode = 0x53 - SLOAD OpCode = 0x54 - SSTORE OpCode = 0x55 - JUMP OpCode = 0x56 - JUMPI OpCode = 0x57 - PC OpCode = 0x58 - MSIZE OpCode = 0x59 - GAS OpCode = 0x5a - JUMPDEST OpCode = 0x5b - BEGINSUB OpCode = 0x5c - RETURNSUB OpCode = 0x5d - JUMPSUB OpCode = 0x5e + POP OpCode = 0x50 + MLOAD OpCode = 0x51 + MSTORE OpCode = 0x52 + MSTORE8 OpCode = 0x53 + SLOAD OpCode = 0x54 + SSTORE OpCode = 0x55 + JUMP OpCode = 0x56 + JUMPI OpCode = 0x57 + PC OpCode = 0x58 + MSIZE OpCode = 0x59 + GAS OpCode = 0x5a + JUMPDEST OpCode = 0x5b ) // 0x60 range. @@ -300,10 +297,6 @@ var opCodeToString = map[OpCode]string{ GAS: "GAS", JUMPDEST: "JUMPDEST", - BEGINSUB: "BEGINSUB", - JUMPSUB: "JUMPSUB", - RETURNSUB: "RETURNSUB", - // 0x60 range - push. PUSH1: "PUSH1", PUSH2: "PUSH2", @@ -468,9 +461,6 @@ var stringToOp = map[string]OpCode{ "MSIZE": MSIZE, "GAS": GAS, "JUMPDEST": JUMPDEST, - "BEGINSUB": BEGINSUB, - "RETURNSUB": RETURNSUB, - "JUMPSUB": JUMPSUB, "PUSH1": PUSH1, "PUSH2": PUSH2, "PUSH3": PUSH3, diff --git a/core/vm/stack.go b/core/vm/stack.go index af27d6552c..c71d2653a5 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -98,34 +98,3 @@ func (st *Stack) Print() { } fmt.Println("#############") } - -var rStackPool = sync.Pool{ - New: func() interface{} { - return &ReturnStack{data: make([]uint32, 0, 10)} - }, -} - -// ReturnStack is an object for basic return stack operations. -type ReturnStack struct { - data []uint32 -} - -func newReturnStack() *ReturnStack { - return rStackPool.Get().(*ReturnStack) -} - -func returnRStack(rs *ReturnStack) { - rs.data = rs.data[:0] - rStackPool.Put(rs) -} - -func (st *ReturnStack) push(d uint32) { - st.data = append(st.data, d) -} - -// A uint32 is sufficient as for code below 4.2G -func (st *ReturnStack) pop() (ret uint32) { - ret = st.data[len(st.data)-1] - st.data = st.data[:len(st.data)-1] - return -} diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index 80775caa8e..4c398edf34 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -544,7 +544,7 @@ func (jst *Tracer) CaptureStart(from common.Address, to common.Address, create b } // CaptureState implements the Tracer interface to trace a single step of VM execution. -func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, rdata []byte, contract *vm.Contract, depth int, err error) error { +func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rdata []byte, contract *vm.Contract, depth int, err error) error { if jst.err == nil { // Initialize the context if it wasn't done yet if !jst.inited { @@ -595,7 +595,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost // CaptureFault implements the Tracer interface to trace an execution fault // while running an opcode. -func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, contract *vm.Contract, depth int, err error) error { +func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { if jst.err == nil { // Apart from the error, everything matches the previous invocation jst.errorValue = new(string) diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index f28e14864b..d96030385c 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -152,10 +152,10 @@ func TestHaltBetweenSteps(t *testing.T) { env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) - tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, contract, 0, nil) timeout := errors.New("stahp") tracer.Stop(timeout) - tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, contract, 0, nil) if _, err := tracer.GetResult(); err.Error() != timeout.Error() { t.Errorf("Expected timeout error, got %v", err) From 430f69e01eb33a1b2f4f3c9372da14d01aaba62e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 2 Mar 2021 23:51:03 +0200 Subject: [PATCH 172/709] core/vm/runtime: more unshipping --- core/vm/runtime/runtime_test.go | 225 +------------------------------- 1 file changed, 2 insertions(+), 223 deletions(-) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index af69e3333f..6e0434c2ca 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -330,14 +330,14 @@ func (s *stepCounter) CaptureStart(from common.Address, to common.Address, creat return nil } -func (s *stepCounter) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, rData []byte, contract *vm.Contract, depth int, err error) error { +func (s *stepCounter) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rData []byte, contract *vm.Contract, depth int, err error) error { s.steps++ // Enable this for more output //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) return nil } -func (s *stepCounter) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, contract *vm.Contract, depth int, err error) error { +func (s *stepCounter) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { return nil } @@ -345,227 +345,6 @@ func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, return nil } -func TestJumpSub1024Limit(t *testing.T) { - state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - address := common.HexToAddress("0x0a") - // Code is - // 0 beginsub - // 1 push 0 - // 3 jumpsub - // - // The code recursively calls itself. It should error when the returns-stack - // grows above 1023 - state.SetCode(address, []byte{ - byte(vm.PUSH1), 3, - byte(vm.JUMPSUB), - byte(vm.BEGINSUB), - byte(vm.PUSH1), 3, - byte(vm.JUMPSUB), - }) - tracer := stepCounter{inner: vm.NewJSONLogger(nil, os.Stdout)} - // Enable 2315 - _, _, err := Call(address, nil, &Config{State: state, - GasLimit: 20000, - ChainConfig: params.AllEthashProtocolChanges, - EVMConfig: vm.Config{ - ExtraEips: []int{2315}, - Debug: true, - //Tracer: vm.NewJSONLogger(nil, os.Stdout), - Tracer: &tracer, - }}) - exp := "return stack limit reached" - if err.Error() != exp { - t.Fatalf("expected %v, got %v", exp, err) - } - if exp, got := 2048, tracer.steps; exp != got { - t.Fatalf("expected %d steps, got %d", exp, got) - } -} - -func TestReturnSubShallow(t *testing.T) { - state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - address := common.HexToAddress("0x0a") - // The code does returnsub without having anything on the returnstack. - // It should not panic, but just fail after one step - state.SetCode(address, []byte{ - byte(vm.PUSH1), 5, - byte(vm.JUMPSUB), - byte(vm.RETURNSUB), - byte(vm.PC), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - byte(vm.PC), - }) - tracer := stepCounter{} - - // Enable 2315 - _, _, err := Call(address, nil, &Config{State: state, - GasLimit: 10000, - ChainConfig: params.AllEthashProtocolChanges, - EVMConfig: vm.Config{ - ExtraEips: []int{2315}, - Debug: true, - Tracer: &tracer, - }}) - - exp := "invalid retsub" - if err.Error() != exp { - t.Fatalf("expected %v, got %v", exp, err) - } - if exp, got := 4, tracer.steps; exp != got { - t.Fatalf("expected %d steps, got %d", exp, got) - } -} - -// disabled -- only used for generating markdown -func DisabledTestReturnCases(t *testing.T) { - cfg := &Config{ - EVMConfig: vm.Config{ - Debug: true, - Tracer: vm.NewMarkdownLogger(nil, os.Stdout), - ExtraEips: []int{2315}, - }, - } - // This should fail at first opcode - Execute([]byte{ - byte(vm.RETURNSUB), - byte(vm.PC), - byte(vm.PC), - }, nil, cfg) - - // Should also fail - Execute([]byte{ - byte(vm.PUSH1), 5, - byte(vm.JUMPSUB), - byte(vm.RETURNSUB), - byte(vm.PC), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - byte(vm.PC), - }, nil, cfg) - - // This should complete - Execute([]byte{ - byte(vm.PUSH1), 0x4, - byte(vm.JUMPSUB), - byte(vm.STOP), - byte(vm.BEGINSUB), - byte(vm.PUSH1), 0x9, - byte(vm.JUMPSUB), - byte(vm.RETURNSUB), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - }, nil, cfg) -} - -// DisabledTestEipExampleCases contains various testcases that are used for the -// EIP examples -// This test is disabled, as it's only used for generating markdown -func DisabledTestEipExampleCases(t *testing.T) { - cfg := &Config{ - EVMConfig: vm.Config{ - Debug: true, - Tracer: vm.NewMarkdownLogger(nil, os.Stdout), - ExtraEips: []int{2315}, - }, - } - prettyPrint := func(comment string, code []byte) { - instrs := make([]string, 0) - it := asm.NewInstructionIterator(code) - for it.Next() { - if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%v 0x%x", it.Op(), it.Arg())) - } else { - instrs = append(instrs, fmt.Sprintf("%v", it.Op())) - } - } - ops := strings.Join(instrs, ", ") - - fmt.Printf("%v\nBytecode: `0x%x` (`%v`)\n", - comment, - code, ops) - Execute(code, nil, cfg) - } - - { // First eip testcase - code := []byte{ - byte(vm.PUSH1), 4, - byte(vm.JUMPSUB), - byte(vm.STOP), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - } - prettyPrint("This should jump into a subroutine, back out and stop.", code) - } - - { - code := []byte{ - byte(vm.PUSH9), 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 4 + 8, - byte(vm.JUMPSUB), - byte(vm.STOP), - byte(vm.BEGINSUB), - byte(vm.PUSH1), 8 + 9, - byte(vm.JUMPSUB), - byte(vm.RETURNSUB), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - } - prettyPrint("This should execute fine, going into one two depths of subroutines", code) - } - // TODO(@holiman) move this test into an actual test, which not only prints - // out the trace. - { - code := []byte{ - byte(vm.PUSH9), 0x01, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 4 + 8, - byte(vm.JUMPSUB), - byte(vm.STOP), - byte(vm.BEGINSUB), - byte(vm.PUSH1), 8 + 9, - byte(vm.JUMPSUB), - byte(vm.RETURNSUB), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - } - prettyPrint("This should fail, since the given location is outside of the "+ - "code-range. The code is the same as previous example, except that the "+ - "pushed location is `0x01000000000000000c` instead of `0x0c`.", code) - } - { - // This should fail at first opcode - code := []byte{ - byte(vm.RETURNSUB), - byte(vm.PC), - byte(vm.PC), - } - prettyPrint("This should fail at first opcode, due to shallow `return_stack`", code) - - } - { - code := []byte{ - byte(vm.PUSH1), 5, // Jump past the subroutine - byte(vm.JUMP), - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - byte(vm.JUMPDEST), - byte(vm.PUSH1), 3, // Now invoke the subroutine - byte(vm.JUMPSUB), - } - prettyPrint("In this example. the JUMPSUB is on the last byte of code. When the "+ - "subroutine returns, it should hit the 'virtual stop' _after_ the bytecode, "+ - "and not exit with error", code) - } - - { - code := []byte{ - byte(vm.BEGINSUB), - byte(vm.RETURNSUB), - byte(vm.STOP), - } - prettyPrint("In this example, the code 'walks' into a subroutine, which is not "+ - "allowed, and causes an error", code) - } -} - // benchmarkNonModifyingCode benchmarks code, but if the code modifies the // state, this should not be used, since it does not reset the state between runs. func benchmarkNonModifyingCode(gas uint64, code []byte, name string, b *testing.B) { From 0540d3c6f60d1cba6a3dd384790f5d1fa0d799bd Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 3 Mar 2021 08:42:59 +0100 Subject: [PATCH 173/709] cmd/geth: put allowUnsecureTx flag in RPC section (#22412) --- cmd/geth/usage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 6935adabc7..2c3cdf1945 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -152,6 +152,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.GraphQLVirtualHostsFlag, utils.RPCGlobalGasCapFlag, utils.RPCGlobalTxFeeCapFlag, + utils.AllowUnprotectedTxs, utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag, From c539a052bd5a31dfaeabf65d789b691f5d03f300 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 3 Mar 2021 18:04:25 +0800 Subject: [PATCH 174/709] params: update chts (#22418) --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 8cbfffc03c..8f8b36b57d 100644 --- a/params/config.go +++ b/params/config.go @@ -74,10 +74,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 345, - SectionHead: common.HexToHash("0x5453bab878704adebc934b41fd214a07ea7a72b8572ff088dca7f7956cd0ef28"), - CHTRoot: common.HexToHash("0x7693d432595846c094f47cb37f5c868b0b7b1968fc6b0fc411ded1345fdaffab"), - BloomRoot: common.HexToHash("0x8b0e7895bc39840d8dac857e26bdf3d0a07684b0b962b252546659e0337a9f70"), + SectionIndex: 364, + SectionHead: common.HexToHash("0x3fd20ff221f5e962bb66f57a61973bfc2ba959879a6509384a80a45d208b5afc"), + CHTRoot: common.HexToHash("0xe35b3b807f4e9427fb4e2929961c78a9dc10f503a538319031cc7d00946a0591"), + BloomRoot: common.HexToHash("0x340553b378b2db214b898be15c80ac5be7caffc2e6448fd6f7aff23290d89296"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -157,10 +157,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 232, - SectionHead: common.HexToHash("0x8170fca4039b11a008c11f9996ff112151cbb17411437bb2f86288e11158b2f0"), - CHTRoot: common.HexToHash("0x4526560d92ae1b3a6d3ee780c3ad289ba2bbf1b5da58d9ea107f2f26412b631f"), - BloomRoot: common.HexToHash("0x82a889098a35d6a21ea8894d35a1db69b94bad61b988bbe5ae4601437320e331"), + SectionIndex: 248, + SectionHead: common.HexToHash("0x26874cf023695778cc3175d1bec19894204d8d0b756b587e81e35f300dc5b33c"), + CHTRoot: common.HexToHash("0xc129d1ed6673c5d3e1068e9d97244e72952b7ca08acbd7b3bfa58bc3085c442c"), + BloomRoot: common.HexToHash("0x1dafe79dcd7d348782aa834a4a4397890d9ad90643736791132ed5c16879a037"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -198,10 +198,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 116, - SectionHead: common.HexToHash("0xf2d200f636f213c9c7bb4e747ff564813da7708253037103aef3d8be5203c5e1"), - CHTRoot: common.HexToHash("0xb0ac83e2ccf6c2776945e099c4e3df50fe6200499c8b2045c34cafdf57d15087"), - BloomRoot: common.HexToHash("0xfb580ad1c611230a4bfc56534f58bcb156d028bc6ce70e35403dc019c7c02d90"), + SectionIndex: 132, + SectionHead: common.HexToHash("0x29fa240c97b47ecbfef3fea8b3cff035d93154d1d48b25e3333cf2f7067c5324"), + CHTRoot: common.HexToHash("0x85e5c59e5b202284291405dadc40dc36ab6417bd189fb18be24f6dcab6b80511"), + BloomRoot: common.HexToHash("0x0b7afdd200477f46e982e2cabc822ac454424986fa50d899685dfaeede1f882d"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. From 07e907c7d4ce01fff663aa7b5a378f647518996f Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 3 Mar 2021 18:04:50 +0800 Subject: [PATCH 175/709] cmd/utils: fix txlookuplimit for archive node (#22419) * cmd/utils: fix exclusive check for archive node * cmd/utils: set the txlookuplimit to 0 --- cmd/utils/flags.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 324a6d6a47..7bbca42654 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1477,8 +1477,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV3Flag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer - CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag) - if ctx.GlobalIsSet(LightServeFlag.Name) && ctx.GlobalIsSet(TxLookupLimitFlag.Name) { + if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { + ctx.GlobalSet(TxLookupLimitFlag.Name, "0") + log.Warn("Disable transaction unindexing for archive node") + } + if ctx.GlobalIsSet(LightServeFlag.Name) && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") } var ks *keystore.KeyStore From ba999105ef89473cfe39e5e53354f7099e67a290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 3 Mar 2021 12:05:27 +0200 Subject: [PATCH 176/709] core/forkid, params: unset Berlin fork number (#22413) --- core/forkid/forkid_test.go | 58 ++++++++++++++++---------------------- params/config.go | 4 --- 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index a20598fa9d..87d64ed67f 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -43,26 +43,24 @@ func TestCreation(t *testing.T) { params.MainnetChainConfig, params.MainnetGenesisHash, []testcase{ - {0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Unsynced - {1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Last Frontier block - {1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // First Homestead block - {1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // Last Homestead block - {1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // First DAO block - {2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // Last DAO block - {2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // First Tangerine block - {2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // Last Tangerine block - {2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // First Spurious block - {4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // Last Spurious block - {4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // First Byzantium block - {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block - {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block - {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block - {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block - {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block - {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // First Muir Glacier block - {12243999, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // Last Muir Glacier block - {12244000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 0}}, // First Berlin block - {20000000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 0}}, // Future Berlin block + {0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Unsynced + {1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Last Frontier block + {1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // First Homestead block + {1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // Last Homestead block + {1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // First DAO block + {2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // Last DAO block + {2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // First Tangerine block + {2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // Last Tangerine block + {2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // First Spurious block + {4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // Last Spurious block + {4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // First Byzantium block + {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block + {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block + {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block + {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block + {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block + {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // First Muir Glacier block + {12243999, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // Future Muir Glacier block }, }, // Ropsten test cases @@ -82,10 +80,8 @@ func TestCreation(t *testing.T) { {6485845, ID{Hash: checksumToBytes(0xd6e2149b), Next: 6485846}}, // Last Petersburg block {6485846, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // First Istanbul block {7117116, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // Last Istanbul block - {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 9812189}}, // First Muir Glacier block - {9812188, ID{Hash: checksumToBytes(0x6727ef90), Next: 9812189}}, // Last Muir Glacier block - {9812189, ID{Hash: checksumToBytes(0xa157d377), Next: 0}}, // First Berlin block - {10000000, ID{Hash: checksumToBytes(0xa157d377), Next: 0}}, // Future Berlin block + {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // First Muir Glacier block + {9812188, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // Future Muir Glacier block }, }, // Rinkeby test cases @@ -104,10 +100,8 @@ func TestCreation(t *testing.T) { {4321233, ID{Hash: checksumToBytes(0xe49cab14), Next: 4321234}}, // Last Constantinople block {4321234, ID{Hash: checksumToBytes(0xafec6b27), Next: 5435345}}, // First Petersburg block {5435344, ID{Hash: checksumToBytes(0xafec6b27), Next: 5435345}}, // Last Petersburg block - {5435345, ID{Hash: checksumToBytes(0xcbdb8838), Next: 8290928}}, // First Istanbul block - {8290927, ID{Hash: checksumToBytes(0xcbdb8838), Next: 8290928}}, // Last Istanbul block - {8290928, ID{Hash: checksumToBytes(0x6910c8bd), Next: 0}}, // First Berlin block - {10000000, ID{Hash: checksumToBytes(0x6910c8bd), Next: 0}}, // Future Berlin block + {5435345, ID{Hash: checksumToBytes(0xcbdb8838), Next: 0}}, // First Istanbul block + {8290927, ID{Hash: checksumToBytes(0xcbdb8838), Next: 0}}, // Future Istanbul block }, }, // Goerli test cases @@ -117,10 +111,8 @@ func TestCreation(t *testing.T) { []testcase{ {0, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople and first Petersburg block {1561650, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}}, // Last Petersburg block - {1561651, ID{Hash: checksumToBytes(0xc25efa5c), Next: 4460644}}, // First Istanbul block - {4460643, ID{Hash: checksumToBytes(0xc25efa5c), Next: 4460644}}, // Last Istanbul block - {4460644, ID{Hash: checksumToBytes(0x757a1c47), Next: 0}}, // First Berlin block - {5000000, ID{Hash: checksumToBytes(0x757a1c47), Next: 0}}, // Future Berlin block + {1561651, ID{Hash: checksumToBytes(0xc25efa5c), Next: 0}}, // First Istanbul block + {4460643, ID{Hash: checksumToBytes(0xc25efa5c), Next: 0}}, // Future Istanbul block }, }, } @@ -193,7 +185,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Berlin, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Istanbul, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). diff --git a/params/config.go b/params/config.go index 8f8b36b57d..9b8c5cf4ed 100644 --- a/params/config.go +++ b/params/config.go @@ -68,7 +68,6 @@ var ( PetersburgBlock: big.NewInt(7_280_000), IstanbulBlock: big.NewInt(9_069_000), MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), Ethash: new(EthashConfig), } @@ -108,7 +107,6 @@ var ( PetersburgBlock: big.NewInt(4_939_394), IstanbulBlock: big.NewInt(6_485_846), MuirGlacierBlock: big.NewInt(7_117_117), - BerlinBlock: big.NewInt(9_812_189), Ethash: new(EthashConfig), } @@ -148,7 +146,6 @@ var ( PetersburgBlock: big.NewInt(4_321_234), IstanbulBlock: big.NewInt(5_435_345), MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(8_290_928), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -189,7 +186,6 @@ var ( PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(1_561_651), MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(4_460_644), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, From b24804d88cdbd38edc85ee9f2afaa9e6cb7a767e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Wed, 3 Mar 2021 15:05:24 +0100 Subject: [PATCH 177/709] les: fix nodiscover option on the client side (#22422) --- les/client.go | 12 +++++++++++- les/vflux/client/serverpool.go | 6 +++--- les/vflux/client/serverpool_test.go | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/les/client.go b/les/client.go index ecabfdf503..605c4d03ca 100644 --- a/les/client.go +++ b/les/client.go @@ -115,7 +115,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { p2pConfig: &stack.Config().P2P, } - leth.serverPool, leth.serverPoolIterator = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, leth.prenegQuery, &mclock.System{}, config.UltraLightServers, requestList) + var prenegQuery vfc.QueryFunc + if leth.p2pServer.DiscV5 != nil { + prenegQuery = leth.prenegQuery + } + leth.serverPool, leth.serverPoolIterator = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, prenegQuery, &mclock.System{}, config.UltraLightServers, requestList) leth.serverPool.AddMetrics(suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge, sessionValueMeter, serverDialedMeter) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.GetTimeout) @@ -194,6 +198,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { // VfluxRequest sends a batch of requests to the given node through discv5 UDP TalkRequest and returns the responses func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.Replies { + if s.p2pServer.DiscV5 == nil { + return nil + } reqsEnc, _ := rlp.EncodeToBytes(&reqs) repliesEnc, _ := s.p2pServer.DiscV5.TalkRequest(s.serverPool.DialNode(n), "vfx", reqsEnc) var replies vflux.Replies @@ -208,6 +215,9 @@ func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.R func (s *LightEthereum) vfxVersion(n *enode.Node) uint { if n.Seq() == 0 { var err error + if s.p2pServer.DiscV5 == nil { + return 0 + } if n, err = s.p2pServer.DiscV5.RequestENR(n); n != nil && err == nil && n.Seq() != 0 { s.serverPool.Persist(n) } else { diff --git a/les/vflux/client/serverpool.go b/les/vflux/client/serverpool.go index 47ec4fee74..e73b277ced 100644 --- a/les/vflux/client/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -91,7 +91,7 @@ type nodeHistoryEnc struct { // queryFunc sends a pre-negotiation query and blocks until a response arrives or timeout occurs. // It returns 1 if the remote node has confirmed that connection is possible, 0 if not // possible and -1 if no response arrived (timeout). -type queryFunc func(*enode.Node) int +type QueryFunc func(*enode.Node) int var ( clientSetup = &nodestate.Setup{Version: 2} @@ -150,7 +150,7 @@ var ( ) // NewServerPool creates a new server pool -func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string, requestList []RequestInfo) (*ServerPool, enode.Iterator) { +func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duration, query QueryFunc, clock mclock.Clock, trustedURLs []string, requestList []RequestInfo) (*ServerPool, enode.Iterator) { s := &ServerPool{ db: db, clock: clock, @@ -246,7 +246,7 @@ func (s *ServerPool) AddSource(source enode.Iterator) { // addPreNegFilter installs a node filter mechanism that performs a pre-negotiation query. // Nodes that are filtered out and does not appear on the output iterator are put back // into redialWait state. -func (s *ServerPool) addPreNegFilter(input enode.Iterator, query queryFunc) enode.Iterator { +func (s *ServerPool) addPreNegFilter(input enode.Iterator, query QueryFunc) enode.Iterator { s.fillSet = NewFillSet(s.ns, input, sfQueried) s.ns.SubscribeState(sfQueried, func(n *enode.Node, oldState, newState nodestate.Flags) { if newState.Equals(sfQueried) { diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index ee299618c6..c777d6c16d 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -107,7 +107,7 @@ func (s *ServerPoolTest) addTrusted(i int) { } func (s *ServerPoolTest) start() { - var testQuery queryFunc + var testQuery QueryFunc if s.preNeg { testQuery = func(node *enode.Node) int { idx := testNodeIndex(node.ID()) From 5a81dd97d5f3347457e640631564fa5b893720c2 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 3 Mar 2021 22:08:14 +0800 Subject: [PATCH 178/709] cmd: retire whisper flags (#22421) * cmd: retire whisper flags * cmd/geth: remove whisper configs --- cmd/geth/config.go | 31 ++----------------------------- cmd/geth/consolecmd.go | 2 +- cmd/geth/main.go | 8 -------- cmd/geth/usage.go | 4 ---- cmd/utils/flags.go | 28 ---------------------------- 5 files changed, 3 insertions(+), 70 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index c77c04c252..6fc75363c6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -43,7 +42,7 @@ var ( Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", - Flags: append(append(nodeFlags, rpcFlags...), whisperFlags...), + Flags: append(nodeFlags, rpcFlags...), Category: "MISCELLANEOUS COMMANDS", Description: `The dumpconfig command shows configuration values.`, } @@ -75,19 +74,8 @@ type ethstatsConfig struct { URL string `toml:",omitempty"` } -// whisper has been deprecated, but clients out there might still have [Shh] -// in their config, which will crash. Cut them some slack by keeping the -// config, and displaying a message that those config switches are ineffectual. -// To be removed circa Q1 2021 -- @gballet. -type whisperDeprecatedConfig struct { - MaxMessageSize uint32 `toml:",omitempty"` - MinimumAcceptedPOW float64 `toml:",omitempty"` - RestrictConnectionBetweenLightClients bool `toml:",omitempty"` -} - type gethConfig struct { Eth ethconfig.Config - Shh whisperDeprecatedConfig Node node.Config Ethstats ethstatsConfig Metrics metrics.Config @@ -132,11 +120,8 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { if err := loadConfig(file, &cfg); err != nil { utils.Fatalf("%v", err) } - - if cfg.Shh != (whisperDeprecatedConfig{}) { - log.Warn("Deprecated whisper config detected. Whisper has been moved to github.com/ethereum/whisper") - } } + // Apply flags. utils.SetNodeConfig(ctx, &cfg.Node) stack, err := node.New(&cfg.Node) @@ -147,22 +132,11 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) { cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) } - utils.SetShhConfig(ctx, stack) - applyMetricConfig(ctx, &cfg) return stack, cfg } -// enableWhisper returns true in case one of the whisper flags is set. -func checkWhisper(ctx *cli.Context) { - for _, flag := range whisperFlags { - if ctx.GlobalIsSet(flag.GetName()) { - log.Warn("deprecated whisper flag detected. Whisper has been moved to github.com/ethereum/whisper") - } - } -} - // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) @@ -171,7 +145,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } backend := utils.RegisterEthService(stack, &cfg.Eth) - checkWhisper(ctx) // Configure GraphQL if requested if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { utils.RegisterGraphQLService(stack, backend, cfg.Node) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index e26481003d..9d8794eb15 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -36,7 +36,7 @@ var ( Action: utils.MigrateFlags(localConsole), Name: "console", Usage: "Start an interactive JavaScript environment", - Flags: append(append(append(nodeFlags, rpcFlags...), consoleFlags...), whisperFlags...), + Flags: append(append(nodeFlags, rpcFlags...), consoleFlags...), Category: "CONSOLE COMMANDS", Description: ` The Geth console is an interactive shell for the JavaScript runtime environment diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9afa031584..207c93c0d8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -186,13 +186,6 @@ var ( utils.AllowUnprotectedTxs, } - whisperFlags = []cli.Flag{ - utils.WhisperEnabledFlag, - utils.WhisperMaxMessageSizeFlag, - utils.WhisperMinPOWFlag, - utils.WhisperRestrictConnectionBetweenLightClientsFlag, - } - metricsFlags = []cli.Flag{ utils.MetricsEnabledFlag, utils.MetricsEnabledExpensiveFlag, @@ -251,7 +244,6 @@ func init() { app.Flags = append(app.Flags, rpcFlags...) app.Flags = append(app.Flags, consoleFlags...) app.Flags = append(app.Flags, debug.Flags...) - app.Flags = append(app.Flags, whisperFlags...) app.Flags = append(app.Flags, metricsFlags...) app.Before = func(ctx *cli.Context) error { diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 2c3cdf1945..daea0afc4e 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -216,10 +216,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ Name: "METRICS AND STATS", Flags: metricsFlags, }, - { - Name: "WHISPER (deprecated)", - Flags: whisperFlags, - }, { Name: "ALIASED (deprecated)", Flags: []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7bbca42654..fc479f3987 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -680,24 +680,6 @@ var ( Usage: "Maximum gas price will be recommended by gpo", Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), } - WhisperEnabledFlag = cli.BoolFlag{ - Name: "shh", - Usage: "Enable Whisper", - } - WhisperMaxMessageSizeFlag = cli.IntFlag{ - Name: "shh.maxmessagesize", - Usage: "Max message size accepted", - Value: 1024 * 1024, - } - WhisperMinPOWFlag = cli.Float64Flag{ - Name: "shh.pow", - Usage: "Minimum POW accepted", - Value: 0.2, - } - WhisperRestrictConnectionBetweenLightClientsFlag = cli.BoolFlag{ - Name: "shh.restrict-light", - Usage: "Restrict connection between two whisper light clients", - } // Metrics flags MetricsEnabledFlag = cli.BoolFlag{ @@ -1461,16 +1443,6 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { } } -// SetShhConfig applies shh-related command line flags to the config. -func SetShhConfig(ctx *cli.Context, stack *node.Node) { - if ctx.GlobalIsSet(WhisperEnabledFlag.Name) || - ctx.GlobalIsSet(WhisperMaxMessageSizeFlag.Name) || - ctx.GlobalIsSet(WhisperMinPOWFlag.Name) || - ctx.GlobalIsSet(WhisperRestrictConnectionBetweenLightClientsFlag.Name) { - log.Warn("Whisper support has been deprecated and the code has been moved to github.com/ethereum/whisper") - } -} - // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags From cd316d7c7158f1ffc99910c8c07b951ec05ed067 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 3 Mar 2021 15:50:07 +0100 Subject: [PATCH 179/709] tests: update to latest tests (#22290) This updates the consensus tests to commit 31d6630 and adds support for access list transactions in the test runner. Co-authored-by: Martin Holst Swende --- tests/block_test.go | 14 +++++--------- tests/gen_stenv.go | 2 ++ tests/gen_sttransaction.go | 37 +++++++++++++++++++++++-------------- tests/gen_vmexec.go | 2 ++ tests/state_test.go | 8 -------- tests/state_test_util.go | 22 +++++++++++++--------- tests/testdata | 2 +- 7 files changed, 46 insertions(+), 41 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index 84618fd0be..2649bae85a 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -25,7 +25,11 @@ func TestBlockchain(t *testing.T) { bt := new(testMatcher) // General state tests are 'exported' as blockchain tests, but we can run them natively. - bt.skipLoad(`^GeneralStateTests/`) + // For speedier CI-runs, the line below can be uncommented, so those are skipped. + // For now, in hardfork-times (Berlin), we run the tests both as StateTests and + // as blockchain tests, since the latter also covers things like receipt root + //bt.skipLoad(`^GeneralStateTests/`) + // Skip random failures due to selfish mining test bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`) @@ -43,15 +47,7 @@ func TestBlockchain(t *testing.T) { // test takes a lot for time and goes easily OOM because of sha3 calculation on a huge range, // using 4.6 TGas bt.skipLoad(`.*randomStatetest94.json.*`) - bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if test.json.Network == "Berlin" { - // Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them - // fail when berlin is defined as YOLOv3. We skip those, until they've been - // regenerated and re-imported - // TODO (@holiman) - return - } if err := bt.checkFailure(t, name+"/trie", test.Run(false)); err != nil { t.Errorf("test without snapshotter failed: %v", err) } diff --git a/tests/gen_stenv.go b/tests/gen_stenv.go index 1d4baf2fd7..bfecc145b4 100644 --- a/tests/gen_stenv.go +++ b/tests/gen_stenv.go @@ -13,6 +13,7 @@ import ( var _ = (*stEnvMarshaling)(nil) +// MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` @@ -30,6 +31,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` diff --git a/tests/gen_sttransaction.go b/tests/gen_sttransaction.go index 451ffcbf43..2670f4f9c8 100644 --- a/tests/gen_sttransaction.go +++ b/tests/gen_sttransaction.go @@ -8,25 +8,29 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" ) var _ = (*stTransactionMarshaling)(nil) +// MarshalJSON marshals as JSON. func (s stTransaction) MarshalJSON() ([]byte, error) { type stTransaction struct { - GasPrice *math.HexOrDecimal256 `json:"gasPrice"` - Nonce math.HexOrDecimal64 `json:"nonce"` - To string `json:"to"` - Data []string `json:"data"` - GasLimit []math.HexOrDecimal64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey hexutil.Bytes `json:"secretKey"` + GasPrice *math.HexOrDecimal256 `json:"gasPrice"` + Nonce math.HexOrDecimal64 `json:"nonce"` + To string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []math.HexOrDecimal64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey hexutil.Bytes `json:"secretKey"` } var enc stTransaction enc.GasPrice = (*math.HexOrDecimal256)(s.GasPrice) enc.Nonce = math.HexOrDecimal64(s.Nonce) enc.To = s.To enc.Data = s.Data + enc.AccessLists = s.AccessLists if s.GasLimit != nil { enc.GasLimit = make([]math.HexOrDecimal64, len(s.GasLimit)) for k, v := range s.GasLimit { @@ -38,15 +42,17 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (s *stTransaction) UnmarshalJSON(input []byte) error { type stTransaction struct { - GasPrice *math.HexOrDecimal256 `json:"gasPrice"` - Nonce *math.HexOrDecimal64 `json:"nonce"` - To *string `json:"to"` - Data []string `json:"data"` - GasLimit []math.HexOrDecimal64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey *hexutil.Bytes `json:"secretKey"` + GasPrice *math.HexOrDecimal256 `json:"gasPrice"` + Nonce *math.HexOrDecimal64 `json:"nonce"` + To *string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []math.HexOrDecimal64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey *hexutil.Bytes `json:"secretKey"` } var dec stTransaction if err := json.Unmarshal(input, &dec); err != nil { @@ -64,6 +70,9 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error { if dec.Data != nil { s.Data = dec.Data } + if dec.AccessLists != nil { + s.AccessLists = dec.AccessLists + } if dec.GasLimit != nil { s.GasLimit = make([]uint64, len(dec.GasLimit)) for k, v := range dec.GasLimit { diff --git a/tests/gen_vmexec.go b/tests/gen_vmexec.go index a5f01cf456..2fe155152d 100644 --- a/tests/gen_vmexec.go +++ b/tests/gen_vmexec.go @@ -14,6 +14,7 @@ import ( var _ = (*vmExecMarshaling)(nil) +// MarshalJSON marshals as JSON. func (v vmExec) MarshalJSON() ([]byte, error) { type vmExec struct { Address common.UnprefixedAddress `json:"address" gencodec:"required"` @@ -37,6 +38,7 @@ func (v vmExec) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (v *vmExec) UnmarshalJSON(input []byte) error { type vmExec struct { Address *common.UnprefixedAddress `json:"address" gencodec:"required"` diff --git a/tests/state_test.go b/tests/state_test.go index 0f2bd38532..b77a898c21 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -62,14 +62,6 @@ func TestState(t *testing.T) { } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { - if subtest.Fork == "Berlin" { - // Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them - // fail when berlin is defined as YOLOv3. We skip those, until they've been - // regenerated and re-imported - // TODO (@holiman) - continue - } - subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 99681baaf1..46834de6da 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -95,13 +95,14 @@ type stEnvMarshaling struct { //go:generate gencodec -type stTransaction -field-override stTransactionMarshaling -out gen_sttransaction.go type stTransaction struct { - GasPrice *big.Int `json:"gasPrice"` - Nonce uint64 `json:"nonce"` - To string `json:"to"` - Data []string `json:"data"` - GasLimit []uint64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey []byte `json:"secretKey"` + GasPrice *big.Int `json:"gasPrice"` + Nonce uint64 `json:"nonce"` + To string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []uint64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey []byte `json:"secretKey"` } type stTransactionMarshaling struct { @@ -292,8 +293,11 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { if err != nil { return nil, fmt.Errorf("invalid tx data %q", dataHex) } - - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, nil, true) + var accessList types.AccessList + if tx.AccessLists != nil && tx.AccessLists[ps.Indexes.Data] != nil { + accessList = *tx.AccessLists[ps.Indexes.Data] + } + msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, accessList, true) return msg, nil } diff --git a/tests/testdata b/tests/testdata index 6c863f03be..31d663076b 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 6c863f03bee8d7a66bb7a028a9f880a86a5f4975 +Subproject commit 31d663076b6678df18983d6da912d7cad4ad3416 From 56dec25ae26bf749b93c3ea69538fabea60c5768 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 3 Mar 2021 17:44:17 +0100 Subject: [PATCH 180/709] params: release geth 1.10.0 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 3447f0f283..de5eac3af9 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 63385374ec0e0a513ff30b9360471769e937244a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 3 Mar 2021 18:01:31 +0100 Subject: [PATCH 181/709] params: begin v1.10.1 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index de5eac3af9..b974c3abda 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 1 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 9230ca4924ef3a75f1653d828a2cd845d54977e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 4 Mar 2021 10:44:37 +0200 Subject: [PATCH 182/709] Revert "core/forkid, params: unset Berlin fork number (#22413)" This reverts commit ba999105ef89473cfe39e5e53354f7099e67a290. --- core/forkid/forkid_test.go | 58 ++++++++++++++++++++++---------------- params/config.go | 4 +++ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 87d64ed67f..a20598fa9d 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -43,24 +43,26 @@ func TestCreation(t *testing.T) { params.MainnetChainConfig, params.MainnetGenesisHash, []testcase{ - {0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Unsynced - {1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Last Frontier block - {1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // First Homestead block - {1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // Last Homestead block - {1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // First DAO block - {2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // Last DAO block - {2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // First Tangerine block - {2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // Last Tangerine block - {2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // First Spurious block - {4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // Last Spurious block - {4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // First Byzantium block - {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block - {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block - {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block - {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block - {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block - {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // First Muir Glacier block - {12243999, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // Future Muir Glacier block + {0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Unsynced + {1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}}, // Last Frontier block + {1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // First Homestead block + {1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}}, // Last Homestead block + {1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // First DAO block + {2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}}, // Last DAO block + {2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // First Tangerine block + {2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}}, // Last Tangerine block + {2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // First Spurious block + {4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}}, // Last Spurious block + {4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // First Byzantium block + {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block + {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block + {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block + {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block + {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block + {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // First Muir Glacier block + {12243999, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // Last Muir Glacier block + {12244000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 0}}, // First Berlin block + {20000000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 0}}, // Future Berlin block }, }, // Ropsten test cases @@ -80,8 +82,10 @@ func TestCreation(t *testing.T) { {6485845, ID{Hash: checksumToBytes(0xd6e2149b), Next: 6485846}}, // Last Petersburg block {6485846, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // First Istanbul block {7117116, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // Last Istanbul block - {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // First Muir Glacier block - {9812188, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // Future Muir Glacier block + {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 9812189}}, // First Muir Glacier block + {9812188, ID{Hash: checksumToBytes(0x6727ef90), Next: 9812189}}, // Last Muir Glacier block + {9812189, ID{Hash: checksumToBytes(0xa157d377), Next: 0}}, // First Berlin block + {10000000, ID{Hash: checksumToBytes(0xa157d377), Next: 0}}, // Future Berlin block }, }, // Rinkeby test cases @@ -100,8 +104,10 @@ func TestCreation(t *testing.T) { {4321233, ID{Hash: checksumToBytes(0xe49cab14), Next: 4321234}}, // Last Constantinople block {4321234, ID{Hash: checksumToBytes(0xafec6b27), Next: 5435345}}, // First Petersburg block {5435344, ID{Hash: checksumToBytes(0xafec6b27), Next: 5435345}}, // Last Petersburg block - {5435345, ID{Hash: checksumToBytes(0xcbdb8838), Next: 0}}, // First Istanbul block - {8290927, ID{Hash: checksumToBytes(0xcbdb8838), Next: 0}}, // Future Istanbul block + {5435345, ID{Hash: checksumToBytes(0xcbdb8838), Next: 8290928}}, // First Istanbul block + {8290927, ID{Hash: checksumToBytes(0xcbdb8838), Next: 8290928}}, // Last Istanbul block + {8290928, ID{Hash: checksumToBytes(0x6910c8bd), Next: 0}}, // First Berlin block + {10000000, ID{Hash: checksumToBytes(0x6910c8bd), Next: 0}}, // Future Berlin block }, }, // Goerli test cases @@ -111,8 +117,10 @@ func TestCreation(t *testing.T) { []testcase{ {0, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople and first Petersburg block {1561650, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}}, // Last Petersburg block - {1561651, ID{Hash: checksumToBytes(0xc25efa5c), Next: 0}}, // First Istanbul block - {4460643, ID{Hash: checksumToBytes(0xc25efa5c), Next: 0}}, // Future Istanbul block + {1561651, ID{Hash: checksumToBytes(0xc25efa5c), Next: 4460644}}, // First Istanbul block + {4460643, ID{Hash: checksumToBytes(0xc25efa5c), Next: 4460644}}, // Last Istanbul block + {4460644, ID{Hash: checksumToBytes(0x757a1c47), Next: 0}}, // First Berlin block + {5000000, ID{Hash: checksumToBytes(0x757a1c47), Next: 0}}, // Future Berlin block }, }, } @@ -185,7 +193,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Istanbul, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Berlin, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). diff --git a/params/config.go b/params/config.go index 9b8c5cf4ed..8f8b36b57d 100644 --- a/params/config.go +++ b/params/config.go @@ -68,6 +68,7 @@ var ( PetersburgBlock: big.NewInt(7_280_000), IstanbulBlock: big.NewInt(9_069_000), MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), Ethash: new(EthashConfig), } @@ -107,6 +108,7 @@ var ( PetersburgBlock: big.NewInt(4_939_394), IstanbulBlock: big.NewInt(6_485_846), MuirGlacierBlock: big.NewInt(7_117_117), + BerlinBlock: big.NewInt(9_812_189), Ethash: new(EthashConfig), } @@ -146,6 +148,7 @@ var ( PetersburgBlock: big.NewInt(4_321_234), IstanbulBlock: big.NewInt(5_435_345), MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(8_290_928), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -186,6 +189,7 @@ var ( PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(1_561_651), MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(4_460_644), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, From de61da99c49146200b0f0db07dd2be6afe1232ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 4 Mar 2021 13:04:50 +0200 Subject: [PATCH 183/709] build: fix PPA failure due to updated debsrc --- build/ci.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build/ci.go b/build/ci.go index 756fba8399..b5c67b2a80 100644 --- a/build/ci.go +++ b/build/ci.go @@ -564,16 +564,17 @@ func doDebianSource(cmdline []string) { build.MustRun(debuild) var ( - basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) - source = filepath.Join(*workdir, basename+".tar.xz") - dsc = filepath.Join(*workdir, basename+".dsc") - changes = filepath.Join(*workdir, basename+"_source.changes") + basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) + source = filepath.Join(*workdir, basename+".tar.xz") + dsc = filepath.Join(*workdir, basename+".dsc") + changes = filepath.Join(*workdir, basename+"_source.changes") + buildinfo = filepath.Join(*workdir, basename+"_source.buildinfo") ) if *signer != "" { build.MustRunCommand("debsign", changes) } if *upload != "" { - ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes}) + ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes, buildinfo}) } } } From 5b95453ef2a52820147332af2c0d98432f470c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 4 Mar 2021 13:57:02 +0200 Subject: [PATCH 184/709] build: add support for Ubuntu Hirsute Hippo --- build/ci.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/ci.go b/build/ci.go index b5c67b2a80..fb06c0f420 100644 --- a/build/ci.go +++ b/build/ci.go @@ -115,7 +115,6 @@ var ( } // A debian package is created for all executables listed here. - debEthereum = debPackage{ Name: "ethereum", Version: params.Version, @@ -137,11 +136,12 @@ var ( // Note: disco is unsupported because it was officially deprecated on Launchpad. // Note: eoan is unsupported because it was officially deprecated on Launchpad. debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", - "xenial": "golang-go", - "bionic": "golang-go", - "focal": "golang-go", - "groovy": "golang-go", + "trusty": "golang-1.11", + "xenial": "golang-go", + "bionic": "golang-go", + "focal": "golang-go", + "groovy": "golang-go", + "hirsute": "golang-go", } debGoBootPaths = map[string]string{ From 72b8cacf132cb6deed7ae3352ccbf32b9d429417 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sun, 7 Mar 2021 20:55:01 +0100 Subject: [PATCH 185/709] tests: update reference tests with 2315 removed from Berlin --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index 31d663076b..c600d7795a 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 31d663076b6678df18983d6da912d7cad4ad3416 +Subproject commit c600d7795aa2ea57a9c856fc79f72fc05b542124 From c2d2f4ed8f232bb11663a1b01a2e578aa22f24bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 8 Mar 2021 11:32:20 +0200 Subject: [PATCH 186/709] params: release Geth v1.10.1 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index b974c3abda..e92ae13a1f 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 1 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 1 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 182670849eafdde729d009ec99d12358f996fbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 8 Mar 2021 11:34:08 +0200 Subject: [PATCH 187/709] params: begin v1.10.2 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index e92ae13a1f..d88b4f8b88 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 1 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From be87f769f676cbe3bf028e856160396ed08c64fc Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 8 Mar 2021 14:23:28 +0100 Subject: [PATCH 188/709] core/types: reduce allocations in GasPriceCmp (#22456) --- core/types/transaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 49127630ae..a35e07a5a3 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -293,7 +293,7 @@ func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) { // GasPriceCmp compares the gas prices of two transactions. func (tx *Transaction) GasPriceCmp(other *Transaction) int { - return tx.inner.gasPrice().Cmp(other.GasPrice()) + return tx.inner.gasPrice().Cmp(other.inner.gasPrice()) } // GasPriceIntCmp compares the gas price of the transaction against the given price. From 3d299b746893e39f580dfcca3cef90604d05f4b6 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 9 Mar 2021 09:04:03 +0100 Subject: [PATCH 189/709] les: fix errors in metric namespace (#22459) * les: add trailing slash to metric namespace * les: omit '.' in metric namespace --- les/client.go | 2 +- les/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/les/client.go b/les/client.go index 605c4d03ca..a557e7ccb3 100644 --- a/les/client.go +++ b/les/client.go @@ -83,7 +83,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/les.client") + lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/") if err != nil { return nil, err } diff --git a/les/server.go b/les/server.go index 63feaf892c..9627d65afa 100644 --- a/les/server.go +++ b/les/server.go @@ -87,7 +87,7 @@ type LesServer struct { } func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*LesServer, error) { - lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/les.server") + lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/") if err != nil { return nil, err } From 22082f9e565665f7da045b9ff708e4698f383c77 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 9 Mar 2021 17:50:25 +0800 Subject: [PATCH 190/709] cmd: extend dumpgenesis to support network flags on the cmd (#22406) --- cmd/geth/chaincmd.go | 7 ++++++- cmd/utils/flags.go | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ff9581fd88..1834dab6a9 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -62,7 +62,11 @@ It expects the genesis file as argument.`, Usage: "Dumps genesis block JSON configuration to stdout", ArgsUsage: "", Flags: []cli.Flag{ - utils.DataDirFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -227,6 +231,7 @@ func initGenesis(ctx *cli.Context) error { } func dumpGenesis(ctx *cli.Context) error { + // TODO(rjl493456442) support loading from the custom datadir genesis := utils.MakeGenesis(ctx) if genesis == nil { genesis = core.DefaultGenesisBlock() diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fc479f3987..5c800a7fe9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1773,6 +1773,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { func MakeGenesis(ctx *cli.Context) *core.Genesis { var genesis *core.Genesis switch { + case ctx.GlobalBool(MainnetFlag.Name): + genesis = core.DefaultGenesisBlock() case ctx.GlobalBool(RopstenFlag.Name): genesis = core.DefaultRopstenGenesisBlock() case ctx.GlobalBool(RinkebyFlag.Name): From aae7660410f0ef90279e14afaaf2f429fdc2a186 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 11 Mar 2021 15:09:25 +0100 Subject: [PATCH 191/709] p2p/enr: fix decoding of incomplete lists (#22484) Given a list of less than two elements DecodeRLP returned rlp.EOL, leading to issues in outer decoders. --- p2p/enr/enr.go | 7 +++++++ p2p/enr/enr_test.go | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index c36ae9e3ed..05e43fd805 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -50,6 +50,7 @@ var ( errNotSorted = errors.New("record key/value pairs are not sorted by key") errDuplicateKey = errors.New("record contains duplicate key") errIncompletePair = errors.New("record contains incomplete k/v pair") + errIncompleteList = errors.New("record contains less than two list elements") errTooBig = fmt.Errorf("record bigger than %d bytes", SizeLimit) errEncodeUnsigned = errors.New("can't encode unsigned record") errNotFound = errors.New("no such key in record") @@ -209,9 +210,15 @@ func decodeRecord(s *rlp.Stream) (dec Record, raw []byte, err error) { return dec, raw, err } if err = s.Decode(&dec.signature); err != nil { + if err == rlp.EOL { + err = errIncompleteList + } return dec, raw, err } if err = s.Decode(&dec.seq); err != nil { + if err == rlp.EOL { + err = errIncompleteList + } return dec, raw, err } // The rest of the record contains sorted k/v pairs. diff --git a/p2p/enr/enr_test.go b/p2p/enr/enr_test.go index 96a9ced5cf..bf3f104744 100644 --- a/p2p/enr/enr_test.go +++ b/p2p/enr/enr_test.go @@ -231,6 +231,29 @@ func TestRecordTooBig(t *testing.T) { require.NoError(t, signTest([]byte{5}, &r)) } +// This checks that incomplete RLP inputs are handled correctly. +func TestDecodeIncomplete(t *testing.T) { + type decTest struct { + input []byte + err error + } + tests := []decTest{ + {[]byte{0xC0}, errIncompleteList}, + {[]byte{0xC1, 0x1}, errIncompleteList}, + {[]byte{0xC2, 0x1, 0x2}, nil}, + {[]byte{0xC3, 0x1, 0x2, 0x3}, errIncompletePair}, + {[]byte{0xC4, 0x1, 0x2, 0x3, 0x4}, nil}, + {[]byte{0xC5, 0x1, 0x2, 0x3, 0x4, 0x5}, errIncompletePair}, + } + for _, test := range tests { + var r Record + err := rlp.DecodeBytes(test.input, &r) + if err != test.err { + t.Errorf("wrong error for %X: %v", test.input, err) + } + } +} + // TestSignEncodeAndDecodeRandom tests encoding/decoding of records containing random key/value pairs. func TestSignEncodeAndDecodeRandom(t *testing.T) { var r Record From 6387c520b724525861539c43b8da98372fc42534 Mon Sep 17 00:00:00 2001 From: michael1011 Date: Fri, 12 Mar 2021 10:16:19 +0100 Subject: [PATCH 192/709] cmd/geth: add ancient datadir flag to snapshot subcommands (#22486) --- cmd/geth/snapshot.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index b59c530c27..068dea0b92 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -57,6 +57,7 @@ var ( Category: "MISCELLANEOUS COMMANDS", Flags: []cli.Flag{ utils.DataDirFlag, + utils.AncientFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, @@ -86,6 +87,7 @@ the trie clean cache with default directory will be deleted. Category: "MISCELLANEOUS COMMANDS", Flags: []cli.Flag{ utils.DataDirFlag, + utils.AncientFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, @@ -105,6 +107,7 @@ In other words, this command does the snapshot to trie conversion. Category: "MISCELLANEOUS COMMANDS", Flags: []cli.Flag{ utils.DataDirFlag, + utils.AncientFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, @@ -126,6 +129,7 @@ It's also usable without snapshot enabled. Category: "MISCELLANEOUS COMMANDS", Flags: []cli.Flag{ utils.DataDirFlag, + utils.AncientFlag, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, From 3f74c8e0e504fe7606bc970dff71cf1266a4a9ed Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 13 Mar 2021 21:33:30 +0100 Subject: [PATCH 193/709] cmd/devp2p: better testcase failure output for ethtests (#22482) --- cmd/devp2p/internal/ethtest/types.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index 734adff366..96012b3156 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -273,14 +273,15 @@ loop: for { switch msg := c.Read().(type) { case *Status: - if msg.Head != chain.blocks[chain.Len()-1].Hash() { - t.Fatalf("wrong head block in status: %s", msg.Head.String()) + if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want { + t.Fatalf("wrong head block in status, want: %#x (block %d) have %#x", + want, chain.blocks[chain.Len()-1].NumberU64(), have) } - if msg.TD.Cmp(chain.TD(chain.Len())) != 0 { - t.Fatalf("wrong TD in status: %v", msg.TD) + if have, want := msg.TD.Cmp(chain.TD(chain.Len())), 0; have != want { + t.Fatalf("wrong TD in status: have %v want %v", have, want) } - if !reflect.DeepEqual(msg.ForkID, chain.ForkID()) { - t.Fatalf("wrong fork ID in status: %v", msg.ForkID) + if have, want := msg.ForkID, chain.ForkID(); !reflect.DeepEqual(have, want) { + t.Fatalf("wrong fork ID in status: have %v, want %v", have, want) } message = msg break loop From c6d45009f1ab46f61ebaa66d71d08d48097cc1a9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sun, 14 Mar 2021 16:13:25 +0100 Subject: [PATCH 194/709] eth, les: properly init statedb accesslist during tracing (#22480) * eth/state, les/state: properly init statedb accesslist when tracing, fixes #22475 * eth: review comments * eth/tracers: fix compilation err * eth/tracers: apply @karalabe's suggested fix --- eth/state_accessor.go | 1 + eth/tracers/api.go | 45 ++++++++++++++++++++++++++++++++++--------- les/state_accessor.go | 1 + 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 869b3d7636..cbbd9a8202 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -217,6 +217,7 @@ func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec } // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, txContext, statedb, eth.blockchain.Config(), vm.Config{}) + statedb.Prepare(tx.Hash(), block.Hash(), idx) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { release() return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 61fe055689..75ab403471 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -169,6 +169,13 @@ type StdTraceConfig struct { TxHash common.Hash } +// txTraceContext is the contextual infos about a transaction before it gets run. +type txTraceContext struct { + index int // Index of the transaction within the block + hash common.Hash // Hash of the transaction + block common.Hash // Hash of the block containing the transaction +} + // txTraceResult is the result of a single transaction trace. type txTraceResult struct { Result interface{} `json:"result,omitempty"` // Trace results produced by the tracer @@ -266,7 +273,12 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config // Trace all the transactions contained within for i, tx := range task.block.Transactions() { msg, _ := tx.AsMessage(signer) - res, err := api.traceTx(ctx, msg, blockCtx, task.statedb, config) + txctx := &txTraceContext{ + index: i, + hash: tx.Hash(), + block: task.block.Hash(), + } + res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) if err != nil { task.results[i] = &txTraceResult{Error: err.Error()} log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) @@ -478,6 +490,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac threads = len(txs) } blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + blockHash := block.Hash() for th := 0; th < threads; th++ { pend.Add(1) go func() { @@ -485,7 +498,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // Fetch and execute the next transaction trace tasks for task := range jobs { msg, _ := txs[task.index].AsMessage(signer) - res, err := api.traceTx(ctx, msg, blockCtx, task.statedb, config) + txctx := &txTraceContext{ + index: task.index, + hash: txs[task.index].Hash(), + block: blockHash, + } + res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) if err != nil { results[task.index] = &txTraceResult{Error: err.Error()} continue @@ -502,9 +520,8 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // Generate the next state snapshot fast without tracing msg, _ := tx.AsMessage(signer) - txContext := core.NewEVMTxContext(msg) - - vmenv := vm.NewEVM(blockCtx, txContext, statedb, api.backend.ChainConfig(), vm.Config{}) + statedb.Prepare(tx.Hash(), block.Hash(), i) + vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { failed = err break @@ -619,6 +636,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) + statedb.Prepare(tx.Hash(), block.Hash(), i) _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) if writer != nil { writer.Flush() @@ -678,7 +696,12 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * } defer release() - return api.traceTx(ctx, msg, vmctx, statedb, config) + txctx := &txTraceContext{ + index: int(index), + hash: hash, + block: blockHash, + } + return api.traceTx(ctx, msg, txctx, vmctx, statedb, config) } // TraceCall lets you trace a given eth_call. It collects the structured logs @@ -713,13 +736,14 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHa // Execute the trace msg := args.ToMessage(api.backend.RPCGasCap()) vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) - return api.traceTx(ctx, msg, vmctx, statedb, config) + + return api.traceTx(ctx, msg, new(txTraceContext), vmctx, statedb, config) } // traceTx configures a new tracer according to the provided configuration, and // executes the given message in the provided environment. The return value will // be tracer dependent. -func (api *API) traceTx(ctx context.Context, message core.Message, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { +func (api *API) traceTx(ctx context.Context, message core.Message, txctx *txTraceContext, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { // Assemble the structured logger or the JavaScript tracer var ( tracer vm.Tracer @@ -753,9 +777,12 @@ func (api *API) traceTx(ctx context.Context, message core.Message, vmctx vm.Bloc default: tracer = vm.NewStructLogger(config.LogConfig) } - // Run the transaction with tracing enabled. vmenv := vm.NewEVM(vmctx, txContext, statedb, api.backend.ChainConfig(), vm.Config{Debug: true, Tracer: tracer}) + + // Call Prepare to clear out the statedb access list + statedb.Prepare(txctx.hash, txctx.block, txctx.index) + result, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) if err != nil { return nil, fmt.Errorf("tracing failed: %v", err) diff --git a/les/state_accessor.go b/les/state_accessor.go index 3c9143c875..2f49bb921e 100644 --- a/les/state_accessor.go +++ b/les/state_accessor.go @@ -72,6 +72,7 @@ func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types. msg, _ := tx.AsMessage(signer) txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), leth.blockchain, nil) + statedb.Prepare(tx.Hash(), block.Hash(), idx) if idx == txIndex { return msg, context, statedb, func() {}, nil } From faacc8e0faa89f5b3a7055e86924a95b6a9be944 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 15 Mar 2021 15:25:45 +0100 Subject: [PATCH 195/709] cmd/geth, eth/downloader: remove copydb command (#22501) * cmd/geth: remove copydb command * eth/downloader: remove fakepeer --- cmd/geth/chaincmd.go | 81 ------------------- cmd/geth/main.go | 1 - eth/downloader/fakepeer.go | 161 ------------------------------------- 3 files changed, 243 deletions(-) delete mode 100644 eth/downloader/fakepeer.go diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 1834dab6a9..2010ae4b30 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -28,14 +28,10 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/trie" "gopkg.in/urfave/cli.v1" ) @@ -150,27 +146,6 @@ be gzipped.`, Category: "BLOCKCHAIN COMMANDS", Description: ` The export-preimages command export hash preimages to an RLP encoded stream`, - } - copydbCommand = cli.Command{ - Action: utils.MigrateFlags(copyDb), - Name: "copydb", - Usage: "Create a local chain from a target chaindata folder", - ArgsUsage: "", - Flags: []cli.Flag{ - utils.DataDirFlag, - utils.CacheFlag, - utils.SyncModeFlag, - utils.FakePoWFlag, - utils.MainnetFlag, - utils.RopstenFlag, - utils.RinkebyFlag, - utils.TxLookupLimitFlag, - utils.GoerliFlag, - utils.YoloV3Flag, - }, - Category: "BLOCKCHAIN COMMANDS", - Description: ` -The first argument must be the directory containing the blockchain to download from`, } dumpCommand = cli.Command{ Action: utils.MigrateFlags(dump), @@ -394,62 +369,6 @@ func exportPreimages(ctx *cli.Context) error { return nil } -func copyDb(ctx *cli.Context) error { - // Ensure we have a source chain directory to copy - if len(ctx.Args()) < 1 { - utils.Fatalf("Source chaindata directory path argument missing") - } - if len(ctx.Args()) < 2 { - utils.Fatalf("Source ancient chain directory path argument missing") - } - // Initialize a new chain for the running node to sync into - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chain, chainDb := utils.MakeChain(ctx, stack, false) - syncMode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode) - - var syncBloom *trie.SyncBloom - if syncMode == downloader.FastSync { - syncBloom = trie.NewSyncBloom(uint64(ctx.GlobalInt(utils.CacheFlag.Name)/2), chainDb) - } - dl := downloader.New(0, chainDb, syncBloom, new(event.TypeMux), chain, nil, nil) - - // Create a source peer to satisfy downloader requests from - db, err := rawdb.NewLevelDBDatabaseWithFreezer(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name)/2, 256, ctx.Args().Get(1), "") - if err != nil { - return err - } - hc, err := core.NewHeaderChain(db, chain.Config(), chain.Engine(), func() bool { return false }) - if err != nil { - return err - } - peer := downloader.NewFakePeer("local", db, hc, dl) - if err = dl.RegisterPeer("local", 63, peer); err != nil { - return err - } - // Synchronise with the simulated peer - start := time.Now() - - currentHeader := hc.CurrentHeader() - if err = dl.Synchronise("local", currentHeader.Hash(), hc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()), syncMode); err != nil { - return err - } - for dl.Synchronising() { - time.Sleep(10 * time.Millisecond) - } - fmt.Printf("Database copy done in %v\n", time.Since(start)) - - // Compact the entire database to remove any sync overhead - start = time.Now() - fmt.Println("Compacting entire database...") - if err = db.Compact(nil, nil); err != nil { - utils.Fatalf("Compaction failed: %v", err) - } - fmt.Printf("Compaction done in %v.\n\n", time.Since(start)) - return nil -} - func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 207c93c0d8..5559835499 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -212,7 +212,6 @@ func init() { exportCommand, importPreimagesCommand, exportPreimagesCommand, - copydbCommand, removedbCommand, dumpCommand, dumpGenesisCommand, diff --git a/eth/downloader/fakepeer.go b/eth/downloader/fakepeer.go deleted file mode 100644 index 3ec90bc9ef..0000000000 --- a/eth/downloader/fakepeer.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" -) - -// FakePeer is a mock downloader peer that operates on a local database instance -// instead of being an actual live node. It's useful for testing and to implement -// sync commands from an existing local database. -type FakePeer struct { - id string - db ethdb.Database - hc *core.HeaderChain - dl *Downloader -} - -// NewFakePeer creates a new mock downloader peer with the given data sources. -func NewFakePeer(id string, db ethdb.Database, hc *core.HeaderChain, dl *Downloader) *FakePeer { - return &FakePeer{id: id, db: db, hc: hc, dl: dl} -} - -// Head implements downloader.Peer, returning the current head hash and number -// of the best known header. -func (p *FakePeer) Head() (common.Hash, *big.Int) { - header := p.hc.CurrentHeader() - return header.Hash(), header.Number -} - -// RequestHeadersByHash implements downloader.Peer, returning a batch of headers -// defined by the origin hash and the associated query parameters. -func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int, reverse bool) error { - var ( - headers []*types.Header - unknown bool - ) - for !unknown && len(headers) < amount { - origin := p.hc.GetHeaderByHash(hash) - if origin == nil { - break - } - number := origin.Number.Uint64() - headers = append(headers, origin) - if reverse { - for i := 0; i <= skip; i++ { - if header := p.hc.GetHeader(hash, number); header != nil { - hash = header.ParentHash - number-- - } else { - unknown = true - break - } - } - } else { - var ( - current = origin.Number.Uint64() - next = current + uint64(skip) + 1 - ) - if header := p.hc.GetHeaderByNumber(next); header != nil { - if p.hc.GetBlockHashesFromHash(header.Hash(), uint64(skip+1))[skip] == hash { - hash = header.Hash() - } else { - unknown = true - } - } else { - unknown = true - } - } - } - p.dl.DeliverHeaders(p.id, headers) - return nil -} - -// RequestHeadersByNumber implements downloader.Peer, returning a batch of headers -// defined by the origin number and the associated query parameters. -func (p *FakePeer) RequestHeadersByNumber(number uint64, amount int, skip int, reverse bool) error { - var ( - headers []*types.Header - unknown bool - ) - for !unknown && len(headers) < amount { - origin := p.hc.GetHeaderByNumber(number) - if origin == nil { - break - } - if reverse { - if number >= uint64(skip+1) { - number -= uint64(skip + 1) - } else { - unknown = true - } - } else { - number += uint64(skip + 1) - } - headers = append(headers, origin) - } - p.dl.DeliverHeaders(p.id, headers) - return nil -} - -// RequestBodies implements downloader.Peer, returning a batch of block bodies -// corresponding to the specified block hashes. -func (p *FakePeer) RequestBodies(hashes []common.Hash) error { - var ( - txs [][]*types.Transaction - uncles [][]*types.Header - ) - for _, hash := range hashes { - block := rawdb.ReadBlock(p.db, hash, *p.hc.GetBlockNumber(hash)) - - txs = append(txs, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - p.dl.DeliverBodies(p.id, txs, uncles) - return nil -} - -// RequestReceipts implements downloader.Peer, returning a batch of transaction -// receipts corresponding to the specified block hashes. -func (p *FakePeer) RequestReceipts(hashes []common.Hash) error { - var receipts [][]*types.Receipt - for _, hash := range hashes { - receipts = append(receipts, rawdb.ReadRawReceipts(p.db, hash, *p.hc.GetBlockNumber(hash))) - } - p.dl.DeliverReceipts(p.id, receipts) - return nil -} - -// RequestNodeData implements downloader.Peer, returning a batch of state trie -// nodes corresponding to the specified trie hashes. -func (p *FakePeer) RequestNodeData(hashes []common.Hash) error { - var data [][]byte - for _, hash := range hashes { - if entry, err := p.db.Get(hash.Bytes()); err == nil { - data = append(data, entry) - } - } - p.dl.DeliverNodeData(p.id, data) - return nil -} From bc47993692a446437b0d91cae758fb6be664050e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 16 Mar 2021 09:43:33 +0100 Subject: [PATCH 196/709] tests/fuzzers: fix goroutine leak in les fuzzer (#22455) The oss-fuzz fuzzer has been reporting some failing testcases for les. They're all spurious, and cannot reliably be reproduced. However, running them showed that there was a goroutine leak: the tests created a lot of new clients, which started an exec queue that was never torn down. This PR fixes the goroutine leak, and also a log message which was erroneously formatted. --- les/server_requests.go | 4 ++-- les/test_helper.go | 8 ++++++-- tests/fuzzers/les/les-fuzzer.go | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/les/server_requests.go b/les/server_requests.go index 07f30b1b73..bab5f733d5 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -206,8 +206,8 @@ func handleGetBlockHeaders(msg Decoder) (serveRequestFn, uint64, uint64, error) next = current + r.Query.Skip + 1 ) if next <= current { - infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") - p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", r.Query.Skip, "next", next, "attacker", infos) + infos, _ := json.Marshal(p.Peer.Info()) + p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", r.Query.Skip, "next", next, "attacker", string(infos)) unknown = true } else { if header := bc.GetHeaderByNumber(next); header != nil { diff --git a/les/test_helper.go b/les/test_helper.go index 39313b1e3b..e1d3beb6a1 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -661,6 +661,10 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC return s, c, teardown } -func NewFuzzerPeer(version int) *clientPeer { - return newClientPeer(version, 0, p2p.NewPeer(enode.ID{}, "", nil), nil) +// NewFuzzerPeer creates a client peer for test purposes, and also returns +// a function to close the peer: this is needed to avoid goroutine leaks in the +// exec queue. +func NewFuzzerPeer(version int) (p *clientPeer, closer func()) { + p = newClientPeer(version, 0, p2p.NewPeer(enode.ID{}, "", nil), nil) + return p, func() { p.peerCommons.close() } } diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 9e896c2c1b..3e10171873 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -261,18 +261,18 @@ func (d dummyMsg) Decode(val interface{}) error { } func (f *fuzzer) doFuzz(msgCode uint64, packet interface{}) { - version := f.randomInt(3) + 2 // [LES2, LES3, LES4] - peer := l.NewFuzzerPeer(version) enc, err := rlp.EncodeToBytes(packet) if err != nil { panic(err) } + version := f.randomInt(3) + 2 // [LES2, LES3, LES4] + peer, closeFn := l.NewFuzzerPeer(version) + defer closeFn() fn, _, _, err := l.Les3[msgCode].Handle(dummyMsg{enc}) if err != nil { panic(err) } fn(f, peer, func() bool { return true }) - } func Fuzz(input []byte) int { From 99830720f681e5120fc131daaf0ad74d329bb074 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 16 Mar 2021 02:48:54 -0600 Subject: [PATCH 197/709] core/types: improve comments in new EIP-2718 code (#22402) Responding to these comments: https://github.com/ethereum/go-ethereum/pull/21502/files#r579010962 https://github.com/ethereum/go-ethereum/pull/21502/files#r579021565 https://github.com/ethereum/go-ethereum/pull/21502/files#r579023510 https://github.com/ethereum/go-ethereum/pull/21502/files#r578983734 --- core/types/hashing.go | 5 +++-- core/types/hashing_test.go | 16 ++++++++++++++++ core/types/receipt.go | 1 + core/types/transaction_signing.go | 6 +----- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/core/types/hashing.go b/core/types/hashing.go index 71efb25a9a..3227cf8a72 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -36,6 +36,7 @@ var encodeBufferPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } +// rlpHash encodes x and hashes the encoded bytes. func rlpHash(x interface{}) (h common.Hash) { sha := hasherPool.Get().(crypto.KeccakState) defer hasherPool.Put(sha) @@ -45,8 +46,8 @@ func rlpHash(x interface{}) (h common.Hash) { return h } -// prefixedRlpHash writes the prefix into the hasher before rlp-encoding the -// given interface. It's used for typed transactions. +// prefixedRlpHash writes the prefix into the hasher before rlp-encoding x. +// It's used for typed transactions. func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) { sha := hasherPool.Get().(crypto.KeccakState) defer hasherPool.Put(sha) diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index a948b10ef6..6d1ebf897c 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -1,3 +1,19 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package types_test import ( diff --git a/core/types/receipt.go b/core/types/receipt.go index 48f4aef06a..e04259b9d8 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -38,6 +38,7 @@ var ( receiptStatusSuccessfulRLP = []byte{0x01} ) +// This error is returned when a typed receipt is decoded, but the string is empty. var errEmptyTypedReceipt = errors.New("empty typed receipt bytes") const ( diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index b4594cb90b..126748efeb 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -207,11 +207,7 @@ func (s eip2930Signer) Sender(tx *Transaction) (common.Address, error) { func (s eip2930Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { switch txdata := tx.inner.(type) { case *LegacyTx: - R, S, V = decodeSignature(sig) - if s.chainId.Sign() != 0 { - V = big.NewInt(int64(sig[64] + 35)) - V.Add(V, s.chainIdMul) - } + return s.EIP155Signer.SignatureValues(tx, sig) case *AccessListTx: // Check that chain ID of tx matches the signer. We also accept ID zero here, // because it indicates that the chain ID was not specified in the tx. From 7076e8e42ff528e0ff058da3e357343dfb6fd71d Mon Sep 17 00:00:00 2001 From: ligi Date: Tue, 16 Mar 2021 09:53:43 +0100 Subject: [PATCH 198/709] cmd/clef: docs - link to ethereum org repo (#22400) --- cmd/clef/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 700f0a144b..27c62817f9 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -919,4 +919,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | -| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| +| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| From 7cbf1d70a7428fdd247882a840c416c80c30454f Mon Sep 17 00:00:00 2001 From: ligi Date: Tue, 16 Mar 2021 09:55:03 +0100 Subject: [PATCH 199/709] cmd/clef (docs): fix image background (#22399) Flatten the image so we do not have dark text on dark background --- cmd/clef/sign_flow.png | Bin 20753 -> 20537 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/cmd/clef/sign_flow.png b/cmd/clef/sign_flow.png index 93ef81a32e8ee72d68f59dbad9db6b03ac7fb782..e7010ab43f3a6f54d69958055b93b611f7853951 100644 GIT binary patch literal 20537 zcmb4q1yodRqb{fzAfN(*G%`bnv^0teLn<(IC_{I5qaY1Kx4;0BgY=LpCEeXQbPq#w z$4~vf|3Bya=ia;5V!_^P^X_-zefCp3=(U0rJ}x;f78VvhNcx2m7S@eEEUaruw=oB> zCS=a;U||vHxv8i-D7|y0wXw4{GPN+Ib#S#Yq&0LgHNwJjnJ7v$b)pm`cz<<3Y=6U# z;>Y4W?U`hXu05orDyY(;N_5&bVn*qL^>O=_Q7MY>10^w42VI(p)zb^5Ezv%=*<-fl3uUo*l+f|ceCHjYOfB~t zg*AirJ=YyLiNRHrh~NrRr&=+sw=NJ#yYub#kN7Q5O31mGgr48|9`iZbg_tBDCd!tPn^rr(8NFeM4hV zy3;;3q;ZL%n|^1ieOZPvvH?1X)Lcq*7CB&F>#F;p75yY(H@4B^!>rD^Fx+Nm|6b6o zvPPw@8t7t@q07p7YOVCgDd(AWqPd7s;fp=tx7c8A_4Fss{7wCOJdVSjJFFY zXSMG+G~S|p(WRqtSDN!d@)C=DW)AFe^OxvP?H$S-?LT~mlK4a9fXFz%oHP%shOGUm zWHYmN#U^<`=1N@bRHz6@m6_z+WbLiVh_-A|1-M9b=#6n_QTc@C;0i#1&%8fh9$ZV1 zDvBO3x7Kmr$?A=I6>xpSL{f?d(v|q#AhTUP`i#+-O`}qOI4RGOf}eC(!*F;wuDGE! zNqDO+BWSvEyP+*j>bozqf*`6qD>vPC1;<=q)^5X6&AM^1r?;@a>~wI$+CY9rLsl{8 zT&mEl+@nr@K>MR$HnkNX zXI{nXq4=f9rOcB^lJpJrxI%$bwKXH-f;4+eoj#}pixWaMo zl@5tgn7nSJ6au}y0Psra((m#~v^{I)CN$0;uMeu#!!*}d_JYGti}C0?v{lF0e{J!3f|}sAcR{Il ze4+q&)ph;^YqeGj*@N~0=4X7i*J@KACg|PlQ#_zlVIHRHKz_NWo~c5X)m(0=!h*Wz zHlRCbkJu{D2iM(>XO7AtlB7rP;z!ITW-l1Q%I*iVow^yO@+x{7Ock=>`aF=okXR=&Akoy(ff;IPvo?$I6;goSE z`>cdSj7b zWn6(T?{@9zZbL|5L02`*y70L*2|R^Z*TWcc4`BkHw9h>+ESgn!%wsbPZ}=pmJhXsP zE4KZP^>&qH4#Q30hQXbTPoD@l!&Yi>#7e{|9X=OT8d6||=0|ppZ5LDF&(bt)NVY$( zoPTaKA+^0e?75#mcf88uJ(_%@^lF%C_}1ykwdZqsMOzv1+Wf+qa*AFz0UoCMKXww& zsA1nf3Tp63Z&Dt;V$YDGc~XpcZQ%L*@*}>Tt^E*wV?V7=hJCeR_Y5^hLC%9TLfRxI zw^efL5?}xaVc%hOn_s0p4qS4MUATEu#5|m6axKsQ>Gi@pi%DBv4w({oHMD(IMg8)h z0|}O5-za7Aiy~F2y|e^2er!?Z-HZy{YNDZFOjc0T;kqF=`o-fgNJbY9-Z~xn*g1;5 zAZ<$ZO6Mz6zmN}GFmv*pTsi4s%Z)Tr?kckmcX!ulkJqJG*zYa5Ip%{RqlUiRG}}z+ zJ_lxn&*8qTvt236%kfL8LH-z@13Z5|(Y4n2&B>8cvTt0+m@qVGw3HujyO0Bi6(Bz~3xOc+NM7o2B^J}3`{8%io zYXXH*4L4u%-!Q)!?cEwaPB+9^cTM1a-;=iVlY5MZtzzKo2(9bOZ49qv4!?V;qQj9x%aF5W%Drx+Tt zN>)%69(~zpWApgO^Nl-${vq#O;d9+x+2z{7owWqjVH)4;hiIFbqVOK13yPrnl z7Qe~;hwpLA9x(n04bf}^RzBK=7dL!xua;nMBDqdwv!9*z38?wrlOi^pmJ2_N@a~P5 zUJi|U5lVN)npP+u_mMK>b9m4OAP-%*OtRmAiF@pjFPiRNgwcJRGx{(l4`K1~j-;RB zgR0XueT)@NGdcFvEmY{c2}p8b#1go>$q?qdg%bzGi{oQ_cz7@9?hx4*x~z&PPg@Su zrMrSJ8US8}A9odH7utevUzPv0lMH>`=UwzTBuMOefRs$XYz=Op)^{ALgM?0v`t=XwG$rvg}HU&efxQh0DTmyo?rZ$WO?G_c`=5sr_d zMO8*3AKe&PQu*^|FXyM{<=n74VTiW~kErb%%U($-9{9dbe>@4UbkYl_&wroHc^P6L zr@h4%FZ$-?N`Of}+`_}CzDp}h>vm;W=~FXxGOFS;UXju)|0L;VZUw#jskgFy7)VN` zZ`Ut>)c1aCm!$9!r)j^Eb$&}t*9zFWSL8@HcSx-Au>QK4qG9^3Y8{`^q-w9~V1 z(97c>5Rywo;}^mjl~xLOA?4^lqdtc8NA)QVe0H+=>LSb=jLhYxeKhFql}Yhv;RVq# znc{hMt8cXAwsFpECvU<#0pVm^PYN*PjAF(|P0ux+3c=)oYk3B7<4Bh4_ zqxeB9@|^dpNf{eiX}a1i5SedD*!{tGT+gq|hTd!myJq>Z3{Q5&$;02dvCrw9aY&p= z#e-F86CwlGh^0r(`oc?{l!tN7W10?RPOZB^PPvR)U!oB^=8L4X)=-zd16yHL8 zy^Sz#^Y13b#!41yrlL1#^%sBP>F#>cb~yIR1*KH!J7NMAv^lxfmmV&Ty@|t#H`|AD zmpsSO=jD2h-Hod+%h#31`l)&$jOF3|SgXjH%&ewvpsUv< zoX32$hcX9+!^|2JZm?Z-k`3%wo?;8$W><(Nd&6q1jL$g;nqLB`=>gwS&xr*15=KS0 zdmZ^o;oonTVtUw+*=)tneMF`+;IGo>S>UdgiS%oSC*WAO+p5g$WoeydE9jN4HN6W+ zXV3N6C>Gn=k|{*EmR;68<**QF*@i7ngm zQC!1YcRU4Cw-6s!!GT8_GY_b5iiW;?yo?H6ha~NG@f05`tE-yE(8~|HT&m8;Q9Pa) zpA^3LBxWmdv$(wOFmh&B+!eM+NW*?LRL7QOYmx;lAq-dZrPRO-fdL6=>t4> zqnphoWuk=kc#Ze6=Dm2MTxAeRSB3`5x%)51) zr+U=ozGv{{Xt-2)BeT)X&Jd5svLqv|t*11vN({b_;JQ58d(Gmh42wOyR`;Toibqyk zW0lJ2I6OS6M|qBb@a8)|=308O({$d)Ti**`(vlatXNqElJQZreW#2e?KWrn$8FVuY z-#c!qR(jO$%0EA+I=xO@jQCLO?a8dl%8Nv$yNu7N4Z3NVVE(mMWPNuQgxH7P38Kur zVxY^}i$p6*-YbYlN8)iYe%X#ev= zyvQf5eH0b*5iCKH7Zh9E!8fa$ys=|;+r#Q79zd|Lu<1=DC0~OiCI3iGjKEAujr=0h z^qM%Ju{<-=QHipd&Oe!0yK6~_#P2m-E3RrTFX&4`b0my-_NzdD;2Q1i24BukbDQ2- z{{9*wHcx2+?JuvU@Ysd#4BEXT94KVbZWO4a)E$?#F8{Jx(qVNk&b9a8l(!XGU_`>H zlQk@YHfDQe`zXIBRwIFwME#b&vFdU_ti=?;IsKmhSoi?j1Mwi|OkQparwiW10P&=i zB!v{6l;j3Gig+T0y^=>
hBRO3@dk_CE#F#9yg$N=P#uXK3TTLIEv)6RKgPv&9s5iCjIrwyYT~%wGJYxRh{zyQ{>ahCLP*9?{PqY^8v_;Ic zoq6i$y6d4ulMQcRz4X5&i{&G69qSrC)@@!a+CHp%CRh@mu(5n^V%;GBx%2Dj?^FIh ziaAB1QlhN5_4=r3O-Qj3G%#e>QQP#QTb0B=`AV)q4MGL_Mu3lHrTr<9)p1Ek!KFV| zfxG;pUVGw-^`Q|L;=+8{oR1-jXtVFn<^OeT%;o=?)N$?gt&w7=eo;K`;gGqeIY2P& z$D5LgV2&A5M@bV2iDc%U#c)qV`vGtx&3p2Mh`*bXAO~-Sh*GoMRq^xgl$1*kBTOXpnm}!4YnW=`vX5ZrBCp*xH=yW{jb}KRR9?3A- z?&A+s$s6*>18l8iF{sUWeH9LiSz_p)xlCM1SHb;bUrx zN*L+|NNK(N<)jETO9E!#%7_$qMUlzpIsnt>?T(jbww+KqDDe|TGp{B_af5*xYUrWS zi4CVK;h0Tsj)10t<0MU*XH83|`Nb{qk)^cxYyeJP`N35AE7`*B{!Re1;kr1lTi+Km zoW0;{qfUXUvwMfQNJ7~3)VN;$Lt-EZm-SvWe5~o@i-Za*XI}E-Y#L#8rBTX3c3r

^iAoLKHd@uPVvi6u0{`U2DpnV73`@pvs-01&D(4WGl24S zpBRaIH0?lRmKfV7h2O00kpRI;M{5sx(o*%V<}Y@{%2PoR&eHqQfLL~B*Hw>ib(sP- z{rhMV>JNKSWpXtz2UpN~ADw0AYR1{J1_HeZU9xOE70@*4c7-mb(lRr*t5L^B1AsE) zIa+t|z1NM;MvFRf0l?}@JdTfQhpk_x517n+RasfOmCwL!t_6X2Q=lt(PQn``b;(|< zXRnl^;nAIDMVc~U&eFj#Do=9VJt?a-ln$A_&k%EA(8t;vB#XeBA<>9T5)(=BHCOBJ z$&R(4f`_PLR~H3$MU0Y5B?OBcE`PKqV>dm_pP^r5;NM{jTjSce{1~f*ow|+U zDl~wbNg1h815)Lb%sQT(9Ipk16W%>>$}+FluY2J-b8ss&#%qb_L)6iS$g`@uZiS); zOQ8g_F;VNTbz|skD?GsTgZYD{dTShB*4KmZAxWra@l&2dak>K0Y=xbk&!}}1lA$$S znkx?UN=c$Ty_rn!P+l3TK*&)y_BqmWqj6b7a1#~0_h8wz*IV$dL`MMjbtU!a=RZ2m zN9bkM&UK`v%xA9VQHk4`bRoru@#W5;PgM$5b=zVuj4n5zvAT(S?qap>8?*DC!I$0M zt+gW==R@u%2+rF3Bp*(qh+cM&BWqNqg%9pOymo^;_^TFPKWmQ)4GMsVeGUIZ`Y8Db zR?9rA!2bqI)t8Fc1yAH$cHeR&Y0*oS-i+TWX(2C>yzXrDiBC8;qz&`=&b7`cn@O(kqQ&f}QwJ(8 zs_UT+cB;7}WeTzr#OE8)?CBBlu+R6u!GQ+mwgabzVD-+uH5qLYk*5}W8lw*rFAQU4 zCev`KOK3s3l>u_4Y^>m4;BrgP#0i~gFsh5e>3B}V5=hVYZEtCrRd0(GzhmP;AeG); zmm0M)kji3$|9r$7X~HYKywy8N4vcfLKy{umbk0035Z78;*R>Q&ox9Iy2}%*==i>Sj z18Z(3O-ZHN**w^jEyy8r=frxu4+PU`cniYDLbK5BIb_j8Cb?V`B<5YZz()tCS*;1< zGEZ8`23@|CE!>eWLpi*gT*NH^K{I3((H$L{##HaX3rG@ z%|{oD=VYLK#9UzXomVn~o~Q7b!Q3lo&H3@lB~@=I-ohz%2_lmaM4 zli}#7Pr}`FOMd6FYBnU*&U6y_T{7r;j(3!-cjZ7Dg=&L3dYc70WU$S;0lj1?>b)sL zo>egob{zbE3BGkyRqchf+3UujH{26T5X7_#2XeffGAK4l3k^i47e3b|7)s>|1~chK zr2Gqq_{A)$ZTBxcO(;%sII%Fd{7*hxeYh)8Yw%o4FwmX6MLkIWK;~*(Lx~ThRD=tK0lmAEVHQ#XJyRWf_HLY1t8Uzo zoD))NQNLvJ8=C#jT3207Cq3f%9g5=-Rw?1n)+cvUEqwXYg^A~)csLUk)2u8B@|5yU zl;V|t^8hFe-}3z0PK3Rvu@S%H5T(P5fizw4xqIu<$bsLfmyVG}{{UQaUC``qP`6-` zUVRqyNPRwHc-P!@Mr^RI?S}}mB<_V~Tr5+2mqjrQ2Q)QL+ZPOrX~t~4ATBOTJGkR- zilMETqo7sZy+{X|Y+_+cD$i4^xkzah8Fp${iI^TxOw~8`v?)d6phcxb#wM47YTK!KCSAD(7)vv; zOB-KLH)5rg4B{lLX9?qKP3RMQlnpm-I1iDUHBkAIT(}dakv|)X;G`W-Sk^QaCtc(l zA%pZ#42ti$i@4M>-jW4lER7v^=Bq}Rf; z5o3?#XBgNg0RzjHk@3HPW1&p)3@tG0aUkz6JDk7c;?@$i$ta5>>l+-Ksp$%uCN>o( zjoO!JO^!+yK9e2x%bY^j2@gSqWRy;8iwcC@n0p+{wd$=;4Z2)FO6uX$I%$w<;rM>P zoeu1Xl$Xl&eSLkecVvfg@1oscs*6R%jGb{YA`ZGT`_W5VsV3s_pF(L6aSy8o=53xH zQQ1%0y46YAc^y4*-JE8v>>0%w?T@n_KRY{wLSgIU6iC56GxiWHR) z%<_?_L*v=$%pE8c>am49C(!~ymws^=jNizT4vnD&(GZexwiwV9xxO}y zZ@|^Px2;MfSyj&+x!a;dByUnH+Uqs8JGT#LR+OjX?P)TBkl7dx!)Gy8^3D+?#B;c3 zVd*a*6C4RM@CKQ4h)!g;R{!woK^Dq}IBa7U66=5h12f<9Cy#SL1KutFlp|J*)|rh-LaCti#Ws4)&DxuWz{gZ(9q*(u+FWszF z3`~Lb)-Co6xdGs-&ZN9EE{@W4a}o5#NE%9%m{;GZl(%uZduoA6HMl!BWyc`p(Tc~k z;v-0?hEik9!j_cyjB;CU&qt!-bh}e>2)}s&*GlK4XTjZIN7498C!*Q)>k(v-qxWD( z-qYFZtz>2Lga+vybu|q-Gb48(3WWHDE^5^y6H>|f2_YyabIb6=8Y=qeE$#ItKAE+c zvJbphiek%4%02om+$X{H_Suq;aXt9ka6*$)4yM9R?oYuLg6*mV-3(iIMlEsI(geLh z!FzM0xl3CT(~q*$ai|67%?^AvU7d~)9AI>h$NOyT+vVR^LMV+zGSyP@#MVJ^H);ED z#o>H~*fwE%MN9vvDW5u$3qqVqEp0;|3?ae1?b7*V#av%8lIcGRE(}Ha2haYUuS^WP zIXYPZWee}#g4=yeuR=n2K?7~}KoIj*gxx!zsoms3d$lYJD`4so$TI*HldrveFRH(r zQ#1*X>fw|OXx~Ug5Fxm+e4WN`E^9uxM;n_rvMUtE0HO#0J5ZM+HZnA|$$g}H&Yucp zo1fXb1KUdrLj%9+Tv_n54+sG*aW7HralLKElfoK}g}CN|Vmd(~fwuzL#aQFr{s;at zY*_Bk$y@9GrqrBMNFd9;)uy3W$|e@;8oRjCP2m-`&C5UlkfC8t#QC_Azth@X6YAj6 z!lUOZ*F9t6)G?V}#V5nIH?|&+mUqgFA5JfFaD?-xE9`!Zt_1W@CnqGRcZf~w^@!0l zmYW6JQf>Wkenu)~zQ-JY=SNt3yvm1!Ydh7gI+mn?#L1|5wHdxvGM>dZkS-A*S=pN} zt(#yd2EAHCQY#cpl8XH4gR4gE*Eq9yzwDE?l0_SZa^aghi5v1veE);HdOKb5Bzw%uiOPM*5;mLi`eOEBEW`HDlX>< zoe!Prs{lx;$Y<2h>_dw%k5q4qyLg_A3(DD(8$U-%DztO$qU}*r%5yW;k&^8;Q&J+W zR>1O&qv3mHDO7mph(~o40BUzFZyysWN+N%-iu{X^{G>2F`2ZQADsioq6UufGIaLh~ zB8gAS5#X}khY<1mrEdndWd%fZo(=R&op!R|=hw+Y?UCS1hog0Nv07TH*Ok`Yx6c`t z7v|eLytDydMQ0@K_c~iAJ&BAXN?|efrv*c%u^d@x6di<7a8;l8Uz$|>?QRk(VTFH z22u2Z=M~a%d76xHM;x(sVE1V|^iAP4;qu3u=j(cl&W)GNVW}?$^B-M>d5(ALYCoVQ ztqXKoli!^G&`+mWUjHR}my(7NBx5=n&U)>hiCV)8dZ~v#8Y*xO(EfTe4z|nEjo*ID zs8+ulp=AegG1E^N7|S4G8{MWb54qh$`s63S`TuuluzUk37Ux)}YwAndUC-Ayr_9*D za8uTG%zMqGxh9;MFnvFM*;F0iW%bnD%&n`93IO8kSWzwdTg^~%{-tK7q^|qk)DBuE zqKUQISUd0hq+Rew*aT8HuOIxlV<7!ot$;Raz0eY6qYt#h9sVepfp4Tq)AdXmrv;bZ zK+`Eo5Tmi9ndnA2jGhme2Lout#qQm!-bnAxd8wHTlmFHzHokx2GV06Y-jn28G}F(l zGZ`_t!7Xk&sr>EwLjdU8NF*rEFuNwy$<}KdGqV8w`zAnYt&9DJcIe^sMAc)NP+4a| zs8Md+g{xN9VydT@m+3HZ7I#)-oWpu-rPA{&2oGtqcrr&tQrX$}76KwY8*0jiqVE04 zv^cb(Rf)oqZ z3Zh6;zYb zPPdEbDh=%~T|shd7H9JLPr~j&0&k-ARvXg$7^{UKv=Sa zx>=uo)nUu-gMnadL)iW_AxKP_puwheImGlRO~4UK%m&&oW&`&nT1Yzo5cCg0vGD60 zD=SqFBpk=NikKU8bH?|7rDqIo_@kUa_#yzDG&@14M=I)G%)I_C(~j!>Vbmdes}jH*|Lo(CIs5%Ym9E(rZ?I5Ve~5FkL5-;By=- zS$O+Wza$rSSxf1K*_N|kfP_%~RrN1_yaLs7THidXF2Ja{FY!0QGYsvB0~T)~cgM-| z*LQb4O>7F#;bNu{Cda$cEBU4aw`O2c*T%Who|Q5z13Wp7TnQy;)wxKqBp{Gh>l zR(dDK0-0R$2|~oYoN~$HsF~gQ*?8;W7C~`!;Uj}n!^N$JpfV|qBc*hN|B8SZ>G-x0 zh>!Gc(b{~8Mb`MrEhiJE;!_9j&x2KiK#+jw#nIOZxJxyqVSWvB-qo>tS{Wb@ReD(g zY`;=YH$LqyAY+&NM35k~1W!D5|D1rvw?y8Aen70no0)J4)Ay|;ULU^iKqUMber z51tfpl>erkaoy24bU|!Bis(n~!3Mc2jrYZ!U*?YFw_h-5NGI`W{&Zu457`f%w!>I9 zpQ~RTzsCA)Gi%e=1*&!d_NX-82HwQz!`EN6F9k*(a^6}mz4?zpj3Gw9Eb0HjyTK5n z-zfU)lwd9|eRi8W=fqmMoV8#6Qil@X`WNv==9ZDTO>E;C*i792eZ0AKyaKnHjl}dq z#eJ)NR~sDG^P6&f=2PLG2g7fdQ@WZwDrx(CzEhb99c?XFRPWmOJ4R)UdgoNynh$rW zh3vW}2A){FxwjWqA<S?0sLOR=r+&*HS75oaxeDY$qhk(<(CG>HA6)>%cUs9 z&XkU)EH7*26a7(I?$Z(eJ#&&}p<;~ZOc+V(9#r{CZL_5Cm4E6V6&btqOE9hue9uOz z#Y%s|e))rZdR%b51V(g084`aa9AoP?qP!y-d66=h#dSZ4g2+Dij|nX}>zf=|ZsGaa zR``V~r?_KqD(YK%8mXNr*GgWcY>wNe)0&{;M5ZM*AtJOOA)%0Jp+#3|=H#-)wE-@3 zOIGf-x2K*p=na==GDe@OlvLSlxmL%K2ACWCbd_?x3jeDOWMf|(c<^aUTn=VL4rzE} z@M>I)-MiBrVJm1{rI$_t`CCASju&x^o68^ZwOWj*x^rG!S0q13?!AU)*RN|Ohjh8R z_eP`Yxnxnvp4t#O6B=YXsrxwGTHZg(=y_S_zWjt?C>OedcU-gjNCYIqay%MAMINch z=;WYOW@Zi?b{45$kZv6(q~4g-8}C^r9`2A`&ymJx!)=HGsuWSysKUFF3ab-s=8k3wVlf1?bYmR*&c*S2_O}-GMKnWavY8~v?r~jv%lKF zrOfzdSU|yb#;&j>H;!C@IA~_3rz#^KDfY^U)6|xgiXad&%Y|ltKpa_yECm7yQ+q28 zLh9A`OO~P-Ts=L^{9y!olo5TeK?9wgmwR&> zh_udj;b%%pP&lBS$K4b}zgDialHX@P@$u*b(kE8jSDX6= zB6U{=B>R6+M#0`a`s(cX^u>7CAK3ew(P7ZbM|(G{;w!MILj5|%5yKEbCTT1FLhQe7 ztbao{%+CKJ2FZQ&{nFL(w7>h-6n=BuU1j?{xrNG1kyZ1s$DR9yH{y z)kXGaj=KjMBHC1=c?+O0Z)w&uR8msqMU$=f?34L^dfcrC_e~@+PDX_gqxoXwdWMAn zM?$+akHv6s9JNT(`tFZA+8Nz+U=}_-7LdCecyH$sJ;=Rqx6SOhLd3e@Xd}`soLo{L zC~sS6xiHGoF>{m?DnBg0IE9Aty~;_`VSric@MRQUXnU~BD!InHS+!k!^36KPuz=Is zPZhcmn1f)38bKZC!dA7!H*N5PUXI0yb%c98#l$hr3=MMPAn2z7f6wHrqw{|Z=f>8l z*tqDK3O?N2MP`|U95~Rv%#{X^Lx%#K*^6^lt$~`VpV@jx@=$(ah;`aVw^gH@&!zn1W#Cmaw+pT1-CZyMrSP+4#|eQ8)rE{PR$zY zWkxV_2}QsSx8dE>g=xbNAx9hZzvp*$4H~9V>{yxRtttxXvH)sNqcE&fyYe(VSB=t^ zLB`lSPxEyQfSJv`{?J)V&G}F%pH+301YXt@YO;g6 zG2*G1sJF|T$9G;gV#urxEEA%&X7iNtEF1*QH@4>bW59t7Wo4*^tj#BfIJq;|ya%1j z?Pg_-L`lK2s@+d(M0t66hoqp!$Wd>Xt6D-%E-N>Wm!Mc}p|Xf%LGiz4pbY}e>>b&b zts`9Pf4hDgf;!oDbtEo0$1xb%!@UL_SO(@v29~pD1#ZV1FV95CLaRjgNwOAGME#&5$o&+g8}rkdZ}w@nLkKeFOJrHW12Pr7wYj2pu3(6bIqUbaZ5u~Ngu_R zGZ?EpovfU2S=l?*u(gcQ;(UbKcbYQ-_2-^*@>wlXDYZ)KGN2bvQ@RU=rVB?ttW6SC z8Bz!o=!TC@E^RE8Y8J$Cq^3kVV1TJOIwSqCPGG*fW*P<#aT^s8$*s@jEpEI;HKwb!tm>?))hs#hKk*miHvN6y5@0&w_7OQ7}nREavSec7#4gOHJn z`>Z;>h;+|I#7@;w;ZD?*Ml8))?r5{y#nz(HuszyM3cfDNZKZ$E!%=^J?~%u9Q9*;W zM@M^Z%8%LH6eG)8nP>hHWJMe-9i)2i8U z*VEs$7XijgSDPXYzB;`@>~X+JO&LZr6tIk$@_Fii>P*ztrzMULy|M4RzcxyTqT~x` zmPuMrpKOMMW9n7Q`D*k74f)77W6Cqa^yV&Xp@+||OCcqpfX>pCh!o{Qp>>GHypYz3 zDQJDXYbLBo^MR>`YNlLOPToLH=^2L1$2x{Q=uT?QxpUZmt7dS3p0R8dt}NO=i}eSr%?p~luQw^7Cx!EhN`d|W5Eyy5 z(eSysmXoBBwTS?~+5$zT2!FcZakRO~3C6P%dlp~_|10yTq`R0N8K$ZyX3SG5A^xl$ zA55Y$qcvj;`vs&j;!Y*7Vnp{RjZU0_0ctMs)|{Y#SwA z)T)(TUL8vi*z{#50g&f*^zaVukEcl@N82P?$PGW(Pm+qL8)lHDjuRPx{Y7YlWFp{9 zc}m66t~Qb$OU3huhlCed&pH=e!Z5(SpZ7mkHWcgFE`z&2!Xj>k#_My42K<6OR}}?l zOoDmKS=!Z;oHx=|*?CRd$EF=F*YG8*%wA@@nl-LJZDKW;i{VwQnAGUyK|iF^#mkiy z(o<@`gpO#tg)-sa6rI z-M2QJqE$bzhpIpMiFJE(sj{Puqt1x4htKGZ#A@8ezrr!820F!cr%X12>r`&1cZOoH zu8h}cvT(3KRc4^w|CX{c0kxb>(aCO__$_1W>8Ga`_2SvG1x7uej~Y`0)9CxKmNMHW zcoch9cjX`DrL~24>aABw46xr9Q=B$hXz(Z#FO63~zQ(37VyqVLJ)&5%D zeqPY|Q^QAte{}lR@Vr_U8C?wV6BwlGRtwO{>=-j`*^YFO(nGJaWTVoJt9To z&8`uAC_9Y}V+0ZhNyuJiFct4O8lOpG3f)TRn;B+~yRSc&NsZgxq zw9)Ro)C$#os(T9A{JxJ+`50=I1`prY+UXb`tIutMJ5~e7GBS22Mzd5un7g@$d2Exe zMfV+jn3Ju|Dni?C-^Nr=)MXxllol4|zr73TmC>9<)!s^~kDIfDMUQY6eAf(Dj3;R5 z+uVNHKA?xXVg)U(NDB!TC~s)w0-4QTl$80#)bMCRRh%a%?Kh(s?OpHP{@vAZTg>BK zGl|tXs)WMuFoH{q(`e7ywZ5cloKP1j!sU`$WQ@+HeuhNQ-{vJY>X{WtRA2Zq%(;`y z2AVDMWX)`@c8I4p{S|u%&5R=#<5lIkSa?p&LAko7tiVHYe*0(VWorZ9ax9yyFRIFB zVGJSMnfmQLFM9=g3=i3Z%GxkwG!ArHk;c`=$LWlt{?E8C^kUK7q{Og+3oj+<*%J9<}` z4W#*3<75zVr!z9LCqw8XhWpcg2+h2?HsDfH#`Jn5O&*52XQn3_i#qp+}gG2>jdde0}>MsgJ%`WdA92~H6R?Kc$ zU_{Twu59>T52r@zqV~z)qg&}$Pgl0ywP{bd&Fn8laqv)dIcafvIiNh@#XB^S=DnoG zG|_OU z6`fR6WQ~sCkHIWL1myfZ1JHg)(Edn(Jx;Odj`Kgz;H&e`2IKK!xr|VyT6IKD4&9Z%J7`k33L5d{5QvquaG;9@PCDA?S7F* zn8azz+qR_H6f97*173P~9k@Dd*ZDgT zL<;SQi*s3GTRKj^)}rkiN4u!!=g&76Mf5X`#{5!aY}d#Cl}cs2ZNFq@)TWx#68J?a zf19%Z5zqgvw}To>3tp7;A%(EC?sImy(Fw1WPmZRRF`>iBJ@A4 zNomy>^pr^SaqAYzjU#g%Tx$i@I>;b$ay)`UJu;4Icefy+Tscx0CBHrF1!355hs3Q5 zU~E)aa@vl0qHT$mjzzA3sT#GVfN4Azmo6lXYo$(>Gm)qIS1TueT-GwRA&!;!7v>7{ z8r+_!e7BnC?fBGKyy~yQ3{>W?R?WI_nyxJX9{xSW8`d~kQz5lIu> zHdC*^;h|$Q%U{6z4BJ(XrCkwP<*aYSiODfN$hVKjy_{9VfXsASsed_~eIkDSCv9!l zTW4oVRIDt^WG+aO+uqj^0f4|0J5x;Dz)cwmTUEY1vj0W_e7O`_80p(qCVG03(yhyk zu_cqHCX8YOT-ak^2ixwZ`#PoNs_Y1{AA}8r_>Tf&H^Ey!Haq-b(I6RlK_(h;(O+#A zhoUO8H#FEy{>2k>DBHIKeN`xE$2F7=k1t_cAMX3 zBTD>{7Z%YTj_O0#5suO+eJs{ zEMA{%HSdEb2#apSvV4h(q(e~#)5|Z=CiU2-|jME2Q z4}Lxq@Vj=h-yC?YPcVcJvtF^@1_t;6(yugb{3Lw8=-s~|0A}Z3N#E}g{yqis@hIu0 zrp_kyk1&rCiCY_=_%srU`<rg zhDk7#2E-w%tSE71|79ZO#l0ftLY}Ym$OqqO+?s>$_5M}w z56hixFA*4j0scLc(E)4p&FT2LNC4QPrbNxrdUQeFAAlnZ{auayH=?6a>^XaQIXA|h z4QPkES0%{T3o9DeV^a&lmu0}`^e3FicYoR>^+Sc|xTW*2JU=8$V>k|mtLHEqTH%Ms z7JZ(_c52m6p3?nP?u>PdV25QeH-Ysb>h@$ ze{}_Fw5XdFusvdul@ZKslu1=x<6hUOeOc?zs$CF>N*$@!{*K5@5X^o_b`98x(ZI!4pGg zfk;lTm9?d-yk4G*74Y2{GwY;DhqaJ#VRLSDtk5Mg=CFC-=0N1q%lzp;k?X~Nzsn>a zuU%;_Tx*qY_`);xGa}V}yf}ZsDB$_KB0CpVHYVIy9EIK+O@B*=b*<>Og(D$nD1mDx zJ0A8ulb3Jlq{0pE`%r`u;{0}gzVa!R&qyY7e_+M^U3TpZyKJ4f*1L;y_diCY|5a-J zTZj-@n|bLjxOgpiWn*G0+?^D&kbS5$=ZDYFGSNw`3X45BnEU%bi~rAp{?IH-!kM*i z#(FvstJ;IuA7HiKi$>iMVE;<6wewJHr$hEC?cq<_a6|K5Rh=)rLB!wB%+&pHcD$&R zdfz~#3jv;eU;9rY0hku+`e(t_kFskkku0%EoPW|vOtZ2@HA(PQ9CLVu{JG{!(C?P) znAQ|;kH%V#?NOzMeH?7OtlHLBpUT8r6g_n-*DY5D$^c;3qA~lEUuMhx9YX$ong$Bv zQmoM79)?uaAHEQ8Tgbj300jrHO#(2kz2Aj)T;Tx5%~#50pIF(8gry6$^qkHa^x{=M zWOtH=>O&@~E4vfQ0Nrl+cmf=Aaer6J5Q3vqm=Z?_t^k7!4W(R!oVWM`MtsQsr{-lGdY`f4q$7^$pTZfdSVW2rY!YK~0X`UR{<;A(XaBx6 z!+t)PqXr6=c9nq2jLB~Nb@SG0-0m4aGE0Pv#o)$WBVo1N{%ITqm&R5ET?v~0FnBpq zv^v4RssG_;2$~LWlGyWFI%H;SN4^K-)f}im(*f2yFHHBxYSk=K;dp;JJx~B}bs<8j zlMVc~dX&C`+}d$%Y#PC)fw{!bCwA%v7F2~O3ECPUm8CgeQTOGqk{hzXm6gMssvj{y zC;-hqry8`d|6DPhnG>iQ2XGZqbY+y5k=c(Gqkf_r9JTaD5Tuj|IAN`XmgySh4P{79 z#Pv&}F|>VKJExFqYPWFe%oem?sTVo7Bf70q>Zx{RVJXU~>T)uFMCi2KiW7_?kf(s9 zHqIJ}4=&Xjm|~vg`RNWxD=kt%59m0X%`*V0Rq-?`r-$2TF> zgI31%0r(XBg;v)6W>PPzILa0#mPRV|gK%tP6=Y;$>b6=&eU5vX!lAPm& zil*#W>U#mEU6mTSYDF=yPxpCfbm}eU)Q$$g@9^j!cA4%lgTk_AXU~*tHV|y2vSqC{ z#tYwsENT=O2?&wJq=ZO{49gfa>`NesIEui~LsJ4CDv*{SMJkasYyk?IvI+#0VGRvR z1X%Hy8qj?+0pu^%bI8jHNI0g35T0PAfH#S{Rlg}P2H5! zxhC@TG&Kfe7k;$E2C9(}5%4N;y~%~1`C;Q?q=abNuru4zPB@@~YwqB)VmEJB0;jrj zY-t4=xl&V=S!wk8%NrZLCdCaE6G7c;PSymQ)6MHR?|-yZ7uG|yi>dHFJs<_fox*Ti zr5LO}P-X#qnz4@0;Ho%G9!2h@YG?_|qBYxzfgPQ}#EN$lpxk0+s>^}Z$MbV|Ge?JpX_@L#yFiaLVF=O;KKD zFA6T=_~IFfT$P}wlKteF1@tGjG*_zVN%&sbc;X>HBa2-gp6MtD8$`!9eu@L3Q~~g% z;Jmt&C?&)ghBk|kuq&bmjRju+9$Rf0uBtUq*%QJ*)2wG{o@0*27C{-XW9^DxlEX9A zn!yWsUu>l!+bPGkV@$jN=ku$}`mU7$^=S3Pq$<1m^8DnetIrqDq5cas!u|Nx`tb>B zzVh_aRNu>pa7FT9U|rfhg2SX;RqA{sVQ7IoZ`Oe6Yg+G#5wfSU$L{X|&QwUNS~sEi z*kEW&T4@CBI|rH5i&M7>EYsTOhzMeddCJ58;qu$eKODr4qU(@2o{S=&e*CfaMfUtQf+hdQ-K5pP!>LnR zH^Lq1PTko~O5t?e=@aoMYS~(n3BgJ1mT`qm-`Vxmi{#hVV*-Q5EO>FlRxh)ymeEMe z_QF=4dR|tYLM-_~E!N>S>GPoD5-VNgT(sk_{0N%&Pp1DnyYBQnS+6!wPTR4p$|!{~ z3s9klSW+|iq?y5^^yMRhp8}lmm`r5~^Ig*BkELz3;%5~n1{`PxnFJwAmu;eV6s@|< zoU)8xvNL${P9IIq(iYRiiJH;ZDFqH#_Qo$mMcxe!uWr5fbD7ukadOXavgvUlRX&@6 za4yk4ge=%kScV}tqJuUAk`Os+xI1^F@cMZ-8uO`|`FiS^htR$DRZFxwBX`MG@N7ON zfPYA%A8w4!Uv{1`L>q%mK*rT5AGZnopK<~Ne3kk}-%&Jx>;U<#wpz+e(dYzP1LrSv z2+_?Q?w^0td~#@g1v;*HA&(>R0HKk=w$o^$8SAYkp;AV@d(k)kwaGW8Mpu1k))9^) zf2F)lh{eut)WBj#Sn_mTe0UyAse>2az|bk~)zi9mt@)B6d1T3w5n7{Ek+fF;=7a`| zZBy}#m#I$60|_rf$7gmzVbH=OsHE}h0cEngsdh)kG`uI1QIboO2CQ!bgRJUs!dy8isM7rgPWw#9r zq3ia(Osf8C`H4FaQje`wL93-dlLA4mQj15_&}{oP+t>wf;Khys5n?5SnT2? z=0u%W%umV{QzC@ilv!ZG;*(v8qGs^vxr1(%x56b|x2~4+5GNCQ0;9k4_QAc=uWhzbIMFr=lPD1ktTLm&`*@-0N* z6ID7JJz#@qE+#Jq0+mIgT^J$(@5zm&l;lAmPZ|)&{{;wi27KiI6$El&1A)F7fItF? zAkaOB)cR*az!&gFvXW0gSFa-bT0o$uOwv!pRNSXG=R8tz$C|FLCf-xOV?wwUK#uU)HCNGYQ)(_ss^xR0GL}C&*55F=_6<%tn(F`dPe~Zx#nr1J z>Tp@uWjrQOIGiYZVf;&BsbdY1cQj#;-OS=_qCt&JReq8#xpeera`@?^D}&~uDA%3k z&H0$eLWM+?MC+T2Q>8wRgmLLr?shuHZZb2BnZUTZ{b_(#gk)fhaPq)Nk-2`pMYs6* zmS70@_U0|e^)C4`aJKpp^6~M?t%{osRx{!7qzdt9r-L#@yLXV7%Am>bHPH?4xqf*{ zV-`o&K_@6#MK^JNVNds|*2jic`j9y@pJ&eO-n)@pFq7>mKh9d$oMz;K32%es%;fIu z;AQ1X;__8=^(%Z=G2O{mwC=BTH*cx-l%5cB(t!q8IDNr+X<4>*ZmuW;X1oejoY|YX zosG4Mbv0m+Q433YRuQiC=p+Bot3;Pg8H(_Nce(-nkA_7 zEfFi@QXf4yW{XoExwESnVaCr37A)k88sxCj7r%YH%l9~#W?>gvmofr5N4qbtP2+^2 z$VuU0g1-tTKjwISC9@9O*5%go@qjdF>gQG_8e#S`B=t|B8=$^&aFJM(C^6;Ox9H@* zKGpMgUvO7NHM+6*@9@z;nVYF2hq3uymvRJw*C-ntP(TQqZk=F^NOqy7D{EIHf#T;p zshvzO;aW_{Xp~#QOlF=zO@*E7_fmT_b!0>pR)n;ubHrN{%D?IMipah3hiZQCv0U@? z9q?_XPK}Z)Iwy?FJ}Y@iY2ib+EJJFdDKSO-as=C7m%b-a<@xZDTLDgc1q-gmC$a7X z`$&WpgdJ2#8XWB|wYP3#?s)8xnhCxXpPghj@(}ECw4TfHQg1UlC`yHfAx@_@oQ}C zX6i*t#Upw~I=Og3li2Nty$9S4-4pK$E<{-KSyT)|_j}je za%CPgeLj6NNna z{LB>>zA_pL)#U1TY<1xj8KW4`u-)=+ZhS2yAgbFnZyTy-er#fS+d8_tTjyvs-^9|j zZf2iJX>4JYdThp@Cmn?^YE-kobSe-EN^CQcC?~pFF^iCz-zha?Yv+5V(Ab>?`NHMJ zZy)&`Qq{dj-Dg0x@WrRZjQlZNE#%AW$D}?pt@?0ig5-J8$6j=Abul`(M!aW|!aN_@^TTpoy&ts2Z%+oUj#vi?jRGnN9*Bj z80!bb)TZMVL6NlJo9e=HYz&ax%4F-#C+mIc#)IIS`+IE`g|~80Kt#7XImnZ2Y%@O3 zIND37oxkLWJoIiAc4nzzz79XtB+g6NYEGzGwXETqUhE0F{I58__UVt5HOA|7-#=68 znX@daORIyJkX~tr2+L<;(1Q6Y=s=+-5Hfa1*i@wCh=BGsrTV#J>A9@kBE1BdK4lH< zAz5f*8$~GHTe`H*OmBldG?|H7vx;E2&0S@Ea$dgc)l$(^V{faUq}%Ia(A?L^-GEaY zhV!?xS!Mz?E3$$7M{n1|?eK-sgA65;* z|9lZE4~m6nes1P$WV$!mBeSu#S>z?UC}HLRkFtZ&e1DgrAyQ|vx5*A=vB~O4Oi)BB zUS=}ZSKapd`ef7Co$Ge+zvAawW%KP)3H9$Ug87!nlCgGI=+QG;rZCl%e)Q!rpA(Gf zmIGzF;mss?&0+bM6cwmp73CP0e|{P7{k5A`I@)ojSyZJVDKDw7gbQV{6y4x~>Y(Eqy`N+QU`;01^-%Y2x;egXmFOP!bF} z#}CKICdNS=FJE}{Evcyuhm~^X^wo4V@1UhG8s)MjmZ{Ef8>hCJ%$uO+sn!8A(I;lT z4nmUH9RXR|I`}q_P#e7O9^cQDi~CiIXt8(WA*%tiwf{PQ+N;`h7CDm#euvAz13&Zl zdm-Pk8KHeyRLzNxz%YtFEYiEx6Mw(KgmCxVMv7Drvbbn+Y7;v3$=g>cq>wL?arCXA zrugkhHVz-Ai66I1{{nORd^W|`9#ad9w*4&mJLa^lreE+kzi6%Y#6je7HtHsw2gkdf zczITV>fnOh#!zLe12}nA+=MUyE1_7|5ekR#rJR`zB=knT|C;VP2g8);Ng z9cd-wm6S#*X(eIkEv-P_*Q=W^XmVm#Tmz`M9V4!@#{YCoJdGz}owh2Sl#+3EH95$5 zvE1A(QfCJ05sQO(;{#ixwuXefUP`h`G0WST+yVY3%tZ{Ks5kp+MjV*xHTIqP7c$V>M*rm593gLP=c+gtZL z<}a!P{9W_XsHLYRQkQJttJ8Oi8u`(4(F8eP=fgfHOFiIRlSIKZGSm|InK?SHBk_)8 zJaUDi$5Wez-50wj;7HM-YPU~yHuT)MHUvH%w1W(tjCthA76FSBsK(sr*0$6}O&rKM z$V3R7jIB|$b|luGV9fTX?^8TFFzggTwzF()I@p4^ zu4-V44AuPp3jP#0xi5E06nf)F0tyocTpD{m$&k#L&A(xTepiikAdBRD@o6S+Of*90 z5MQZ}?(CTyd&io|H&dOqiE65<{)0oC1S(Yt?SROQms&#^iyb{%aEuKgG$;bR* zW;I|lGt5FGu8JP%^OlQ(LIKMNoFtinxs=n`b+k|Z9h-VrB9-i;DnXN{T!gUcv88+j zrG8TxJ6+*?ypIiLfD*&^fJYVC=_lGN>rGH zO@v;;eMRo3T6%=}mK)Rsx4%E0((l=vWM?kBlGy%K0RBCrI6=&txLuCN(bmr*WqM^! zNnQ=5J(76H@g;eC@+b#G{|g)oRFWVki*eRoJ|e69u>AO0#~bnMBFV*E@g-Fa5BIzp zZ|-Pv*jH)&72x)uXywv=j<>gr%T6G!d?SBGQc%2JjO^MyJ#IE{P8ig^sXCwkC`?u} zN+m>poJ^gtVngHsz=c zs;*R$uy`WbeVXwJk#}&r~3}Y#M<(a zSw-H5)4S6A=bs3dVyL(oA~Lx>3+do^%F}ZmHaIVBvrRitn~$a4-^g`6vz-cUusvM4d`B$Tm3+vy1Jz#T}?$?v=*Pil&uN9{7< zbWi-EVD|Dmt>mtJ=dKOm=S7QLFuu4POwm$PK7v_2Lk-5))1zL>S_5W$HIypCC3{>( zedI{ywjaD&LzQo&*P*Fx<$$+p->dV$-EKTCP2EI@gZk`AA-UQzvlq2O#QZReXBSdg z`D`KI46Q7Xl%_C=5~4_+f4yt|anEr*uG9Xler4nqJI(cDZ{+M8{$Y}q1u`*u*@w22 zf~VKzA-RFQ2~9kae_iQm3yHKFrf()44NOcrp7bSsva2@Z#jl%pAyRw+wqbT_3J3ch z1gnV^bx)&+IoJp3L-rXI#$wgMGRRzG5)Z96zSFvfQpc4Dv)A+V=t=pRWl&UeMza8O*n@_*#trwSkj-|?fvun=t z4^{jP9;@i;Q4K9Je>zp?rj}M}hus2rfExsFKYH749!Y@*fdJ?v~!h;?|87e*B@I*`l-SKgYjo6SX^g1skxHxE6!I~^o9?NN6MK;V>rvdg=ZxmUc`mn zc12k{w^?+@FN;7Te!3-cM9mknuGuD_mq|U8(xB0Utmbg9fvU>U-hTZErgabXgom}_ z`EhLmsk9tAkn-{;AHj%cp01RoW7WPIfK?++?_g{(J2~6G_um(ex@KC&;uVBGSSc9d z9uG4zGhe5vhsc~E9+0K0lMKkn&-52>%M#v}FQh$59Pa5y^NV0ZojzY7rGXDb!c8boY6cy=2_^t?U5d~V@IrBRS)CTLwpb7Y{*jDd-F*Ib5gMe5KWZE>3@0i-1LJrC#4F(0yPGGosR-N<; z`2&0bxfkC%kV|08CQ58Aa6$?o6Qp7(7I^o2U$xrsN$H%_^gem3l|QHTXYr^>Cm=RWf9U$P*qI&RpRNS zF`#mX>?uwmKBONi4-jrjbJ<3j{e-Tt2x@M-N2#Z6&k9rs#WQg3rO7@+5m?bVcX@Ia zRC|4F^v#STn2TPSpetOnU{-C!FGthcT0b|s$Y@HxGO2saeH7fF?UznD3Fi-eehM4+39q(HJ^!FiXBv_hMU~HR@kR^A z8@(%MP9R+OLbIDrU2+gvzDEp| zWy&7Iz_gNR$&7HoiGy=izfW0UHi`{dXrrLQ*r(?Q~p7&-az5+e`D-dkR zI^3P82ykh9C=2>o#Up3FR;**vHGVKU>OyA3oUy|uF5S(-L>&!_9ObT-QylnYb;uX0 zIjCQkXy68F9z>2X$6nWHg@cjqj@u55q14s5@j0EmzKhS1oBBQQ7i}PU7bma`T`D5i zhrdg$`l`Uu=9MBplPX$RVuC71qI91z?o|Re#&k3z z!DD@yx`q!~19nbZB9oq8Y3?t^Qa{jbWvSO)9{)>zAz|`tvU;nffHWnWrj34JdI^9& zD0HGabxA1IKonH+;Div=fa6VRKNUaxj<&qo5vwpJ*afw%3Gt7>xfe$5p3p8m9CtV^iRvUPr=w+Y0RCu}_RdbU~S*d(q0BJL`i6=2WJS9~I8 z_(NQ?uTW07EcNNB7GCylZ^|6Bjn8Ngu#K~am^Su2fEC@q`=DgW_rmEi>4LnABn;^| zHvqap54RyB;~TFb&Mrpc{HEHERCBAG<>IG@qOoFN=ZO-m3dR_I(&L)FmpTb0s!F-4 zHBOXXFiS8!V@&t6kt_AF6`2Du7N^9=PMAfjJ8iA!RpSY>szK&fO38RlqcP-PtbZ-Pu6Z5v=isd z*su?Crn0m~8_7r&_DnevYj!vWLPy++$AnL)QfJPwmf4HaP(L9mw<~EYM4-S|@%Rhw z*W%(q8DIok&A~y9of1(v@Z7r z&fsoJ;(*{rC$;xS6_CIP@(G9F5Sm6W;o`l%6+0TR)E@Pj?I?;iQMY?;1teYPLy-wC z%S64sqi{kQ0_RlJv)MGgxzoG%z@&raNgc5({MECUw9=;Tc;>ddK>9jcT^m}#>{cFn zm(-A|4cr>PAasr_JJq`3{FYsPDxQ%dlCh%0u+JR9Ix+fCz6yG4URENl`h>x68DYF4 zae;a?kapL6-i)%hHnJ&3tH$)QEO*dP%~%LH1dex#R4!u6;LGK*_FI?(lZqt(<5Kt@ z%hA@}K6e+N$Rlha4=1W_{Bp@{XnuA6;M`>>RNFmrvd)o+ zoQnYcX_DwbiM`!y8qTaiy6y2Giub zO0pN=AG3w0IK_@{iCo9OG0lg8ZtkurTTspcb5>Ggk;lnTGFC+9(>K(@&Jzbn_+_l! z$O?%4-l;{atdp*~r++bQuEsdt#b@LTe4HY8{Zql3_Vn$87bOVS7tLx)wz19PY(D`~ z{*#c=9=j5@t!=n_*vu9M?}@_=#_a`q?xkB-qWpVQg>K@_5&8GCLn=TK=c78)i ztEAK6^U#6vlHP=Oz(8}JM=|?#(m@siO4<8aD08NN0LN&g=}h?xc$POfl8$7;lVL3g zzlogggB4--XtJyGks^G!uV#5x;tBzm<6zf2%`D z%K>k+E<&9ksi^9;k;HJMQ7C<`?YvLy#L7NM;Q;UHRVDSwC%bn@m$&sQiB#Gn^`Eu5 zRp#tr&?aiudA3uI8kXrl>BCy;%wFy$Uy-a<53Cg}iR1zMX~h@niG&57ICWl@mB|)` z1qYu8x--%H@CO?+%#OdX$YAQfZDcoJok}=or26DZBgv+bQryuq;$rCD!F+C`ZTc@l zXrFW&|M60G(K&E=`GU}xp6K8q^b}n=-(fX9_>xJ?JUB1uVcSe|E5NSV{br(a zGI~7EdJeadywV_o>qw22i{q8ikS5C?r#yOo`c1X-C$GavN}yW|{WKQOkDfybYEX(* zc&KJaR-u^z!TGSN4>vfsqPBtckXMS9ywK=CEsOR&j;%kBH~K^R7U-RNQV}~AS zT0KrV5L5xp6}k{>*O+o{bB2FdK4REbu_P;euFEjCLSm8h9j7UukePKR9zIiZZ=oL~ zx9F&_D7NBAYa&0UR+AxW&-rlWE$}>Nd)^%~z%91yC(~#z%07@H{UU6J0OkDl9dbSy zS#e6m+^t*6G5ua3{0?>HpnJAQ(0a983;&U{gymE8o0jkxY3CG5Muf zzg?=ncs_c9jZ(o|T+n!EI|@i;0H-}flfG0-^+BjRLMR%K7@70sar84{e}zheyW39E zN&e5O^-K?-bta8=G;Hr=)?E#m$v{-<{43VIUoO&wrv&R%ZB6Xhit8A&GOVUok~I?K zKHJr0LeVzM;jK20D;NeBt`X)ff6*crrt}s`EF`nvfP-=>+|P+qGM3=WR7{2=dpmz! zfG1wv3j)ZpVB_Ezx`{>HILQYEopvmZ*xkp)b<>W2(mtw&Wa2ul;YF+NB3~3Cuor~Y z9(dHP65|!P76}SMYMsM!P^Qqto{aLIb4Q{Y$6Nhtk1K@E-$|#asy6}&zdJyz1E{a6 zJXRmNel zOlZBJGbyf?=AdMf5*2!q7^~Nj)WG})@l#kEIdUR@5Oy>y-1)J*(RRQ^+_D5J?AD7H z)4drgWUER1u@IY`&i6j{M8_63b{S8nWX)Gx{m3=VL*CBQVM;BevfNakWqT(_yw$d5 zcXSJ6u5SNaPrRYevAQ^jINocY2mTgWegX3zO5*3QrwNDv;Pb@53yKwp|Bg1%R-Ios z^T+=ma{uJg1yNswjA-9QX?#dO%zE(Wx%oq9p3BMi z;^I4?g5ALvr(WL>9O)Z;|bfgZ;O&y>bL##I9u5g=;U+=al5LG z84G7^D9PUVUrv*rDLq)^@edD*X4`(*;qJ<`-UB2#LthT&b3?q0nx4b4$gdwT4ssD5 zdlaWa`(okMOpsxnRpy~k+YiEnDciKYXwy=3q;KNw%w)KeT=!3LXbnY zYv?$G{!pJ3YB#Pv=@%zFubJBQe75)D_rlptn8QIDAbqo;8;|h6&ns>(d zXf!=}gRdfjgJ}a3$)*&-?aCM^DCQ$v+3A50BsEbA$qRI+r=Xxg-1UC)L=IzVQ|SCs z)j47$I1!&!{NsTI9%JE9yiQltVBIi%F#p!sQOJxvo!dKw z^0czE>m2agoGwi|&cvU)*Uam|CW@u?tCq+?N14{;O*cc2^YOjscn9F(FcbCX>#A7@ zWW0*DM2%&mRl<^2Pt0?PMlJir53*UE;3oUd<+9l7ntB8Y)}2co$Z%>?aok_m&E8() zn!&7;JFcT2^gSPae%hBS>X#LJkMEs~?Ubk2sY?#EZf`6}-|OXZmd)b7OgXc#7{uP$ z>zfB#lBJni{2~(++ILC%QQ5oCO|wnDaq!JkShb0LDUF_D%Sz=7?lpGREBEtU{iqCm zyQTtSdqI)>qx!hb!mj%mLcKDW?-t#iY)h9`YseI-9ykE8^u#sv1%6uy`#ImQ}2)wAYb z%E%0jPZdtWK6Y_kvTzJ`>CgGYYKTmEin;B?jX-LA9^K3a>Z$7aS9zCuEm0SE z;F*8y;}dd?c+1B)FC(KD5|iat88#0VE0SAh4jmrmMm5f;Wz!q1Ew?iZQvY5YzB|k; z&@o}`spa>mQjs(b=i_YSxh`Ju(nv5}_}1F&Qqh7l5IF|&UptjKp0g8x=O-t78oECpWYjyC0$sqntjQjvQPP&J~# zS9B7#s;h$=ihzc#2P;;DnknA!nFUEeVVAr7v(_tUtXi8jOyB`CZ@>f5_}f6S%)Akj zxx+i%EGsag{C^FlZeDHS2YIjO_ySEqjct5K*{?|<#vV%6zxFI3LZiF!MLFg}1%^J~Le^*@NGoPyD%t-tB@@DZ%R%?U? z{hUxXPxV3gY{}JfD}?C)+bV4$7>Q7q@ccr(v%Uj2fq=f@S3T%YLeNh(iQejH#>rwY z`%8bXL_)2NiPM4e{!|AsB2%POefCR#7YBSq7bke#E%Zu<=s)T=D4Kq_adeJb;K_Dy zuscWaY1HQDulR+4_`YdzFD7~}O@h4aZ;qVYA;%ctLL4_Ef;- zX9*@nORZfd3YsHNJLwbt01_NVwrKAHF)fp8Avh4Tk@>6hMZ^0a$c*(6wUX1ovBiCH ze&?vw?9b^DEM0l%d_8wnRQsJSI7h?R01m)&B%uyuv*MHcEOAy3mIiSF1ogu3-kDc6whVO?t+zPC~!p1F$0hskgOh`9-58%UA z!aFc|kYb6+gzRaKRN4kHG}`-mU3@WOcRCUPn*35icawZ2;y5)qYC%+AqXARajzC14{>S&+JMkMQ@j-o(g@0joosy3I_Nss6-giaiKAYW7i#vfY`V7yw) z=(bo!w_8Sbg*|%7k*+A*ts;J{#RF!is@>gQpgiLoYYgQy`X5+Q&2>8XIT_RlTS9z+ z8m%i)10NgP+SuDUOkwvhG2PA>2r%CU0aAir9o$Ta9k}!g8?3vL9{XFxuiV-!Gp`H_ zgnm0<7;p-N%;0&HF3rAG_inl?S;IaMNjl6B%#B^fRNpz(P09Gyd;_YQ%8^Mh1b`%5 zaRHqCDbp<@t?-(qK^rb4UljT1?9D}ay;a(S?h|a?{Kmn_Xh26h=cb(e^w~-e|G_of zP9wwFlCX{}M7+KR^uL=uXJSPw9!M9ICE-rqR$_JpV3I07{TeM*I59J%s|Kz$sQ;mO z^ryS@$1eXWg8U2$wkTfgDUt3qdNsR<0dxC8M~jURhky{Trma0QQ;7EX z6yjDl??;r~ne$KP)?SYbv3cWL4cwFd;{kqqe%*WX-kn$Ku;PXG04-ZvTl-QL?6 zXoLG8z^QYl6#Z?5Sx0HKl2?%<#T29ZG6T%7Z%hTk(Hy7}$*D3x2s@>26LAd!F15ek{@sUc;xpX zU8e%(6#+=yl5MX%Sn^wF+jv2fVUg}pF$CS-__Q-bG7h}$iJxdSl~d2}n$lpOmnS(T zw(sMrl(~71gsnBov3d?OS}w3O4jw-A`raQ4sn_1Vmr2%Q%ADaKiC0~NHF&qN$8Alc z*k7%B`>GX(AaeUr_M4*X!L!&bp8G4O0%-y=m+i5Cb*TO-lW3|z%QD&FqBUqE9l1cq zRC}ovM^n4*)mSC+B5r^&+%vZXxae|?bi2Bz{|i;($A3}XUM)myBN<}n=4b^=BO)BNCy1!>MUXKh( z)9d7(pA1xP#~9>lM6!TjfUO)OIpLsQKk$}{qhzs0x*G|mL-oFcC00%F(ejznRq08h zJh94vmOj*Y-VEOBvw>GXUxeuU=YA^enQ@!v2)B>kVx*iP>~QK8=D3g+ z)ZE;UCqD_#_B9ch{t~aG@>qD^(`#zUCOGzh<_D%Ff0$*Tc!5M#3>6TL%oa=*`Yi`F zc&KVhM@TFrXw_)eONw?h=Cdl*{1}!MTurbC6O^Hx;}r;-bVY4PTEpqCl|_3f)8#%b zuU*iM*aq1X&Cr+Zt17o@ZiN!bx``cgu@?pj zUy!kPz7&8QGlK$p%Q7M>>jb8Pl$C!BlsMvHBEc%=fVK zz=VEs#*4qDE{ect$%;H=G=A1o&G5a=k5navPnxPWRC1TKJRLX^z%FXe3=Ip4b&*+9 zfuj4m^F|VQ)o!3t_JqqiNQq9O9^_dZPr~MT9s1IJQs}eh$LEmOafPJ}4J_#9sG}zd zta$llJAxKf9os-_k%(`cUsNlRraE~3!^}pnLh2w2Vi%kPMv8kFIJNU`XpB7@jA3Sv zq!L4|lw=#i^kIbxRKk1Q=$-+Sgl)n)-Q+cE3>s=IXwUul4grDEtM34ZJdhdgHCRMIT^ zT>b)w@3p-5Yw&Q}_`TaIzPcKNSkaj)ZbJZ0qgzauz-=BD8H~pNA3<68{Bt;B_!%j(>V#+J%%_#RzrK{klpCiUF z4nHog4LYWi_pK=sB%=~df=hExu#CH9F*c*e5@!hvXB%L?kx@0;C<$C@FfHQrCo-m9 zUq5~M1Mt(LjCi*gl+_k=+Gv_bTGNYrD(h%`^y6Go%95_QGNoNE@mcaMJ!1kZI;#mI zAIPm>%;?A?rAl$epTS3cQ$uTbX!VxHk!*WcP58=bZ4BIKTO#%}Rosb`l@msv|J(^+ zWcfPslM5|EaZo3#+T@X}W~QGDTO`aog(??Z3uN-@RHD`oeC$YMHxToEy*uN9TI(i? zodP-=ps2k_BbUaAywR-=A(R$M#`wOslrhkcS9!AQmr`ne{`w}!{i|B%ru6w+5p{Ed zzsKd@N2o5ucYXeOSv1$S8mBrxov8wcO7*Z<^STxc1qNc#At7j(a)R378jx+}a2>z@ z{h@%fR|o#t7x&J#an*8G$KLaSn|8Ginbh_39%114XG>g-ZR7UZ>@)kbzkYJ}H6eSe}O!Ga`=5mL|I;9oet*`sH%h-xTS8Q=~ua> zqo7^A6^I1MWsa$PnMWXIIbSWMG-oBd6E*h~?F^^Ug7+QJVL{Q4j`!IbPoFLVP7Js6 z8Ccl{A_pi=5U+OQ8;WlX%+I;+VXnQL#pq z{!8C7J8)%0@93U9tkQH{W&;8~BKN`V~Qmp5uidIuB!wYW~R(dKm6RF=8nN0-fS|2jdCshM4ev@9!smX2w-i)I8f+O#H~{Z29uQookdY zj3Re0@8&H3pb81uc8{SXumwhWzdEi)9HpWN@KiUZ1Rd_ur3+(7@LP*}P0#~( zYI;H5Pqn&HvGdqoPtN&hd~rVZETeY*=t3jH#{S(6mmjy#$tUUw_=CqyuIrA%f$95X zb8nTsg>&;FRb5-^#>45t7oN0Okj)v(G4;;G&-(0G5EE}^$);lW#zb#aAM7wF0M7@| zowo0<=7N)DO;oU6H^wI_M zuMGaVdm`VnQR?B{3?-n$LrHk)r)xL)^XE7LZyM>t@c*vv{~;pXkU&46_BZtXjVSsH zJoBYJLJ6vx`+>GI)|yU>X445-z?Y?DCwuBfvM~mfumxS$o?T<}8Kh_`Qzu0EnUU*^ z+T_$>W#<7klZo7!`<*9A^aEhriu4vouz>uj508UTm_`>orqudo@8cpE4rwkOv*0V< z$<~+yh8oqI=CLU+uT1|gk0e$&2gDQt6m%#H=01Ha6}PwTZIN;!E3s?4?+N<>v( z!EMsN5^tCQkB<&obPlis3AHi^fQG`uQ;}MsYxSdy zjX=8?ek?tIdOtc;Z(eE!-6!$WjLo86giN)~bpz0ds|Q+w)x?CxfTDY<@F`(U&htxs zn?)^!TuCHrtVJ-qKP`t$U;-Rqq(yCgN^0n$RGs+!x__*@K2kMnF47TUlrM;rL&W4F zjO;xT?J!`In@qcQ&T{i*l%9Z>x zo{D%uKJvQF_MGne3`K?NQi+C^kGevVWX&1RTyOF2lh*9e1oC~(Qv+a1x&4s-7|i9h zW8U1Wt+~K9(>E$-rk=d2>jw8YCMTBH{-iahix}&lx^LmqLM&* zj?LojW_c=ZCnT<4)~8`XdchPU9*&F*mHR_Kdc`gU}$)CAS^U_sXHvaT10hYVJd6s}IQ{{Mk-vuzQe*S#QN%Rjr{e*W~ z0@eHuWxwwL>64YwFRM^t3;xK0!qadOUm)=TuHxU!vi~!z08r~D9r_I~f6tC)o}o2l zimeZ#xjsU0q!z8gp_0Y}FyjXeQrt%LX&?B)fq!E9c67GP4L|btBm9Y11+9u}mEqMF zz#0Vkam@126#>l}mZM$3ywp0XbUnL#c+&*GnarJ4V*O>k0vz}`=Tl*ikSpkzHJ`wZ z*2R)K8{A0&C3d`V&LlG5^Ho~M=q#fRIxrya9d)rcXGJCGNg?q1nG!KEn$P|8CP_I>4tj6Z4@Ke74>UxU?gSPw`}m4Z_&Oc# zulf$krzZn6kCt-PbFzJ7J97-msl;c#c+|69Q1l1>zjGgYkgR2!s@$r8o~a3;-vIRC zmMQsFTv>5%Z9seChe6BEUXKagZhBltEMkQ&t7VILC!Cs!P@GeyK@rTpGwXw1Ab)c_PMI6bXb55C>x@y3R*{^(>utjO zIRLN^tCh^Ro_z2_VHXBuQDm@}(wXC(t|z&Fc!SD?Nj}N~6f)CM$VhPTeY}rL!NGJ9 z<6sXhH|2Sk40I!38(SOnp8AYZJr$$5y+|n9pBdSoKB9D=3kCs6%O`SS2^2=2m7#GS z#|Y|R&5IeO#zW(TU=h)~{&v-BO1bDd`fq4|i7w3J_ zagv0gqgR~UUeQDbW}i9Fo0}4?FM05$MMK^YAS0)XX`TO7lC#3unZ5bHwV|b$H_LD4P`m7DJ)1UP~v_ zbz9~Mu!_Q8?PK%{XeNH?c?5Z!K}e0UkQcB8d!V6XhjU={*1*91K;V=VKy#Ns1?WU@ z+wtMNz@>YH3#bnf8SXtx(oW#3QBR-I(ocTu@LY-GvvKn?v2JsN2)zcKJESS2(XQUn zoNRHxM4Y;wh^8b13sW&oG1|wC=X}BU68pi#?bo`h>byOz7F)=%$F#CC@W5IZxGtm8 z?1IdPQ#kj=>t93?bZ68C_1^#V6ck}b7&S%AMCzcv!70Lo;98udcihrjL)=)Rnv?NH)*AJ8yKUF`B#g8+@O?9@8ub@In` z-9A-Wu-0X-IUwJdHfuj1KifLB(Kjb8jh(Se`4Y^2zAL3P9wMpHPY+oV@ywZFXxP%; z=;pj;N56Jc00yA9)re%aU6)l~;PLNVgzsCV#QJ0`0t?Jge^oOJZ@+znV8GVa$`QGu zSA&J)k%BSX`wZ76k8&MY-hnpO>SULZu8K$n;RpE4H3sqO=o|BkegPjXThGR0qX)LY z?H~_v^Y{F0;YaenvB@WqYR=4h>Ym&aHQ-fvOTekQOiy_oi%xwyJzX@xR^JZT-7JcdH{j`b zv)IXufcK&5mTMre2+5EwQ7RG6hFU()jrHy`+y5@b^uJ3n{qIst|GN~^|1QP!|I<=T zfDZ6~|L1sq1$QAZ>)2Wp?E|bLNbM}H;cQ~)Y${;vXbNmV9Bk}-%xwJ39Goib>;i1O v0&LujY-|E-YI*tr_m0VFN1@TBCCLBRh3B+C&T From 94ab4ea341c34ee692b707928a6da285fab85ea3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 16 Mar 2021 11:15:14 +0100 Subject: [PATCH 200/709] core/rawdb: fix transaction indexing/unindexing hashing error (#22457) * core/rawdb: more verbose error logs + better hashing * core/rawdb: add failing testcase * core/rawdb: properly hash transactions while indexing/unindexing * core/rawdb: exit on error + better log msg --- core/rawdb/chain_iterator.go | 29 +++--------- core/rawdb/chain_iterator_test.go | 73 ++++++++++++++++++++++++------- rlp/iterator.go | 1 + 3 files changed, 65 insertions(+), 38 deletions(-) diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 862a549540..ad222005be 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -23,10 +23,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" ) // InitDatabaseFromFreezer reinitializes an empty database from a previous batch @@ -135,32 +135,15 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool close(hashesCh) } }() - - var hasher = sha3.NewLegacyKeccak256() for data := range rlpCh { - it, err := rlp.NewListIterator(data.rlp) - if err != nil { - log.Warn("tx iteration error", "error", err) - return - } - it.Next() - txs := it.Value() - txIt, err := rlp.NewListIterator(txs) - if err != nil { - log.Warn("tx iteration error", "error", err) + var body types.Body + if err := rlp.DecodeBytes(data.rlp, &body); err != nil { + log.Warn("Failed to decode block body", "block", data.number, "error", err) return } var hashes []common.Hash - for txIt.Next() { - if err := txIt.Err(); err != nil { - log.Warn("tx iteration error", "error", err) - return - } - var txHash common.Hash - hasher.Reset() - hasher.Write(txIt.Value()) - hasher.Sum(txHash[:0]) - hashes = append(hashes, txHash) + for _, tx := range body.Transactions { + hashes = append(hashes, tx.Hash()) } result := &blockTxHashes{ hashes: hashes, diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 90b2639d38..45cc6323e0 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -33,14 +33,34 @@ func TestChainIterator(t *testing.T) { var block *types.Block var txs []*types.Transaction - for i := uint64(0); i <= 10; i++ { - if i == 0 { - block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, nil, nil, nil, newHasher()) // Empty genesis block + to := common.BytesToAddress([]byte{0x11}) + block = types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, nil, newHasher()) // Empty genesis block + WriteBlock(chainDb, block) + WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) + for i := uint64(1); i <= 10; i++ { + var tx *types.Transaction + if i%2 == 0 { + tx = types.NewTx(&types.LegacyTx{ + Nonce: i, + GasPrice: big.NewInt(11111), + Gas: 1111, + To: &to, + Value: big.NewInt(111), + Data: []byte{0x11, 0x11, 0x11}, + }) } else { - tx := types.NewTransaction(i, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - txs = append(txs, tx) - block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher()) + tx = types.NewTx(&types.AccessListTx{ + ChainID: big.NewInt(1337), + Nonce: i, + GasPrice: big.NewInt(11111), + Gas: 1111, + To: &to, + Value: big.NewInt(111), + Data: []byte{0x11, 0x11, 0x11}, + }) } + txs = append(txs, tx) + block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher()) WriteBlock(chainDb, block) WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) } @@ -66,7 +86,7 @@ func TestChainIterator(t *testing.T) { numbers = append(numbers, int(h.number)) if len(h.hashes) > 0 { if got, exp := h.hashes[0], txs[h.number-1].Hash(); got != exp { - t.Fatalf("hash wrong, got %x exp %x", got, exp) + t.Fatalf("block %d: hash wrong, got %x exp %x", h.number, got, exp) } } } @@ -88,14 +108,37 @@ func TestIndexTransactions(t *testing.T) { var block *types.Block var txs []*types.Transaction - for i := uint64(0); i <= 10; i++ { - if i == 0 { - block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, nil, nil, nil, newHasher()) // Empty genesis block + to := common.BytesToAddress([]byte{0x11}) + + // Write empty genesis block + block = types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, nil, newHasher()) + WriteBlock(chainDb, block) + WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) + + for i := uint64(1); i <= 10; i++ { + var tx *types.Transaction + if i%2 == 0 { + tx = types.NewTx(&types.LegacyTx{ + Nonce: i, + GasPrice: big.NewInt(11111), + Gas: 1111, + To: &to, + Value: big.NewInt(111), + Data: []byte{0x11, 0x11, 0x11}, + }) } else { - tx := types.NewTransaction(i, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - txs = append(txs, tx) - block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher()) + tx = types.NewTx(&types.AccessListTx{ + ChainID: big.NewInt(1337), + Nonce: i, + GasPrice: big.NewInt(11111), + Gas: 1111, + To: &to, + Value: big.NewInt(111), + Data: []byte{0x11, 0x11, 0x11}, + }) } + txs = append(txs, tx) + block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher()) WriteBlock(chainDb, block) WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) } @@ -108,10 +151,10 @@ func TestIndexTransactions(t *testing.T) { } number := ReadTxLookupEntry(chainDb, txs[i-1].Hash()) if exist && number == nil { - t.Fatalf("Transaction indice missing") + t.Fatalf("Transaction index %d missing", i) } if !exist && number != nil { - t.Fatalf("Transaction indice is not deleted") + t.Fatalf("Transaction index %d is not deleted", i) } } number := ReadTxIndexTail(chainDb) diff --git a/rlp/iterator.go b/rlp/iterator.go index c28866dbc1..559e03a868 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -23,6 +23,7 @@ type listIterator struct { } // NewListIterator creates an iterator for the (list) represented by data +// TODO: Consider removing this implementation, as it is no longer used. func NewListIterator(data RawValue) (*listIterator, error) { k, t, c, err := readKind(data) if err != nil { From 62d8022b513772666d92c3ba45312b61c403fea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 16 Mar 2021 12:53:54 +0100 Subject: [PATCH 201/709] les: fix UDP connection query (#22451) This PR fixes multiple issues with the UDP connection pre-negotiation feature: - the enable condition was wrong (it checked the existence of the DiscV5 struct where it wasn't initialized yet, disabling the feature even if discv5 was enabled) - the server pool queried already connected nodes when the discovery iterators returned them again - servers responded positively before they were synced and really willing to accept connections Metrics are also added on the server side that count the positive and negative replies to served connection queries. --- les/client.go | 18 ++++-- les/clientpool.go | 18 +++++- les/clientpool_test.go | 24 ++++---- les/enr_entry.go | 5 +- les/metrics.go | 10 ++-- les/server.go | 2 +- les/test_helper.go | 6 +- les/vflux/client/serverpool.go | 103 ++++++++++++++++++--------------- 8 files changed, 110 insertions(+), 76 deletions(-) diff --git a/les/client.go b/les/client.go index a557e7ccb3..99f79bb20c 100644 --- a/les/client.go +++ b/les/client.go @@ -73,8 +73,9 @@ type LightEthereum struct { accountManager *accounts.Manager netRPCService *ethapi.PublicNetAPI - p2pServer *p2p.Server - p2pConfig *p2p.Config + p2pServer *p2p.Server + p2pConfig *p2p.Config + udpEnabled bool } // New creates an instance of the light client. @@ -113,10 +114,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), p2pServer: stack.Server(), p2pConfig: &stack.Config().P2P, + udpEnabled: stack.Config().P2P.DiscoveryV5, } var prenegQuery vfc.QueryFunc - if leth.p2pServer.DiscV5 != nil { + if leth.udpEnabled { prenegQuery = leth.prenegQuery } leth.serverPool, leth.serverPoolIterator = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, prenegQuery, &mclock.System{}, config.UltraLightServers, requestList) @@ -198,7 +200,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { // VfluxRequest sends a batch of requests to the given node through discv5 UDP TalkRequest and returns the responses func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.Replies { - if s.p2pServer.DiscV5 == nil { + if !s.udpEnabled { return nil } reqsEnc, _ := rlp.EncodeToBytes(&reqs) @@ -215,7 +217,7 @@ func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.R func (s *LightEthereum) vfxVersion(n *enode.Node) uint { if n.Seq() == 0 { var err error - if s.p2pServer.DiscV5 == nil { + if !s.udpEnabled { return 0 } if n, err = s.p2pServer.DiscV5.RequestENR(n); n != nil && err == nil && n.Seq() != 0 { @@ -346,7 +348,11 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { func (s *LightEthereum) Start() error { log.Warn("Light client mode is an experimental feature") - discovery, err := s.setupDiscovery(s.p2pConfig) + if s.udpEnabled && s.p2pServer.DiscV5 == nil { + s.udpEnabled = false + log.Error("Discovery v5 is not initialized") + } + discovery, err := s.setupDiscovery() if err != nil { return err } diff --git a/les/clientpool.go b/les/clientpool.go index 1aa63a281e..3965d54508 100644 --- a/les/clientpool.go +++ b/les/clientpool.go @@ -72,6 +72,7 @@ type clientPool struct { clock mclock.Clock closed bool removePeer func(enode.ID) + synced func() bool ns *nodestate.NodeStateMachine pp *vfs.PriorityPool bt *vfs.BalanceTracker @@ -107,7 +108,7 @@ type clientInfo struct { } // newClientPool creates a new client pool -func newClientPool(ns *nodestate.NodeStateMachine, lesDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID)) *clientPool { +func newClientPool(ns *nodestate.NodeStateMachine, lesDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID), synced func() bool) *clientPool { pool := &clientPool{ ns: ns, BalanceTrackerSetup: balanceTrackerSetup, @@ -116,6 +117,7 @@ func newClientPool(ns *nodestate.NodeStateMachine, lesDb ethdb.Database, minCap minCap: minCap, connectedBias: connectedBias, removePeer: removePeer, + synced: synced, } pool.bt = vfs.NewBalanceTracker(ns, balanceTrackerSetup, lesDb, clock, &utils.Expirer{}, &utils.Expirer{}) pool.pp = vfs.NewPriorityPool(ns, priorityPoolSetup, clock, minCap, connectedBias, 4) @@ -396,6 +398,13 @@ func (f *clientPool) serveCapQuery(id enode.ID, freeID string, data []byte) []by if l := len(req.AddTokens); l == 0 || l > vflux.CapacityQueryMaxLen { return nil } + result := make(vflux.CapacityQueryReply, len(req.AddTokens)) + if !f.synced() { + capacityQueryZeroMeter.Mark(1) + reply, _ := rlp.EncodeToBytes(&result) + return reply + } + node := f.ns.GetNode(id) if node == nil { node = enode.SignNull(&enr.Record{}, id) @@ -416,7 +425,6 @@ func (f *clientPool) serveCapQuery(id enode.ID, freeID string, data []byte) []by } // use vfs.CapacityCurve to answer request for multiple newly bought token amounts curve := f.pp.GetCapacityCurve().Exclude(id) - result := make(vflux.CapacityQueryReply, len(req.AddTokens)) bias := time.Second * time.Duration(req.Bias) if f.connectedBias > bias { bias = f.connectedBias @@ -434,6 +442,12 @@ func (f *clientPool) serveCapQuery(id enode.ID, freeID string, data []byte) []by result[i] = 0 } } + // add first result to metrics (don't care about priority client multi-queries yet) + if result[0] == 0 { + capacityQueryZeroMeter.Mark(1) + } else { + capacityQueryNonZeroMeter.Mark(1) + } reply, _ := rlp.EncodeToBytes(&result) return reply } diff --git a/les/clientpool_test.go b/les/clientpool_test.go index 345b373b0f..2aee444545 100644 --- a/les/clientpool_test.go +++ b/les/clientpool_test.go @@ -133,7 +133,7 @@ func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, rando disconnFn = func(id enode.ID) { disconnCh <- int(id[0]) + int(id[1])<<8 } - pool = newClientPool(testStateMachine(), db, 1, 0, &clock, disconnFn) + pool = newClientPool(testStateMachine(), db, 1, 0, &clock, disconnFn, alwaysTrueFn) ) pool.ns.Start() @@ -239,7 +239,7 @@ func TestConnectPaidClient(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) @@ -255,7 +255,7 @@ func TestConnectPaidClientToSmallPool(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -274,7 +274,7 @@ func TestConnectPaidClientToFullPool(t *testing.T) { db = rawdb.NewMemoryDatabase() ) removeFn := func(enode.ID) {} // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -304,7 +304,7 @@ func TestPaidClientKickedOut(t *testing.T) { removeFn := func(id enode.ID) { kickedCh <- int(id[0]) } - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) pool.ns.Start() pool.bt.SetExpirationTCs(0, 0) defer pool.stop() @@ -335,7 +335,7 @@ func TestConnectFreeClient(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) @@ -352,7 +352,7 @@ func TestConnectFreeClientToFullPool(t *testing.T) { db = rawdb.NewMemoryDatabase() ) removeFn := func(enode.ID) {} // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -382,7 +382,7 @@ func TestFreeClientKickedOut(t *testing.T) { kicked = make(chan int, 100) ) removeFn := func(id enode.ID) { kicked <- int(id[0]) } - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -424,7 +424,7 @@ func TestPositiveBalanceCalculation(t *testing.T) { kicked = make(chan int, 10) ) removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -448,7 +448,7 @@ func TestDowngradePriorityClient(t *testing.T) { kicked = make(chan int, 10) ) removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -483,7 +483,7 @@ func TestNegativeBalanceCalculation(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(10, uint64(10)) // Total capacity limit is 10 @@ -521,7 +521,7 @@ func TestInactiveClient(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}) + pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) pool.ns.Start() defer pool.stop() pool.setLimits(2, uint64(2)) diff --git a/les/enr_entry.go b/les/enr_entry.go index 8be4a7a00e..892c3530d3 100644 --- a/les/enr_entry.go +++ b/les/enr_entry.go @@ -18,7 +18,6 @@ package les import ( "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" @@ -42,7 +41,7 @@ type ethEntry struct { func (ethEntry) ENRKey() string { return "eth" } // setupDiscovery creates the node discovery source for the eth protocol. -func (eth *LightEthereum) setupDiscovery(cfg *p2p.Config) (enode.Iterator, error) { +func (eth *LightEthereum) setupDiscovery() (enode.Iterator, error) { it := enode.NewFairMix(0) // Enable DNS discovery. @@ -56,7 +55,7 @@ func (eth *LightEthereum) setupDiscovery(cfg *p2p.Config) (enode.Iterator, error } // Enable DHT. - if cfg.DiscoveryV5 && eth.p2pServer.DiscV5 != nil { + if eth.udpEnabled { it.AddSource(eth.p2pServer.DiscV5.RandomNodes()) } diff --git a/les/metrics.go b/les/metrics.go index 9a79fd1bbd..5a8d4bbe02 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -73,10 +73,12 @@ var ( serverConnectionGauge = metrics.NewRegisteredGauge("les/connection/server", nil) clientConnectionGauge = metrics.NewRegisteredGauge("les/connection/client", nil) - totalCapacityGauge = metrics.NewRegisteredGauge("les/server/totalCapacity", nil) - totalRechargeGauge = metrics.NewRegisteredGauge("les/server/totalRecharge", nil) - totalConnectedGauge = metrics.NewRegisteredGauge("les/server/totalConnected", nil) - blockProcessingTimer = metrics.NewRegisteredTimer("les/server/blockProcessingTime", nil) + totalCapacityGauge = metrics.NewRegisteredGauge("les/server/totalCapacity", nil) + totalRechargeGauge = metrics.NewRegisteredGauge("les/server/totalRecharge", nil) + totalConnectedGauge = metrics.NewRegisteredGauge("les/server/totalConnected", nil) + blockProcessingTimer = metrics.NewRegisteredTimer("les/server/blockProcessingTime", nil) + capacityQueryZeroMeter = metrics.NewRegisteredMeter("les/server/capQueryZero", nil) + capacityQueryNonZeroMeter = metrics.NewRegisteredMeter("les/server/capQueryNonZero", nil) requestServedMeter = metrics.NewRegisteredMeter("les/server/req/avgServedTime", nil) requestServedTimer = metrics.NewRegisteredTimer("les/server/req/servedTime", nil) diff --git a/les/server.go b/les/server.go index 9627d65afa..0351bdd801 100644 --- a/les/server.go +++ b/les/server.go @@ -149,7 +149,7 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les srv.maxCapacity = totalRecharge } srv.fcManager.SetCapacityLimits(srv.minCapacity, srv.maxCapacity, srv.minCapacity*2) - srv.clientPool = newClientPool(ns, lesDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient) + srv.clientPool = newClientPool(ns, lesDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient, issync) srv.clientPool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}) checkpoint := srv.latestLocalCheckpoint() diff --git a/les/test_helper.go b/les/test_helper.go index e1d3beb6a1..e49bfc8738 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -307,7 +307,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da } server.costTracker, server.minCapacity = newCostTracker(db, server.config) server.costTracker.testCostList = testCostList(0) // Disable flow control mechanism. - server.clientPool = newClientPool(ns, db, testBufRecharge, defaultConnectedBias, clock, func(id enode.ID) {}) + server.clientPool = newClientPool(ns, db, testBufRecharge, defaultConnectedBias, clock, func(id enode.ID) {}, alwaysTrueFn) server.clientPool.setLimits(10000, 10000) // Assign enough capacity for clientpool server.handler = newServerHandler(server, simulation.Blockchain(), db, txpool, func() bool { return true }) if server.oracle != nil { @@ -319,6 +319,10 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da return server.handler, simulation } +func alwaysTrueFn() bool { + return true +} + // testPeer is a simulated peer to allow testing direct network calls. type testPeer struct { cpeer *clientPeer diff --git a/les/vflux/client/serverpool.go b/les/vflux/client/serverpool.go index e73b277ced..cc0254c127 100644 --- a/les/vflux/client/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -47,7 +47,8 @@ const ( nodeWeightThreshold = 100 // minimum weight for keeping a node in the the known (valuable) set minRedialWait = 10 // minimum redial wait time in seconds preNegLimit = 5 // maximum number of simultaneous pre-negotiation queries - maxQueryFails = 100 // number of consecutive UDP query failures before we print a warning + warnQueryFails = 20 // number of consecutive UDP query failures before we print a warning + maxQueryFails = 100 // number of consecutive UDP query failures when then chance of skipping a query reaches 50% ) // ServerPool provides a node iterator for dial candidates. The output is a mix of newly discovered @@ -94,16 +95,16 @@ type nodeHistoryEnc struct { type QueryFunc func(*enode.Node) int var ( - clientSetup = &nodestate.Setup{Version: 2} - sfHasValue = clientSetup.NewPersistentFlag("hasValue") - sfQueried = clientSetup.NewFlag("queried") - sfCanDial = clientSetup.NewFlag("canDial") - sfDialing = clientSetup.NewFlag("dialed") - sfWaitDialTimeout = clientSetup.NewFlag("dialTimeout") - sfConnected = clientSetup.NewFlag("connected") - sfRedialWait = clientSetup.NewFlag("redialWait") - sfAlwaysConnect = clientSetup.NewFlag("alwaysConnect") - sfDisableSelection = nodestate.MergeFlags(sfQueried, sfCanDial, sfDialing, sfConnected, sfRedialWait) + clientSetup = &nodestate.Setup{Version: 2} + sfHasValue = clientSetup.NewPersistentFlag("hasValue") + sfQuery = clientSetup.NewFlag("query") + sfCanDial = clientSetup.NewFlag("canDial") + sfDialing = clientSetup.NewFlag("dialed") + sfWaitDialTimeout = clientSetup.NewFlag("dialTimeout") + sfConnected = clientSetup.NewFlag("connected") + sfRedialWait = clientSetup.NewFlag("redialWait") + sfAlwaysConnect = clientSetup.NewFlag("alwaysConnect") + sfDialProcess = nodestate.MergeFlags(sfQuery, sfCanDial, sfDialing, sfConnected, sfRedialWait) sfiNodeHistory = clientSetup.NewPersistentField("nodeHistory", reflect.TypeOf(nodeHistory{}), func(field interface{}) ([]byte, error) { @@ -162,8 +163,8 @@ func NewServerPool(db ethdb.KeyValueStore, dbKey []byte, mixTimeout time.Duratio } s.recalTimeout() s.mixer = enode.NewFairMix(mixTimeout) - knownSelector := NewWrsIterator(s.ns, sfHasValue, sfDisableSelection, sfiNodeWeight) - alwaysConnect := NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) + knownSelector := NewWrsIterator(s.ns, sfHasValue, sfDialProcess, sfiNodeWeight) + alwaysConnect := NewQueueIterator(s.ns, sfAlwaysConnect, sfDialProcess, true, nil) s.mixSources = append(s.mixSources, knownSelector) s.mixSources = append(s.mixSources, alwaysConnect) @@ -226,7 +227,7 @@ func (s *ServerPool) AddMetrics( s.totalValueGauge = totalValueGauge s.sessionValueMeter = sessionValueMeter if serverSelectableGauge != nil { - s.ns.AddLogMetrics(sfHasValue, sfDisableSelection, "selectable", nil, nil, serverSelectableGauge) + s.ns.AddLogMetrics(sfHasValue, sfDialProcess, "selectable", nil, nil, serverSelectableGauge) } if serverDialedMeter != nil { s.ns.AddLogMetrics(sfDialing, nodestate.Flags{}, "dialed", serverDialedMeter, nil, nil) @@ -247,43 +248,51 @@ func (s *ServerPool) AddSource(source enode.Iterator) { // Nodes that are filtered out and does not appear on the output iterator are put back // into redialWait state. func (s *ServerPool) addPreNegFilter(input enode.Iterator, query QueryFunc) enode.Iterator { - s.fillSet = NewFillSet(s.ns, input, sfQueried) - s.ns.SubscribeState(sfQueried, func(n *enode.Node, oldState, newState nodestate.Flags) { - if newState.Equals(sfQueried) { - fails := atomic.LoadUint32(&s.queryFails) - if fails == maxQueryFails { - log.Warn("UDP pre-negotiation query does not seem to work") + s.fillSet = NewFillSet(s.ns, input, sfQuery) + s.ns.SubscribeState(sfDialProcess, func(n *enode.Node, oldState, newState nodestate.Flags) { + if !newState.Equals(sfQuery) { + if newState.HasAll(sfQuery) { + // remove query flag if the node is already somewhere in the dial process + s.ns.SetStateSub(n, nodestate.Flags{}, sfQuery, 0) } - if fails > maxQueryFails { - fails = maxQueryFails - } - if rand.Intn(maxQueryFails*2) < int(fails) { - // skip pre-negotiation with increasing chance, max 50% - // this ensures that the client can operate even if UDP is not working at all - s.ns.SetStateSub(n, sfCanDial, nodestate.Flags{}, time.Second*10) - // set canDial before resetting queried so that FillSet will not read more - // candidates unnecessarily - s.ns.SetStateSub(n, nodestate.Flags{}, sfQueried, 0) - return + return + } + fails := atomic.LoadUint32(&s.queryFails) + failMax := fails + if failMax > maxQueryFails { + failMax = maxQueryFails + } + if rand.Intn(maxQueryFails*2) < int(failMax) { + // skip pre-negotiation with increasing chance, max 50% + // this ensures that the client can operate even if UDP is not working at all + s.ns.SetStateSub(n, sfCanDial, nodestate.Flags{}, time.Second*10) + // set canDial before resetting queried so that FillSet will not read more + // candidates unnecessarily + s.ns.SetStateSub(n, nodestate.Flags{}, sfQuery, 0) + return + } + go func() { + q := query(n) + if q == -1 { + atomic.AddUint32(&s.queryFails, 1) + fails++ + if fails%warnQueryFails == 0 { + // warn if a large number of consecutive queries have failed + log.Warn("UDP connection queries failed", "count", fails) + } + } else { + atomic.StoreUint32(&s.queryFails, 0) } - go func() { - q := query(n) - if q == -1 { - atomic.AddUint32(&s.queryFails, 1) + s.ns.Operation(func() { + // we are no longer running in the operation that the callback belongs to, start a new one because of setRedialWait + if q == 1 { + s.ns.SetStateSub(n, sfCanDial, nodestate.Flags{}, time.Second*10) } else { - atomic.StoreUint32(&s.queryFails, 0) + s.setRedialWait(n, queryCost, queryWaitStep) } - s.ns.Operation(func() { - // we are no longer running in the operation that the callback belongs to, start a new one because of setRedialWait - if q == 1 { - s.ns.SetStateSub(n, sfCanDial, nodestate.Flags{}, time.Second*10) - } else { - s.setRedialWait(n, queryCost, queryWaitStep) - } - s.ns.SetStateSub(n, nodestate.Flags{}, sfQueried, 0) - }) - }() - } + s.ns.SetStateSub(n, nodestate.Flags{}, sfQuery, 0) + }) + }() }) return NewQueueIterator(s.ns, sfCanDial, nodestate.Flags{}, false, func(waiting bool) { if waiting { From 6d9707a458ad0d82e7f53722219656c432c9bb92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 16 Mar 2021 12:54:45 +0100 Subject: [PATCH 202/709] les: fix UDP connection query (#22451) This PR fixes multiple issues with the UDP connection pre-negotiation feature: - the enable condition was wrong (it checked the existence of the DiscV5 struct where it wasn't initialized yet, disabling the feature even if discv5 was enabled) - the server pool queried already connected nodes when the discovery iterators returned them again - servers responded positively before they were synced and really willing to accept connections Metrics are also added on the server side that count the positive and negative replies to served connection queries. From 91726e8aadeebc4bda8f136aebf4b5c7da8b9d74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 16 Mar 2021 12:55:43 +0100 Subject: [PATCH 203/709] les: allow either full enode strings or raw hex ids in the API (#22423) --- les/api.go | 51 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/les/api.go b/les/api.go index 6491c4dcc4..a930524516 100644 --- a/les/api.go +++ b/les/api.go @@ -49,6 +49,18 @@ func NewPrivateLightServerAPI(server *LesServer) *PrivateLightServerAPI { } } +// parseNode parses either an enode address a raw hex node id +func parseNode(node string) (enode.ID, error) { + if id, err := enode.ParseID(node); err == nil { + return id, nil + } + if node, err := enode.Parse(enode.ValidSchemes, node); err == nil { + return node.ID(), nil + } else { + return enode.ID{}, err + } +} + // ServerInfo returns global server parameters func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { res := make(map[string]interface{}) @@ -59,7 +71,14 @@ func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { } // ClientInfo returns information about clients listed in the ids list or matching the given tags -func (api *PrivateLightServerAPI) ClientInfo(ids []enode.ID) map[enode.ID]map[string]interface{} { +func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[string]interface{} { + var ids []enode.ID + for _, node := range nodes { + if id, err := parseNode(node); err == nil { + ids = append(ids, id) + } + } + res := make(map[enode.ID]map[string]interface{}) api.server.clientPool.forClients(ids, func(client *clientInfo) { res[client.node.ID()] = api.clientInfo(client) @@ -159,8 +178,18 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien // SetClientParams sets client parameters for all clients listed in the ids list // or all connected clients if the list is empty -func (api *PrivateLightServerAPI) SetClientParams(ids []enode.ID, params map[string]interface{}) error { - var err error +func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[string]interface{}) error { + var ( + ids []enode.ID + err error + ) + for _, node := range nodes { + if id, err := parseNode(node); err != nil { + return err + } else { + ids = append(ids, id) + } + } api.server.clientPool.forClients(ids, func(client *clientInfo) { if client.connected { posFactors, negFactors := client.balance.GetPriceFactors() @@ -201,7 +230,11 @@ func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error { // AddBalance adds the given amount to the balance of a client if possible and returns // the balance before and after the operation -func (api *PrivateLightServerAPI) AddBalance(id enode.ID, amount int64) (balance [2]uint64, err error) { +func (api *PrivateLightServerAPI) AddBalance(node string, amount int64) (balance [2]uint64, err error) { + var id enode.ID + if id, err = parseNode(node); err != nil { + return + } api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo) { balance[0], balance[1], err = c.balance.AddBalance(amount) }) @@ -297,8 +330,14 @@ func NewPrivateDebugAPI(server *LesServer) *PrivateDebugAPI { } // FreezeClient forces a temporary client freeze which normally happens when the server is overloaded -func (api *PrivateDebugAPI) FreezeClient(id enode.ID) error { - var err error +func (api *PrivateDebugAPI) FreezeClient(node string) error { + var ( + id enode.ID + err error + ) + if id, err = parseNode(node); err != nil { + return err + } api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo) { if c.connected { c.peer.freeze() From 410089afea1907d4abbf1dc4d1b42bda1094a5e5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 17 Mar 2021 09:36:34 +0100 Subject: [PATCH 204/709] eth/protocols/snap, eth/downloader: don't use bloom filter in snap sync --- eth/downloader/downloader.go | 2 +- eth/handler.go | 6 +++++- eth/protocols/snap/sync.go | 13 ++++--------- eth/protocols/snap/sync_test.go | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 5ddd2f9848..a5ed3761b1 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -240,7 +240,7 @@ func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, headerProcCh: make(chan []*types.Header, 1), quitCh: make(chan struct{}), stateCh: make(chan dataPack), - SnapSyncer: snap.NewSyncer(stateDb, stateBloom), + SnapSyncer: snap.NewSyncer(stateDb), stateSyncStart: make(chan *stateSync), syncStatsState: stateSyncStats{ processed: rawdb.ReadFastTrieProgress(stateDb), diff --git a/eth/handler.go b/eth/handler.go index 13fa701935..11c8565de1 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -177,7 +177,11 @@ func newHandler(config *handlerConfig) (*handler, error) { // Construct the downloader (long sync) and its backing state bloom if fast // sync is requested. The downloader is responsible for deallocating the state // bloom when it's done. - if atomic.LoadUint32(&h.fastSync) == 1 { + // Note: we don't enable it if snap-sync is performed, since it's very heavy + // and the heal-portion of the snap sync is much lighter than fast. What we particularly + // want to avoid, is a 90%-finished (but restarted) snap-sync to begin + // indexing the entire trie + if atomic.LoadUint32(&h.fastSync) == 1 && atomic.LoadUint32(&h.snapSync) == 0 { h.stateBloom = trie.NewSyncBloom(config.BloomCache, config.Database) } h.downloader = downloader.New(h.checkpointNumber, config.Database, h.stateBloom, h.eventMux, h.chain, nil, h.removePeer) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 1cfdef15bd..0303e65eda 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -376,8 +376,7 @@ type SyncPeer interface { // - The peer delivers a stale response after a previous timeout // - The peer delivers a refusal to serve the requested state type Syncer struct { - db ethdb.KeyValueStore // Database to store the trie nodes into (and dedup) - bloom *trie.SyncBloom // Bloom filter to deduplicate nodes for state fixup + db ethdb.KeyValueStore // Database to store the trie nodes into (and dedup) root common.Hash // Current state trie root being synced tasks []*accountTask // Current account task set being synced @@ -446,10 +445,9 @@ type Syncer struct { // NewSyncer creates a new snapshot syncer to download the Ethereum state over the // snap protocol. -func NewSyncer(db ethdb.KeyValueStore, bloom *trie.SyncBloom) *Syncer { +func NewSyncer(db ethdb.KeyValueStore) *Syncer { return &Syncer{ - db: db, - bloom: bloom, + db: db, peers: make(map[string]SyncPeer), peerJoin: new(event.Feed), @@ -546,7 +544,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { s.lock.Lock() s.root = root s.healer = &healTask{ - scheduler: state.NewStateSync(root, s.db, s.bloom), + scheduler: state.NewStateSync(root, s.db, nil), trieTasks: make(map[common.Hash]trie.SyncPath), codeTasks: make(map[common.Hash]struct{}), } @@ -1660,7 +1658,6 @@ func (s *Syncer) processBytecodeResponse(res *bytecodeResponse) { bytes += common.StorageSize(len(code)) rawdb.WriteCode(batch, hash, code) - s.bloom.Add(hash[:]) } if err := batch.Write(); err != nil { log.Crit("Failed to persist bytecodes", "err", err) @@ -1796,7 +1793,6 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { } // Node is not a boundary, persist to disk batch.Put(it.Key(), it.Value()) - s.bloom.Add(it.Key()) bytes += common.StorageSize(common.HashLength + len(it.Value())) nodes++ @@ -1953,7 +1949,6 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } // Node is neither a boundary, not an incomplete account, persist to disk batch.Put(it.Key(), it.Value()) - s.bloom.Add(it.Key()) bytes += common.StorageSize(common.HashLength + len(it.Value())) nodes++ diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 0b048786e8..49dff7bb3a 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -525,7 +525,7 @@ func TestSyncBloatedProof(t *testing.T) { func setupSyncer(peers ...*testPeer) *Syncer { stateDb := rawdb.NewMemoryDatabase() - syncer := NewSyncer(stateDb, trie.NewSyncBloom(1, stateDb)) + syncer := NewSyncer(stateDb) for _, peer := range peers { syncer.Register(peer) peer.remote = syncer From 117fa7d4a1ac0211b6a59aac5f3c5f7e5890e053 Mon Sep 17 00:00:00 2001 From: wuff1996 <33193253+wuff1996@users.noreply.github.com> Date: Fri, 19 Mar 2021 15:49:24 +0800 Subject: [PATCH 205/709] eth/protocols/snap: fix typo (#22530) --- eth/protocols/snap/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 24c8599552..b9515b8a39 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -115,7 +115,7 @@ func handle(backend Backend, peer *Peer) error { } // handleMessage is invoked whenever an inbound message is received from a -// remote peer on the `spap` protocol. The remote connection is torn down upon +// remote peer on the `snap` protocol. The remote connection is torn down upon // returning any error. func handleMessage(backend Backend, peer *Peer) error { // Read the next message from the remote peer, and ensure it's fully consumed From 6a528fce3357176e2da2fe07b379842f596bb566 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 19 Mar 2021 09:57:23 +0000 Subject: [PATCH 206/709] cmd/devp2p/internal/ethtest: return request ID in BlockHeaders response (#22508) This PR fixes an issue with the eth66 test suite where, during a readAndServe when the test is manually responding to GetBlockHeader requests, it now responds with a BlockHeaders eth66 packet that includes the inbound request ID. --- cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go index b7fa1dce26..a3f1aaa4f8 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -141,8 +141,11 @@ func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Mess if err != nil { return 0, errorf("could not get headers for inbound header request: %v", err) } - - if err := c.Write(headers); err != nil { + resp := ð.BlockHeadersPacket66{ + RequestId: reqID, + BlockHeadersPacket: eth.BlockHeadersPacket(headers), + } + if err := c.write66(resp, BlockHeaders{}.Code()); err != nil { return 0, errorf("could not write to connection: %v", err) } default: From aa8b2189c6ad5148377076b3fab7ca5e0d8c0b62 Mon Sep 17 00:00:00 2001 From: meowsbits Date: Fri, 19 Mar 2021 05:14:23 -0500 Subject: [PATCH 207/709] ethclient: fix error handling for header test (#22514) The wantErr field was disused, and the error returned by HeaderByNumber was not properly tested. This simplifies the error checking using errors.Is and asserts that getting an expected missing header returns ethereum.NotFound. Also adds a nil check condition for header.Number before using big.Int's Sign method. --- ethclient/ethclient_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 9a5a45e34f..9fa5bf87a4 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -288,8 +288,9 @@ func testHeader(t *testing.T, chain []*types.Block, client *rpc.Client) { want: chain[1].Header(), }, "future_block": { - block: big.NewInt(1000000000), - want: nil, + block: big.NewInt(1000000000), + want: nil, + wantErr: ethereum.NotFound, }, } for name, tt := range tests { @@ -299,10 +300,10 @@ func testHeader(t *testing.T, chain []*types.Block, client *rpc.Client) { defer cancel() got, err := ec.HeaderByNumber(ctx, tt.block) - if tt.wantErr != nil && (err == nil || err.Error() != tt.wantErr.Error()) { + if !errors.Is(err, tt.wantErr) { t.Fatalf("HeaderByNumber(%v) error = %q, want %q", tt.block, err, tt.wantErr) } - if got != nil && got.Number.Sign() == 0 { + if got != nil && got.Number != nil && got.Number.Sign() == 0 { got.Number = big.NewInt(0) // hack to make DeepEqual work } if !reflect.DeepEqual(got, tt.want) { From 38ea7f2cf460fb12a5769f8c0767726334111b99 Mon Sep 17 00:00:00 2001 From: Martin Redmond <21436+reds@users.noreply.github.com> Date: Fri, 19 Mar 2021 06:56:10 -0400 Subject: [PATCH 208/709] accounts/abi/bind: add NoSend transact option (#22446) This adds a new option to avoid sending the transaction which is created by calling a bound contract method. --- accounts/abi/bind/base.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index f5a6fe22fc..55aca31a1a 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -54,6 +54,8 @@ type TransactOpts struct { GasLimit uint64 // Gas limit to set for the transaction execution (0 = estimate) Context context.Context // Network context to support cancellation and timeouts (nil = no timeout) + + NoSend bool // Do all transact steps but do not send the transaction } // FilterOpts is the collection of options to fine tune filtering for events @@ -260,6 +262,9 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i if err != nil { return nil, err } + if opts.NoSend { + return signedTx, nil + } if err := c.transactor.SendTransaction(ensureContext(opts.Context), signedTx); err != nil { return nil, err } From a90861ae0c2a5d294fb4abb46eac65181d560264 Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 19 Mar 2021 18:58:12 +0800 Subject: [PATCH 209/709] go.mod: upgrade goleveldb to commit 64b5b1c (#22436) This pulls in a fix for a corruption issue when the process crashes while a new manifest file is being added. --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 96a217ed06..48259118ac 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/shirou/gopsutil v2.20.5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 - github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca + github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c diff --git a/go.sum b/go.sum index af76759c51..7dac5e874e 100644 --- a/go.sum +++ b/go.sum @@ -431,8 +431,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= From 345890a558257a76acb92a3ae88fbfb0dfea4520 Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 19 Mar 2021 19:03:33 +0800 Subject: [PATCH 210/709] go.mod: upgrade goupnp to commit 0ca76305 (#22479) This pulls in a fix to skip the broadcast on interfaces which are down. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 48259118ac..3addd54005 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 - github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 + github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 github.com/influxdata/influxdb v1.8.3 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e diff --git a/go.sum b/go.sum index 7dac5e874e..82990d1a9e 100644 --- a/go.sum +++ b/go.sum @@ -243,8 +243,8 @@ github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 h1:HarGZ5h9HD9LgEg1yRVMXyfiw4wlXiLiYM2oMjeA/SE= -github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= +github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 h1:bcAj8KroPf552TScjFPIakjH2/tdIrIH8F+cc4v4SRo= +github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= From d50e9d24be6ae410af7b5975f453456367a7b28c Mon Sep 17 00:00:00 2001 From: jacksoom Date: Fri, 19 Mar 2021 19:04:15 +0800 Subject: [PATCH 211/709] consensus/ethash: remove unnecessary variable definition (#22512) --- consensus/ethash/ethash.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 550d99893d..1afdc9381a 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -537,7 +537,6 @@ func NewShared() *Ethash { // Close closes the exit channel to notify all backend threads exiting. func (ethash *Ethash) Close() error { - var err error ethash.closeOnce.Do(func() { // Short circuit if the exit channel is not allocated. if ethash.remote == nil { @@ -546,7 +545,7 @@ func (ethash *Ethash) Close() error { close(ethash.remote.requestExit) <-ethash.remote.exitCh }) - return err + return nil } // cache tries to retrieve a verification cache for the specified block number From e3a3f7cd6472f692f4fad97a302d09fb60f53430 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Fri, 19 Mar 2021 06:15:57 -0600 Subject: [PATCH 212/709] cmd/devp2p: use AWS-SDK v2 (#22360) This updates the DNS deployer to use AWS SDK v2. Migration is relatively seamless, although there were two locations that required a slightly different approach to achieve the same results. In particular, waiting for DNS change propagation is very different with SDK v2. This change also optimizes DNS updates by publishing all changes before waiting for propagation. --- cmd/devp2p/dns_route53.go | 136 +++++++++++++++++++++------------ cmd/devp2p/dns_route53_test.go | 72 ++++++++--------- go.mod | 7 +- go.sum | 25 ++++++ 4 files changed, 153 insertions(+), 87 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index c5f99529b8..5f534ff9f7 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -17,16 +17,19 @@ package main import ( + "context" "errors" "fmt" "sort" "strconv" "strings" + "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/route53" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/route53" + "github.com/aws/aws-sdk-go-v2/service/route53/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "gopkg.in/urfave/cli.v1" @@ -38,6 +41,7 @@ const ( // https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets route53ChangeSizeLimit = 32000 route53ChangeCountLimit = 1000 + maxRetryLimit = 60 ) var ( @@ -58,7 +62,7 @@ var ( ) type route53Client struct { - api *route53.Route53 + api *route53.Client zoneID string } @@ -74,13 +78,13 @@ func newRoute53Client(ctx *cli.Context) *route53Client { if akey == "" || asec == "" { exit(fmt.Errorf("need Route53 Access Key ID and secret proceed")) } - config := &aws.Config{Credentials: credentials.NewStaticCredentials(akey, asec, "")} - session, err := session.NewSession(config) + creds := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(akey, asec, "")) + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithCredentialsProvider(creds)) if err != nil { - exit(fmt.Errorf("can't create AWS session: %v", err)) + exit(fmt.Errorf("can't initialize AWS configuration: %v", err)) } return &route53Client{ - api: route53.New(session), + api: route53.NewFromConfig(cfg), zoneID: ctx.String(route53ZoneIDFlag.Name), } } @@ -105,25 +109,43 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { return nil } - // Submit change batches. + // Submit all change batches. batches := splitChanges(changes, route53ChangeSizeLimit, route53ChangeCountLimit) + changesToCheck := make([]*route53.ChangeResourceRecordSetsOutput, len(batches)) for i, changes := range batches { log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes))) - batch := new(route53.ChangeBatch) - batch.SetChanges(changes) - batch.SetComment(fmt.Sprintf("enrtree update %d/%d of %s at seq %d", i+1, len(batches), name, t.Seq())) + batch := &types.ChangeBatch{ + Changes: changes, + Comment: aws.String(fmt.Sprintf("enrtree update %d/%d of %s at seq %d", i+1, len(batches), name, t.Seq())), + } req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch} - resp, err := c.api.ChangeResourceRecordSets(req) + changesToCheck[i], err = c.api.ChangeResourceRecordSets(context.TODO(), req) if err != nil { return err } + } - log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id)) - wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id} - if err := c.api.WaitUntilResourceRecordSetsChanged(wreq); err != nil { - return err + // wait for all change batches to propagate + for _, change := range changesToCheck { + log.Info(fmt.Sprintf("Waiting for change request %s", *change.ChangeInfo.Id)) + wreq := &route53.GetChangeInput{Id: change.ChangeInfo.Id} + var count int + for { + wresp, err := c.api.GetChange(context.TODO(), wreq) + if err != nil { + return err + } + + count++ + + if wresp.ChangeInfo.Status == types.ChangeStatusInsync || count >= maxRetryLimit { + break + } + + time.Sleep(30 * time.Second) } } + return nil } @@ -140,7 +162,7 @@ func (c *route53Client) findZoneID(name string) (string, error) { log.Info(fmt.Sprintf("Finding Route53 Zone ID for %s", name)) var req route53.ListHostedZonesByNameInput for { - resp, err := c.api.ListHostedZonesByName(&req) + resp, err := c.api.ListHostedZonesByName(context.TODO(), &req) if err != nil { return "", err } @@ -149,7 +171,7 @@ func (c *route53Client) findZoneID(name string) (string, error) { return *zone.Id, nil } } - if !*resp.IsTruncated { + if !resp.IsTruncated { break } req.DNSName = resp.NextDNSName @@ -159,7 +181,7 @@ func (c *route53Client) findZoneID(name string) (string, error) { } // computeChanges creates DNS changes for the given record. -func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []*route53.Change { +func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []types.Change { // Convert all names to lowercase. lrecords := make(map[string]string, len(records)) for name, r := range records { @@ -167,7 +189,7 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e } records = lrecords - var changes []*route53.Change + var changes []types.Change for path, val := range records { ttl := int64(rootTTL) if path != name { @@ -204,21 +226,21 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e } // sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order. -func sortChanges(changes []*route53.Change) { +func sortChanges(changes []types.Change) { score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3} sort.Slice(changes, func(i, j int) bool { - if *changes[i].Action == *changes[j].Action { + if changes[i].Action == changes[j].Action { return *changes[i].ResourceRecordSet.Name < *changes[j].ResourceRecordSet.Name } - return score[*changes[i].Action] < score[*changes[j].Action] + return score[string(changes[i].Action)] < score[string(changes[j].Action)] }) } // splitChanges splits up DNS changes such that each change batch // is smaller than the given RDATA limit. -func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*route53.Change { +func splitChanges(changes []types.Change, sizeLimit, countLimit int) [][]types.Change { var ( - batches [][]*route53.Change + batches [][]types.Change batchSize int batchCount int ) @@ -241,7 +263,7 @@ func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*rou } // changeSize returns the RDATA size of a DNS change. -func changeSize(ch *route53.Change) int { +func changeSize(ch types.Change) int { size := 0 for _, rr := range ch.ResourceRecordSet.ResourceRecords { if rr.Value != nil { @@ -251,8 +273,8 @@ func changeSize(ch *route53.Change) int { return size } -func changeCount(ch *route53.Change) int { - if *ch.Action == "UPSERT" { +func changeCount(ch types.Change) int { + if ch.Action == types.ChangeActionUpsert { return 2 } return 1 @@ -262,13 +284,19 @@ func changeCount(ch *route53.Change) int { func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) { log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID)) var req route53.ListResourceRecordSetsInput - req.SetHostedZoneId(c.zoneID) + req.HostedZoneId = &c.zoneID existing := make(map[string]recordSet) - err := c.api.ListResourceRecordSetsPages(&req, func(resp *route53.ListResourceRecordSetsOutput, last bool) bool { + for { + resp, err := c.api.ListResourceRecordSets(context.TODO(), &req) + if err != nil { + return existing, err + } + for _, set := range resp.ResourceRecordSets { - if !isSubdomain(*set.Name, name) || *set.Type != "TXT" { + if !isSubdomain(*set.Name, name) || set.Type != types.RRTypeTxt { continue } + s := recordSet{ttl: *set.TTL} for _, rec := range set.ResourceRecords { s.values = append(s.values, *rec.Value) @@ -276,28 +304,38 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error name := strings.TrimSuffix(*set.Name, ".") existing[name] = s } - return true - }) - return existing, err + + if !resp.IsTruncated { + break + } + + // sets the cursor to the next batch + req.StartRecordIdentifier = resp.NextRecordIdentifier + } + + return existing, nil } // newTXTChange creates a change to a TXT record. -func newTXTChange(action, name string, ttl int64, values ...string) *route53.Change { - var c route53.Change - var r route53.ResourceRecordSet - var rrs []*route53.ResourceRecord +func newTXTChange(action, name string, ttl int64, values ...string) types.Change { + r := types.ResourceRecordSet{ + Type: types.RRTypeTxt, + Name: &name, + TTL: &ttl, + } + var rrs []types.ResourceRecord for _, val := range values { - rr := new(route53.ResourceRecord) - rr.SetValue(val) + var rr types.ResourceRecord + rr.Value = aws.String(val) rrs = append(rrs, rr) } - r.SetType("TXT") - r.SetName(name) - r.SetTTL(ttl) - r.SetResourceRecords(rrs) - c.SetAction(action) - c.SetResourceRecordSet(&r) - return &c + + r.ResourceRecords = rrs + + return types.Change{ + Action: types.ChangeAction(action), + ResourceRecordSet: &r, + } } // isSubdomain returns true if name is a subdomain of domain. diff --git a/cmd/devp2p/dns_route53_test.go b/cmd/devp2p/dns_route53_test.go index a2ef3791f6..600c281a28 100644 --- a/cmd/devp2p/dns_route53_test.go +++ b/cmd/devp2p/dns_route53_test.go @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/aws/aws-sdk-go/service/route53" + "github.com/aws/aws-sdk-go-v2/service/route53/types" ) // This test checks that computeChanges/splitChanges create DNS changes in @@ -43,93 +43,93 @@ func TestRoute53ChangeSort(t *testing.T) { "MHTDO6TMUBRIA2XWG5LUDACK24.n": "enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o", } - wantChanges := []*route53.Change{ + wantChanges := []types.Change{ { - Action: sp("CREATE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "CREATE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("2xs2367yhaxjfglzhvawlqd4zy.n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA"`), }}, TTL: ip(treeNodeTTL), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("CREATE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "CREATE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("c7hrfpf3blgf3yr4dy5kx3smbe.n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org"`), }}, TTL: ip(treeNodeTTL), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("CREATE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "CREATE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("h4fht4b454p6uxfd7jcyq5pwdy.n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enr:-HW4QAggRauloj2SDLtIHN1XBkvhFZ1vtf1raYQp9TBW2RD5EEawDzbtSmlXUfnaHcvwOizhVYLtr7e6vw7NAf6mTuoCgmlkgnY0iXNlY3AyNTZrMaECjrXI8TLNXU0f8cthpAMxEshUyQlK-AM0PW2wfrnacNI"`), }}, TTL: ip(treeNodeTTL), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("CREATE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "CREATE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("jwxydbpxywg6fx3gmdibfa6cj4.n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enrtree-branch:2XS2367YHAXJFGLZHVAWLQD4ZY,H4FHT4B454P6UXFD7JCYQ5PWDY,MHTDO6TMUBRIA2XWG5LUDACK24"`), }}, TTL: ip(treeNodeTTL), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("CREATE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "CREATE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("mhtdo6tmubria2xwg5ludack24.n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o"`), }}, TTL: ip(treeNodeTTL), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("UPSERT"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "UPSERT", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA"`), }}, TTL: ip(rootTTL), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("DELETE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "DELETE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("2kfjogvxdqtxxugbh7gs7naaai.n"), - ResourceRecords: []*route53.ResourceRecord{ + ResourceRecords: []types.ResourceRecord{ {Value: sp(`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-""vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`)}, }, TTL: ip(3333), - Type: sp("TXT"), + Type: "TXT", }, }, { - Action: sp("DELETE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: "DELETE", + ResourceRecordSet: &types.ResourceRecordSet{ Name: sp("fdxn3sn67na5dka4j2gok7bvqi.n"), - ResourceRecords: []*route53.ResourceRecord{{ + ResourceRecords: []types.ResourceRecord{{ Value: sp(`"enrtree-branch:"`), }}, TTL: ip(treeNodeTTL), - Type: sp("TXT"), + Type: "TXT", }, }, } @@ -141,7 +141,7 @@ func TestRoute53ChangeSort(t *testing.T) { } // Check splitting according to size. - wantSplit := [][]*route53.Change{ + wantSplit := [][]types.Change{ wantChanges[:4], wantChanges[4:6], wantChanges[6:], @@ -152,7 +152,7 @@ func TestRoute53ChangeSort(t *testing.T) { } // Check splitting according to count. - wantSplit = [][]*route53.Change{ + wantSplit = [][]types.Change{ wantChanges[:5], wantChanges[5:], } diff --git a/go.mod b/go.mod index 3addd54005..dc020f9a65 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,14 @@ module github.com/ethereum/go-ethereum -go 1.13 +go 1.15 require ( github.com/Azure/azure-storage-blob-go v0.7.0 github.com/VictoriaMetrics/fastcache v1.5.7 - github.com/aws/aws-sdk-go v1.25.48 + github.com/aws/aws-sdk-go-v2 v1.2.0 + github.com/aws/aws-sdk-go-v2/config v1.1.1 + github.com/aws/aws-sdk-go-v2/credentials v1.1.1 + github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1 github.com/btcsuite/btcd v0.20.1-beta github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 diff --git a/go.sum b/go.sum index 82990d1a9e..813aa8e27d 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,24 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.2.0 h1:BS+UYpbsElC82gB+2E2jiCBg36i8HlubTB/dO/moQ9c= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1 h1:ZAoq32boMzcaTW9bcUacBswAmHTbvlvDJICgHFZuECo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1 h1:NbvWIM1Mx6sNPTxowHgS2ewXCRp+NGTzUYb/96FZJbY= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 h1:EtEU7WRaWliitZh2nmuxEXrN0Cb8EgPUFGIoTMeqbzI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 h1:4AH9fFjUlVktQMznF+YN33aWNXaR4VgDXyP28qokJC0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1 h1:cKr6St+CtC3/dl/rEBJvlk7A/IN5D5F02GNkGzfbtVU= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 h1:37QubsarExl5ZuCBlnRP+7l1tNwZPBSTqpTBrPH98RU= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 h1:TJoIfnIFubCX0ACVeJ0w46HEH5MwjwYN4iFhuYIhfIY= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0 h1:D6CSsM3gdxaGaqXnPgOBCeL6Mophqzu7KJOu7zW78sU= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -193,6 +211,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -265,6 +286,10 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= From 5bc0343ed3de7dd491edf42f9714ad6b93930408 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 19 Mar 2021 13:20:27 +0100 Subject: [PATCH 213/709] p2p/dnsdisc: fix flaw in dns size calculation (#22533) This fixes the calculation of the tree branch factor. With the new formula, we now creat at most 13 children instead of 30, ensuring the TXT record size will be below 370 bytes. --- p2p/dnsdisc/tree.go | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 82a935ca41..410ec3b854 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -113,10 +113,41 @@ func (t *Tree) Nodes() []*enode.Node { return nodes } +/* +We want to keep the UDP size below 512 bytes. The UDP size is roughly: +UDP length = 8 + UDP payload length ( 229 ) +UPD Payload length: + - dns.id 2 + - dns.flags 2 + - dns.count.queries 2 + - dns.count.answers 2 + - dns.count.auth_rr 2 + - dns.count.add_rr 2 + - queries (query-size + 6) + - answers : + - dns.resp.name 2 + - dns.resp.type 2 + - dns.resp.class 2 + - dns.resp.ttl 4 + - dns.resp.len 2 + - dns.txt.length 1 + - dns.txt resp_data_size + +So the total size is roughly a fixed overhead of `39`, and the size of the +query (domain name) and response. +The query size is, for example, FVY6INQ6LZ33WLCHO3BPR3FH6Y.snap.mainnet.ethdisco.net (52) + +We also have some static data in the response, such as `enrtree-branch:`, and potentially +splitting the response up with `" "`, leaving us with a size of roughly `400` that we need +to stay below. + +The number `370` is used to have some margin for extra overhead (for example, the dns query +may be larger - more subdomains). +*/ const ( - hashAbbrev = 16 - maxChildren = 300 / hashAbbrev * (13 / 8) - minHashLength = 12 + hashAbbrevSize = 1 + 16*13/8 // Size of an encoded hash (plus comma) + maxChildren = 370 / hashAbbrevSize // 13 children + minHashLength = 12 ) // MakeTree creates a tree containing the given nodes and links. From c454717fa67df71b4a3c2f16cd0f6bc186148a04 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Fri, 19 Mar 2021 13:32:57 +0100 Subject: [PATCH 214/709] core: fix potential race in chainIndexerTest (#22346) --- core/chain_indexer_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index b76203dc8f..f099609015 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -18,6 +18,7 @@ package core import ( "context" + "errors" "fmt" "math/big" "math/rand" @@ -224,7 +225,10 @@ func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Heade //t.processCh <- header.Number.Uint64() select { case <-time.After(10 * time.Second): - b.t.Fatal("Unexpected call to Process") + b.t.Error("Unexpected call to Process") + // Can't use Fatal since this is not the test's goroutine. + // Returning error stops the chainIndexer's updateLoop + return errors.New("Unexpected call to Process") case b.processCh <- header.Number.Uint64(): } return nil From d3040a80d7a0721297cfac93b82e0e9428ddd3d1 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 19 Mar 2021 14:15:39 +0000 Subject: [PATCH 215/709] cmd/devp2p/internal/ethtest: skip eth/66 tests when v66 not supported (#22460) --- cmd/devp2p/internal/ethtest/chain_test.go | 58 +++++++++++++++++-- cmd/devp2p/internal/ethtest/eth66_suite.go | 10 ++++ .../internal/ethtest/eth66_suiteHelpers.go | 1 + cmd/devp2p/internal/ethtest/suite.go | 37 +++++++++++- cmd/devp2p/internal/ethtest/types.go | 17 +++--- cmd/devp2p/rlpxcmd.go | 8 ++- 6 files changed, 116 insertions(+), 15 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain_test.go b/cmd/devp2p/internal/ethtest/chain_test.go index 5e4289d80a..ec98833ab5 100644 --- a/cmd/devp2p/internal/ethtest/chain_test.go +++ b/cmd/devp2p/internal/ethtest/chain_test.go @@ -35,7 +35,31 @@ func TestEthProtocolNegotiation(t *testing.T) { expected uint32 }{ { - conn: &Conn{}, + conn: &Conn{ + ourHighestProtoVersion: 65, + }, + caps: []p2p.Cap{ + {Name: "eth", Version: 63}, + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + }, + expected: uint32(65), + }, + { + conn: &Conn{ + ourHighestProtoVersion: 65, + }, + caps: []p2p.Cap{ + {Name: "eth", Version: 63}, + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + }, + expected: uint32(65), + }, + { + conn: &Conn{ + ourHighestProtoVersion: 65, + }, caps: []p2p.Cap{ {Name: "eth", Version: 63}, {Name: "eth", Version: 64}, @@ -44,7 +68,20 @@ func TestEthProtocolNegotiation(t *testing.T) { expected: uint32(65), }, { - conn: &Conn{}, + conn: &Conn{ + ourHighestProtoVersion: 64, + }, + caps: []p2p.Cap{ + {Name: "eth", Version: 63}, + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + }, + expected: 64, + }, + { + conn: &Conn{ + ourHighestProtoVersion: 65, + }, caps: []p2p.Cap{ {Name: "eth", Version: 0}, {Name: "eth", Version: 89}, @@ -53,7 +90,20 @@ func TestEthProtocolNegotiation(t *testing.T) { expected: uint32(65), }, { - conn: &Conn{}, + conn: &Conn{ + ourHighestProtoVersion: 64, + }, + caps: []p2p.Cap{ + {Name: "eth", Version: 63}, + {Name: "eth", Version: 64}, + {Name: "wrongProto", Version: 65}, + }, + expected: uint32(64), + }, + { + conn: &Conn{ + ourHighestProtoVersion: 65, + }, caps: []p2p.Cap{ {Name: "eth", Version: 63}, {Name: "eth", Version: 64}, @@ -66,7 +116,7 @@ func TestEthProtocolNegotiation(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { tt.conn.negotiateEthProtocol(tt.caps) - assert.Equal(t, tt.expected, uint32(tt.conn.ethProtocolVersion)) + assert.Equal(t, tt.expected, uint32(tt.conn.negotiatedProtoVersion)) }) } } diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 644fed61eb..63fb1af597 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -26,6 +26,16 @@ import ( "github.com/ethereum/go-ethereum/p2p" ) +// Is_66 checks if the node supports the eth66 protocol version, +// and if not, exists the test suite +func (s *Suite) Is_66(t *utesting.T) { + conn := s.dial66(t) + conn.handshake(t) + if conn.negotiatedProtoVersion < 66 { + t.Fail() + } +} + // TestStatus_66 attempts to connect to the given node and exchange // a status message with it on the eth66 protocol, and then check to // make sure the chain head is correct. diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go index a3f1aaa4f8..4ef349740f 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -46,6 +46,7 @@ func (s *Suite) dial66(t *utesting.T) *Conn { t.Fatalf("could not dial: %v", err) } conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) + conn.ourHighestProtoVersion = 66 return conn } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 48010b90dd..7b2a22db70 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -65,7 +65,7 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e }, nil } -func (s *Suite) EthTests() []utesting.Test { +func (s *Suite) AllEthTests() []utesting.Test { return []utesting.Test{ // status {Name: "Status", Fn: s.TestStatus}, @@ -97,6 +97,38 @@ func (s *Suite) EthTests() []utesting.Test { } } +func (s *Suite) EthTests() []utesting.Test { + return []utesting.Test{ + {Name: "Status", Fn: s.TestStatus}, + {Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "GetBlockBodies", Fn: s.TestGetBlockBodies}, + {Name: "Broadcast", Fn: s.TestBroadcast}, + {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, + {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus}, + {Name: "TestTransactions", Fn: s.TestTransaction}, + {Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx}, + } +} + +func (s *Suite) Eth66Tests() []utesting.Test { + return []utesting.Test{ + // only proceed with eth66 test suite if node supports eth 66 protocol + {Name: "Status_66", Fn: s.TestStatus_66}, + {Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, + {Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66}, + {Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66}, + {Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66}, + {Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, + {Name: "Broadcast_66", Fn: s.TestBroadcast_66}, + {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, + {Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66}, + {Name: "TestTransactions_66", Fn: s.TestTransaction_66}, + {Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66}, + } +} + // TestStatus attempts to connect to the given node and exchange // a status message with it, and then check to make sure // the chain head is correct. @@ -125,7 +157,7 @@ func (s *Suite) TestMaliciousStatus(t *utesting.T) { // get protoHandshake conn.handshake(t) status := &Status{ - ProtocolVersion: uint32(conn.ethProtocolVersion), + ProtocolVersion: uint32(conn.negotiatedProtoVersion), NetworkID: s.chain.chainConfig.ChainID.Uint64(), TD: largeNumber(2), Head: s.chain.blocks[s.chain.Len()-1].Hash(), @@ -421,6 +453,7 @@ func (s *Suite) dial() (*Conn, error) { {Name: "eth", Version: 64}, {Name: "eth", Version: 65}, } + conn.ourHighestProtoVersion = 65 return &conn, nil } diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index 96012b3156..1e2ae77965 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -123,9 +123,10 @@ func (nb NewPooledTransactionHashes) Code() int { return 24 } // Conn represents an individual connection with a peer type Conn struct { *rlpx.Conn - ourKey *ecdsa.PrivateKey - ethProtocolVersion uint - caps []p2p.Cap + ourKey *ecdsa.PrivateKey + negotiatedProtoVersion uint + ourHighestProtoVersion uint + caps []p2p.Cap } func (c *Conn) Read() Message { @@ -236,7 +237,7 @@ func (c *Conn) handshake(t *utesting.T) Message { c.SetSnappy(true) } c.negotiateEthProtocol(msg.Caps) - if c.ethProtocolVersion == 0 { + if c.negotiatedProtoVersion == 0 { t.Fatalf("unexpected eth protocol version") } return msg @@ -254,11 +255,11 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) { if capability.Name != "eth" { continue } - if capability.Version > highestEthVersion && capability.Version <= 65 { + if capability.Version > highestEthVersion && capability.Version <= c.ourHighestProtoVersion { highestEthVersion = capability.Version } } - c.ethProtocolVersion = highestEthVersion + c.negotiatedProtoVersion = highestEthVersion } // statusExchange performs a `Status` message exchange with the given @@ -295,13 +296,13 @@ loop: } } // make sure eth protocol version is set for negotiation - if c.ethProtocolVersion == 0 { + if c.negotiatedProtoVersion == 0 { t.Fatalf("eth protocol version must be set in Conn") } if status == nil { // write status message to client status = &Status{ - ProtocolVersion: uint32(c.ethProtocolVersion), + ProtocolVersion: uint32(c.negotiatedProtoVersion), NetworkID: chain.chainConfig.ChainID.Uint64(), TD: chain.TD(chain.Len()), Head: chain.blocks[chain.Len()-1].Hash(), diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index ac92818aa4..24a16f0b3c 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" "github.com/ethereum/go-ethereum/rlp" @@ -98,5 +99,10 @@ func rlpxEthTest(ctx *cli.Context) error { if err != nil { exit(err) } - return runTests(ctx, suite.EthTests()) + // check if given node supports eth66, and if so, run eth66 protocol tests as well + is66Failed, _ := utesting.Run(utesting.Test{Name: "Is_66", Fn: suite.Is_66}) + if is66Failed { + return runTests(ctx, suite.EthTests()) + } + return runTests(ctx, suite.AllEthTests()) } From 9429ab14727f55d3d0231a86a3cf37039ea79c9a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 20 Mar 2021 00:22:24 +0100 Subject: [PATCH 216/709] cmd/devp2p: add flag for AWS region (#22537) --- cmd/devp2p/dns_route53.go | 8 +++++++- cmd/devp2p/dnscmd.go | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 5f534ff9f7..b32a885350 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -59,6 +59,11 @@ var ( Name: "zone-id", Usage: "Route53 Zone ID", } + route53RegionFlag = cli.StringFlag{ + Name: "aws-region", + Usage: "AWS Region", + Value: "eu-central-1", + } ) type route53Client struct { @@ -76,13 +81,14 @@ func newRoute53Client(ctx *cli.Context) *route53Client { akey := ctx.String(route53AccessKeyFlag.Name) asec := ctx.String(route53AccessSecretFlag.Name) if akey == "" || asec == "" { - exit(fmt.Errorf("need Route53 Access Key ID and secret proceed")) + exit(fmt.Errorf("need Route53 Access Key ID and secret to proceed")) } creds := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(akey, asec, "")) cfg, err := config.LoadDefaultConfig(context.Background(), config.WithCredentialsProvider(creds)) if err != nil { exit(fmt.Errorf("can't initialize AWS configuration: %v", err)) } + cfg.Region = ctx.String(route53RegionFlag.Name) return &route53Client{ api: route53.NewFromConfig(cfg), zoneID: ctx.String(route53ZoneIDFlag.Name), diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index f56f0f34e4..50ab7bf983 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -77,7 +77,12 @@ var ( Usage: "Deploy DNS TXT records to Amazon Route53", ArgsUsage: "", Action: dnsToRoute53, - Flags: []cli.Flag{route53AccessKeyFlag, route53AccessSecretFlag, route53ZoneIDFlag}, + Flags: []cli.Flag{ + route53AccessKeyFlag, + route53AccessSecretFlag, + route53ZoneIDFlag, + route53RegionFlag, + }, } ) From 5bf6612a2e19693b7faa032187f02a019cbea700 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 20 Mar 2021 10:35:22 +0100 Subject: [PATCH 217/709] cmd/devp2p: fix error in updating the cursor when collecting records for route53 (#22538) This PR fixes a regression introduced in #22360, when we updated to the v2 of the AWS sdk, which causes current crawler to just get the same first 100 results over and over, and get stuck in a loop. --- cmd/devp2p/dns_route53.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index b32a885350..010913060a 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -315,8 +315,16 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error break } - // sets the cursor to the next batch + // Set the cursor to the next batc. From the AWS docs: + // + // To display the next page of results, get the values of NextRecordName, + // NextRecordType, and NextRecordIdentifier (if any) from the response. Then submit + // another ListResourceRecordSets request, and specify those values for + // StartRecordName, StartRecordType, and StartRecordIdentifier. + req.StartRecordIdentifier = resp.NextRecordIdentifier + req.StartRecordName = resp.NextRecordName + req.StartRecordType = resp.NextRecordType } return existing, nil From 36b51b8156a4079952d646aac2800423dd8df163 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Sat, 20 Mar 2021 18:50:44 +0000 Subject: [PATCH 218/709] cmd/devp2p: add old block announcement test to eth test suite (#22474) Add old block announcement test to eth test suite, checks to make sure old block announcement isn't propagated --- cmd/devp2p/internal/ethtest/eth66_suite.go | 4 +++ cmd/devp2p/internal/ethtest/suite.go | 33 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 63fb1af597..0995dcb3e4 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -215,6 +215,10 @@ func (s *Suite) TestLargeAnnounce_66(t *utesting.T) { } } +func (s *Suite) TestOldAnnounce_66(t *utesting.T) { + s.oldAnnounce(t, s.setupConnection66(t), s.setupConnection66(t)) +} + // TestMaliciousHandshake_66 tries to send malicious data during the handshake. func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) { conn := s.dial66(t) diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 7b2a22db70..66fb8026a0 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -19,6 +19,7 @@ package ethtest import ( "fmt" "net" + "strings" "time" "github.com/davecgh/go-spew/spew" @@ -84,6 +85,8 @@ func (s *Suite) AllEthTests() []utesting.Test { {Name: "Broadcast_66", Fn: s.TestBroadcast_66}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, + {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, + {Name: "TestOldAnnounce_66", Fn: s.TestOldAnnounce_66}, // malicious handshakes + status {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, @@ -389,6 +392,36 @@ func (s *Suite) TestLargeAnnounce(t *utesting.T) { } } +func (s *Suite) TestOldAnnounce(t *utesting.T) { + s.oldAnnounce(t, s.setupConnection(t), s.setupConnection(t)) +} + +func (s *Suite) oldAnnounce(t *utesting.T, sendConn, receiveConn *Conn) { + oldBlockAnnounce := &NewBlock{ + Block: s.chain.blocks[len(s.chain.blocks)/2], + TD: s.chain.blocks[len(s.chain.blocks)/2].Difficulty(), + } + + if err := sendConn.Write(oldBlockAnnounce); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + + switch msg := receiveConn.ReadAndServe(s.chain, timeout*2).(type) { + case *NewBlock: + t.Fatalf("unexpected: block propagated: %s", pretty.Sdump(msg)) + case *NewBlockHashes: + t.Fatalf("unexpected: block announced: %s", pretty.Sdump(msg)) + case *Error: + errMsg := *msg + // check to make sure error is timeout (propagation didn't come through == test successful) + if !strings.Contains(errMsg.String(), "timeout") { + t.Fatalf("unexpected error: %v", pretty.Sdump(msg)) + } + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } +} + func (s *Suite) testAnnounce(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) { // Announce the block. if err := sendConn.Write(blockAnnouncement); err != nil { From ec73ec092d149e20e13cdb0f4b9dbefa9d21a19e Mon Sep 17 00:00:00 2001 From: Tobias Hildebrandt <79341166+tobias-hildebrandt@users.noreply.github.com> Date: Sat, 20 Mar 2021 18:54:17 +0000 Subject: [PATCH 219/709] cmd/utils: fix compilation issue on openbsd (#22511) --- cmd/utils/diskusage.go | 2 +- cmd/utils/diskusage_openbsd.go | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 cmd/utils/diskusage_openbsd.go diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index da696de6bf..c3d51765f8 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build !windows +// +build !windows,!openbsd package utils diff --git a/cmd/utils/diskusage_openbsd.go b/cmd/utils/diskusage_openbsd.go new file mode 100644 index 0000000000..54f759d291 --- /dev/null +++ b/cmd/utils/diskusage_openbsd.go @@ -0,0 +1,43 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build openbsd + +package utils + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func getFreeDiskSpace(path string) (uint64, error) { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return 0, fmt.Errorf("failed to call Statfs: %v", err) + } + + // Available blocks * size per block = available space in bytes + var bavail = stat.F_bavail + // Not sure if the following check is necessary for OpenBSD + if stat.F_bavail < 0 { + // FreeBSD can have a negative number of blocks available + // because of the grace limit. + bavail = 0 + } + //nolint:unconvert + return uint64(bavail) * uint64(stat.F_bsize), nil +} From eaccdba4ab310e3fb98edbc4b340b5e7c4d767fd Mon Sep 17 00:00:00 2001 From: Derek Chiang Date: Mon, 22 Mar 2021 00:10:51 -0700 Subject: [PATCH 220/709] core: fix method comment for `txpool.requestReset` (#22543) --- core/tx_pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 4c1bd809fd..5db1d3df32 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -949,7 +949,7 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { } } -// requestPromoteExecutables requests a pool reset to the new head block. +// requestReset requests a pool reset to the new head block. // The returned channel is closed when the reset has occurred. func (pool *TxPool) requestReset(oldHead *types.Header, newHead *types.Header) chan struct{} { select { From aab35600bcfbf1c7cfff39c4b7b8b594d19ce358 Mon Sep 17 00:00:00 2001 From: MrChico Date: Mon, 22 Mar 2021 09:29:32 +0100 Subject: [PATCH 221/709] accounts: eip-712 signing for ledger (#22378) * accounts: eip-712 signing for ledger * address review comments --- accounts/usbwallet/ledger.go | 82 ++++++++++++++++++++++++++++++++++++ accounts/usbwallet/trezor.go | 4 ++ accounts/usbwallet/wallet.go | 43 ++++++++++++++++++- 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 71f0f9392f..3de3b4091c 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -52,8 +52,10 @@ const ( ledgerOpRetrieveAddress ledgerOpcode = 0x02 // Returns the public key and Ethereum address for a given BIP 32 path ledgerOpSignTransaction ledgerOpcode = 0x04 // Signs an Ethereum transaction after having the user validate the parameters ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration + ledgerOpSignTypedMessage ledgerOpcode = 0x0c // Signs an Ethereum message following the EIP 712 specification ledgerP1DirectlyFetchAddress ledgerParam1 = 0x00 // Return address directly from the wallet + ledgerP1InitTypedMessageData ledgerParam1 = 0x00 // First chunk of Typed Message data ledgerP1InitTransactionData ledgerParam1 = 0x00 // First transaction data block for signing ledgerP1ContTransactionData ledgerParam1 = 0x80 // Subsequent transaction data block for signing ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address @@ -170,6 +172,24 @@ func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio return w.ledgerSign(path, tx, chainID) } +// SignTypedMessage implements usbwallet.driver, sending the message to the Ledger and +// waiting for the user to sign or deny the transaction. +// +// Note: this was introduced in the ledger 1.5.0 firmware +func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash []byte, messageHash []byte) ([]byte, error) { + // If the Ethereum app doesn't run, abort + if w.offline() { + return nil, accounts.ErrWalletClosed + } + // Ensure the wallet is capable of signing the given transaction + if w.version[0] < 1 && w.version[1] < 5 { + //lint:ignore ST1005 brand name displayed on the console + return nil, fmt.Errorf("Ledger version >= 1.5.0 required for EIP-712 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2]) + } + // All infos gathered and metadata checks out, request signing + return w.ledgerSignTypedMessage(path, domainHash, messageHash) +} + // ledgerVersion retrieves the current version of the Ethereum wallet app running // on the Ledger wallet. // @@ -367,6 +387,68 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction return sender, signed, nil } +// ledgerSignTypedMessage sends the transaction to the Ledger wallet, and waits for the user +// to confirm or deny the transaction. +// +// The signing protocol is defined as follows: +// +// CLA | INS | P1 | P2 | Lc | Le +// ----+-----+----+-----------------------------+-----+--- +// E0 | 0C | 00 | implementation version : 00 | variable | variable +// +// Where the input is: +// +// Description | Length +// -------------------------------------------------+---------- +// Number of BIP 32 derivations to perform (max 10) | 1 byte +// First derivation index (big endian) | 4 bytes +// ... | 4 bytes +// Last derivation index (big endian) | 4 bytes +// domain hash | 32 bytes +// message hash | 32 bytes +// +// +// +// And the output data is: +// +// Description | Length +// ------------+--------- +// signature V | 1 byte +// signature R | 32 bytes +// signature S | 32 bytes +func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHash []byte, messageHash []byte) ([]byte, error) { + // Flatten the derivation path into the Ledger request + path := make([]byte, 1+4*len(derivationPath)) + path[0] = byte(len(derivationPath)) + for i, component := range derivationPath { + binary.BigEndian.PutUint32(path[1+4*i:], component) + } + // Create the 712 message + payload := append(path, domainHash...) + payload = append(payload, messageHash...) + + // Send the request and wait for the response + var ( + op = ledgerP1InitTypedMessageData + reply []byte + err error + ) + + // Send the message over, ensuring it's processed correctly + reply, err = w.ledgerExchange(ledgerOpSignTypedMessage, op, 0, payload) + + if err != nil { + return nil, err + } + + // Extract the Ethereum signature and do a sanity validation + if len(reply) != crypto.SignatureLength { + return nil, errors.New("reply lacks signature") + } + signature := append(reply[1:], reply[0]) + return signature, nil +} + // ledgerExchange performs a data exchange with the Ledger wallet, sending it a // message and retrieving the response. // diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index 0546458c47..c2182b88d0 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -185,6 +185,10 @@ func (w *trezorDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio return w.trezorSign(path, tx, chainID) } +func (w *trezorDriver) SignTypedMessage(path accounts.DerivationPath, domainHash []byte, messageHash []byte) ([]byte, error) { + return nil, accounts.ErrNotSupported +} + // trezorDerive sends a derivation request to the Trezor device and returns the // Ethereum address located on that path. func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, error) { diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 9f74e5554f..b6f1814488 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -67,6 +67,8 @@ type driver interface { // SignTx sends the transaction to the USB device and waits for the user to confirm // or deny the transaction. SignTx(path accounts.DerivationPath, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) + + SignTypedMessage(path accounts.DerivationPath, messageHash []byte, domainHash []byte) ([]byte, error) } // wallet represents the common functionality shared by all USB hardware @@ -524,7 +526,46 @@ func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { - return w.signHash(account, crypto.Keccak256(data)) + + // Unless we are doing 712 signing, simply dispatch to signHash + if !(mimeType == accounts.MimetypeTypedData && len(data) == 66 && data[0] == 0x19 && data[1] == 0x01) { + return w.signHash(account, crypto.Keccak256(data)) + } + + // dispatch to 712 signing if the mimetype is TypedData and the format matches + w.stateLock.RLock() // Comms have own mutex, this is for the state fields + defer w.stateLock.RUnlock() + + // If the wallet is closed, abort + if w.device == nil { + return nil, accounts.ErrWalletClosed + } + // Make sure the requested account is contained within + path, ok := w.paths[account.Address] + if !ok { + return nil, accounts.ErrUnknownAccount + } + // All infos gathered and metadata checks out, request signing + <-w.commsLock + defer func() { w.commsLock <- struct{}{} }() + + // Ensure the device isn't screwed with while user confirmation is pending + // TODO(karalabe): remove if hotplug lands on Windows + w.hub.commsLock.Lock() + w.hub.commsPend++ + w.hub.commsLock.Unlock() + + defer func() { + w.hub.commsLock.Lock() + w.hub.commsPend-- + w.hub.commsLock.Unlock() + }() + // Sign the transaction + signature, err := w.driver.SignTypedMessage(path, data[2:34], data[34:66]) + if err != nil { + return nil, err + } + return signature, nil } // SignDataWithPassphrase implements accounts.Wallet, attempting to sign the given From 0c70b83e0012ce0f9c7b76e61eef7dde5a360838 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 23 Mar 2021 02:06:30 +0800 Subject: [PATCH 222/709] all: add read-only option to database (#22407) * all: add read-only option to database * all: fixes tests * cmd/geth: migrate flags * cmd/geth: fix the compact * cmd/geth: fix the format * cmd/geth: fix log * cmd: add chain-readonly * core: add readonly notion to freezer * core/rawdb: add log * core/rawdb: fix freezer close * cmd: fix * cmd, core: construct db * core: update tests --- cmd/geth/chaincmd.go | 33 +++--- cmd/geth/dao_test.go | 2 +- cmd/geth/dbcmd.go | 151 ++++++++++++++++---------- cmd/geth/snapshot.go | 57 +++++----- cmd/utils/flags.go | 22 ++-- core/bench_test.go | 8 +- core/blockchain_repair_test.go | 6 +- core/blockchain_sethead_test.go | 4 +- core/blockchain_snapshot_test.go | 4 +- core/blockchain_test.go | 18 +-- core/rawdb/accessors_chain.go | 26 +++++ core/rawdb/accessors_chain_test.go | 2 +- core/rawdb/database.go | 25 +++-- core/rawdb/freezer.go | 18 ++- core/state/pruner/pruner.go | 36 ++---- core/state/snapshot/disklayer_test.go | 2 +- eth/backend.go | 2 +- eth/filters/bench_test.go | 6 +- eth/filters/filter_test.go | 4 +- eth/protocols/eth/protocol_test.go | 2 +- ethdb/leveldb/leveldb.go | 5 +- les/client.go | 4 +- les/server.go | 2 +- node/node.go | 8 +- node/node_test.go | 6 +- trie/trie_test.go | 2 +- 26 files changed, 265 insertions(+), 190 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2010ae4b30..61401dd59f 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -191,7 +192,7 @@ func initGenesis(ctx *cli.Context) error { defer stack.Close() for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabase(name, 0, 0, "") + chaindb, err := stack.OpenDatabase(name, 0, 0, "", false) if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -229,7 +230,7 @@ func importChain(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, db := utils.MakeChain(ctx, stack, false) + chain, db := utils.MakeChain(ctx, stack) defer db.Close() // Start periodically gathering memory profiles @@ -304,7 +305,7 @@ func exportChain(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, _ := utils.MakeChain(ctx, stack, true) + chain, _ := utils.MakeChain(ctx, stack) start := time.Now() var err error @@ -340,7 +341,7 @@ func importPreimages(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, false) start := time.Now() if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil { @@ -359,7 +360,7 @@ func exportPreimages(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) start := time.Now() if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil { @@ -373,21 +374,27 @@ func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, chainDb := utils.MakeChain(ctx, stack, true) - defer chainDb.Close() + db := utils.MakeChainDatabase(ctx, stack, true) for _, arg := range ctx.Args() { - var block *types.Block + var header *types.Header if hashish(arg) { - block = chain.GetBlockByHash(common.HexToHash(arg)) + hash := common.HexToHash(arg) + number := rawdb.ReadHeaderNumber(db, hash) + if number != nil { + header = rawdb.ReadHeader(db, hash, *number) + } } else { - num, _ := strconv.Atoi(arg) - block = chain.GetBlockByNumber(uint64(num)) + number, _ := strconv.Atoi(arg) + hash := rawdb.ReadCanonicalHash(db, uint64(number)) + if hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, uint64(number)) + } } - if block == nil { + if header == nil { fmt.Println("{}") utils.Fatalf("block not found") } else { - state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil) + state, err := state.New(header.Root, state.NewDatabase(db), nil) if err != nil { utils.Fatalf("could not create new state: %v", err) } diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 29b1a7f474..b7f26b3652 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc } // Retrieve the DAO config flag from the database path := filepath.Join(datadir, "geth", "chaindata") - db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "") + db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false) if err != nil { t.Fatalf("test %d: failed to open test database: %v", test, err) } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 48478f613e..078cad53b4 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -28,9 +28,7 @@ import ( "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/log" - "github.com/syndtr/goleveldb/leveldb/opt" "gopkg.in/urfave/cli.v1" ) @@ -65,43 +63,98 @@ Remove blockchain and state databases`, Action: utils.MigrateFlags(inspect), Name: "inspect", ArgsUsage: " ", - + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } dbStatCmd = cli.Command{ - Action: dbStats, + Action: utils.MigrateFlags(dbStats), Name: "stats", Usage: "Print leveldb statistics", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, } dbCompactCmd = cli.Command{ - Action: dbCompact, + Action: utils.MigrateFlags(dbCompact), Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + utils.CacheFlag, + utils.CacheDatabaseFlag, + }, Description: `This command performs a database compaction. WARNING: This operation may take a very long time to finish, and may cause database corruption if it is aborted during execution'!`, } dbGetCmd = cli.Command{ - Action: dbGet, - Name: "get", - Usage: "Show the value of a database key", - ArgsUsage: "", + Action: utils.MigrateFlags(dbGet), + Name: "get", + Usage: "Show the value of a database key", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, Description: "This command looks up the specified database key from the database.", } dbDeleteCmd = cli.Command{ - Action: dbDelete, + Action: utils.MigrateFlags(dbDelete), Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } dbPutCmd = cli.Command{ - Action: dbPut, + Action: utils.MigrateFlags(dbPut), Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } @@ -192,10 +245,10 @@ func inspect(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - _, chainDb := utils.MakeChain(ctx, stack, true) - defer chainDb.Close() + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() - return rawdb.InspectDatabase(chainDb, prefix, start) + return rawdb.InspectDatabase(db, prefix, start) } func showLeveldbStats(db ethdb.Stater) { @@ -214,48 +267,32 @@ func showLeveldbStats(db ethdb.Stater) { func dbStats(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - path := stack.ResolvePath("chaindata") - db, err := leveldb.NewCustom(path, "", func(options *opt.Options) { - options.ReadOnly = true - }) - if err != nil { - return err - } + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + showLeveldbStats(db) - err = db.Close() - if err != nil { - log.Info("Close err", "error", err) - } return nil } func dbCompact(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - path := stack.ResolvePath("chaindata") - cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100 - db, err := leveldb.NewCustom(path, "", func(options *opt.Options) { - options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles() - options.BlockCacheCapacity = cache / 2 * opt.MiB - options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally - }) - if err != nil { - return err - } + + db := utils.MakeChainDatabase(ctx, stack, false) + defer db.Close() + + log.Info("Stats before compaction") showLeveldbStats(db) + log.Info("Triggering compaction") - err = db.Compact(nil, nil) - if err != nil { + if err := db.Compact(nil, nil); err != nil { log.Info("Compact err", "error", err) + return err } + log.Info("Stats after compaction") showLeveldbStats(db) - log.Info("Closing db") - err = db.Close() - if err != nil { - log.Info("Close err", "error", err) - } - log.Info("Exiting") - return err + return nil } // dbGet shows the value of a given database key @@ -265,14 +302,10 @@ func dbGet(ctx *cli.Context) error { } stack, _ := makeConfigNode(ctx) defer stack.Close() - path := stack.ResolvePath("chaindata") - db, err := leveldb.NewCustom(path, "", func(options *opt.Options) { - options.ReadOnly = true - }) - if err != nil { - return err - } + + db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() + key, err := hexutil.Decode(ctx.Args().Get(0)) if err != nil { log.Info("Could not decode the key", "error", err) @@ -283,7 +316,7 @@ func dbGet(ctx *cli.Context) error { log.Info("Get operation failed", "error", err) return err } - fmt.Printf("key %#x:\n\t%#x\n", key, data) + fmt.Printf("key %#x: %#x\n", key, data) return nil } @@ -294,13 +327,19 @@ func dbDelete(ctx *cli.Context) error { } stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack) + + db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() + key, err := hexutil.Decode(ctx.Args().Get(0)) if err != nil { log.Info("Could not decode the key", "error", err) return err } + data, err := db.Get(key) + if err == nil { + fmt.Printf("Previous value: %#x\n", data) + } if err = db.Delete(key); err != nil { log.Info("Delete operation returned an error", "error", err) return err @@ -315,8 +354,10 @@ func dbPut(ctx *cli.Context) error { } stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack) + + db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() + var ( key []byte value []byte @@ -335,7 +376,7 @@ func dbPut(ctx *cli.Context) error { } data, err = db.Get(key) if err == nil { - fmt.Printf("Previous value:\n%#x\n", data) + fmt.Printf("Previous value: %#x\n", data) } return db.Put(key, value) } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 068dea0b92..e8f6a35438 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -152,10 +152,8 @@ func pruneState(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) defer stack.Close() - chain, chaindb := utils.MakeChain(ctx, stack, true) - defer chaindb.Close() - - pruner, err := pruner.NewPruner(chaindb, chain.CurrentBlock().Header(), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) + chaindb := utils.MakeChainDatabase(ctx, stack, false) + pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) if err != nil { log.Error("Failed to open snapshot tree", "error", err) return err @@ -183,10 +181,13 @@ func verifyState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, chaindb := utils.MakeChain(ctx, stack, true) - defer chaindb.Close() - - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, chain.CurrentBlock().Root(), false, false, false) + chaindb := utils.MakeChainDatabase(ctx, stack, true) + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + log.Error("Failed to load head block") + return errors.New("no head block") + } + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) if err != nil { log.Error("Failed to open snapshot tree", "error", err) return err @@ -195,7 +196,7 @@ func verifyState(ctx *cli.Context) error { log.Error("Too many arguments given") return errors.New("too many arguments") } - var root = chain.CurrentBlock().Root() + var root = headBlock.Root() if ctx.NArg() == 1 { root, err = parseRoot(ctx.Args()[0]) if err != nil { @@ -218,19 +219,16 @@ func traverseState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, chaindb := utils.MakeChain(ctx, stack, true) - defer chaindb.Close() - + chaindb := utils.MakeChainDatabase(ctx, stack, true) + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + log.Error("Failed to load head block") + return errors.New("no head block") + } if ctx.NArg() > 1 { log.Error("Too many arguments given") return errors.New("too many arguments") } - // Use the HEAD root as the default - head := chain.CurrentBlock() - if head == nil { - log.Error("Head block is missing") - return errors.New("head block is missing") - } var ( root common.Hash err error @@ -243,8 +241,8 @@ func traverseState(ctx *cli.Context) error { } log.Info("Start traversing the state", "root", root) } else { - root = head.Root() - log.Info("Start traversing the state", "root", root, "number", head.NumberU64()) + root = headBlock.Root() + log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) t, err := trie.NewSecure(root, triedb) @@ -311,19 +309,16 @@ func traverseRawState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, chaindb := utils.MakeChain(ctx, stack, true) - defer chaindb.Close() - + chaindb := utils.MakeChainDatabase(ctx, stack, true) + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + log.Error("Failed to load head block") + return errors.New("no head block") + } if ctx.NArg() > 1 { log.Error("Too many arguments given") return errors.New("too many arguments") } - // Use the HEAD root as the default - head := chain.CurrentBlock() - if head == nil { - log.Error("Head block is missing") - return errors.New("head block is missing") - } var ( root common.Hash err error @@ -336,8 +331,8 @@ func traverseRawState(ctx *cli.Context) error { } log.Info("Start traversing the state", "root", root) } else { - root = head.Root() - log.Info("Start traversing the state", "root", root, "number", head.NumberU64()) + root = headBlock.Root() + log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) t, err := trie.NewSecure(root, triedb) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5c800a7fe9..00b28bddf6 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1628,7 +1628,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(DataDirFlag.Name) { // Check if we have an already initialized chain and fall back to // that if so. Otherwise we need to generate a new genesis spec. - chaindb := MakeChainDatabase(ctx, stack) + chaindb := MakeChainDatabase(ctx, stack, true) if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { cfg.Genesis = nil // fallback to db content } @@ -1749,7 +1749,7 @@ func SplitTagsFlag(tagsFlag string) map[string]string { } // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. -func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { +func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database { var ( cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 handles = MakeDatabaseHandles() @@ -1759,10 +1759,10 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { ) if ctx.GlobalString(SyncModeFlag.Name) == "light" { name := "lightchaindata" - chainDb, err = stack.OpenDatabase(name, cache, handles, "") + chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly) } else { name := "chaindata" - chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "") + chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) } if err != nil { Fatalf("Could not open database: %v", err) @@ -1790,9 +1790,9 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { } // MakeChain creates a chain manager from set command line flags. -func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.BlockChain, chainDb ethdb.Database) { +func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) { var err error - chainDb = MakeChainDatabase(ctx, stack) + chainDb = MakeChainDatabase(ctx, stack, false) // TODO(rjl493456442) support read-only database config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx)) if err != nil { Fatalf("%v", err) @@ -1841,12 +1841,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)} - var limit *uint64 - if ctx.GlobalIsSet(TxLookupLimitFlag.Name) && !readOnly { - l := ctx.GlobalUint64(TxLookupLimitFlag.Name) - limit = &l - } - chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, limit) + + // TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only. + // Disable transaction indexing/unindexing by default. + chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil) if err != nil { Fatalf("Can't create BlockChain: %v", err) } diff --git a/core/bench_test.go b/core/bench_test.go index 85653ea5db..0c49907e64 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -157,7 +157,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { b.Fatalf("cannot create temporary directory: %v", err) } defer os.RemoveAll(dir) - db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "") + db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } @@ -255,7 +255,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("cannot create temporary directory: %v", err) } - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "") + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } @@ -272,7 +272,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { } defer os.RemoveAll(dir) - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "") + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } @@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { b.ResetTimer() for i := 0; i < b.N; i++ { - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "") + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index b5cd232a9c..8bb39d2607 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1762,7 +1762,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -1817,7 +1817,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) + Freeze(threshold uint64) error Ancients() (uint64, error) } db.(freezer).Freeze(tt.freezeThreshold) @@ -1830,7 +1830,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { db.Close() // Start a new blockchain back up and see where the repait leads us - db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") + db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 45c4073eb4..e99b09cf8c 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -1961,7 +1961,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { } os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -2023,7 +2023,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { } // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) + Freeze(threshold uint64) error Ancients() (uint64, error) } db.(freezer).Freeze(tt.freezeThreshold) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 96a5c7a8d4..70ae31f7d8 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -65,7 +65,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo } os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "") + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -261,7 +261,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { db.Close() // Start a new blockchain back up and see where the repair leads us - newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "") + newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "", false) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 3e4757f8b6..fd7f1aea1b 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -651,7 +651,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -725,7 +725,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(dir) - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "") + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1592,7 +1592,7 @@ func TestBlockchainRecovery(t *testing.T) { } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1649,7 +1649,7 @@ func TestIncompleteAncientReceiptChainInsertion(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1848,7 +1848,7 @@ func testInsertKnownChainData(t *testing.T, typ string) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(dir) - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "") + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2128,7 +2128,7 @@ func TestTransactionIndices(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2156,7 +2156,7 @@ func TestTransactionIndices(t *testing.T) { // Init block chain with external ancients, check all needed indices has been indexed. limit := []uint64{0, 32, 64, 128} for _, l := range limit { - ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2176,7 +2176,7 @@ func TestTransactionIndices(t *testing.T) { } // Reconstruct a block chain which only reserves HEAD-64 tx indices - ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2255,7 +2255,7 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "") + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 461e1cbb17..92450313b4 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -825,3 +825,29 @@ func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header { } return a } + +// ReadHeadHeader returns the current canonical head header. +func ReadHeadHeader(db ethdb.Reader) *types.Header { + headHeaderHash := ReadHeadHeaderHash(db) + if headHeaderHash == (common.Hash{}) { + return nil + } + headHeaderNumber := ReadHeaderNumber(db, headHeaderHash) + if headHeaderNumber == nil { + return nil + } + return ReadHeader(db, headHeaderHash, *headHeaderNumber) +} + +// ReadHeadHeader returns the current canonical head block. +func ReadHeadBlock(db ethdb.Reader) *types.Block { + headBlockHash := ReadHeadBlockHash(db) + if headBlockHash == (common.Hash{}) { + return nil + } + headBlockNumber := ReadHeaderNumber(db, headBlockHash) + if headBlockNumber == nil { + return nil + } + return ReadBlock(db, headBlockHash, *headBlockNumber) +} diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index a5804cd309..ea9dc436cf 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -440,7 +440,7 @@ func TestAncientStorage(t *testing.T) { } defer os.Remove(frdir) - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "") + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) if err != nil { t.Fatalf("failed to create database with ancient backend") } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 91171ef92c..94759eb984 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -57,7 +57,10 @@ func (frdb *freezerdb) Close() error { // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. -func (frdb *freezerdb) Freeze(threshold uint64) { +func (frdb *freezerdb) Freeze(threshold uint64) error { + if frdb.AncientStore.(*freezer).readonly { + return errReadOnly + } // Set the freezer threshold to a temporary value defer func(old uint64) { atomic.StoreUint64(&frdb.AncientStore.(*freezer).threshold, old) @@ -68,6 +71,7 @@ func (frdb *freezerdb) Freeze(threshold uint64) { trigger := make(chan struct{}, 1) frdb.AncientStore.(*freezer).trigger <- trigger <-trigger + return nil } // nofreezedb is a database wrapper that disables freezer data retrievals. @@ -121,9 +125,9 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold // storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) { +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newFreezer(freezer, namespace) + frdb, err := newFreezer(freezer, namespace, readonly) if err != nil { return nil, err } @@ -192,8 +196,9 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st } } // Freezer is consistent with the key-value database, permit combining the two - go frdb.freeze(db) - + if !frdb.readonly { + go frdb.freeze(db) + } return &freezerdb{ KeyValueStore: db, AncientStore: frdb, @@ -215,8 +220,8 @@ func NewMemoryDatabaseWithCap(size int) ethdb.Database { // NewLevelDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) { - db, err := leveldb.New(file, cache, handles, namespace) +func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + db, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } @@ -225,12 +230,12 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string) ( // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a // freezer moving immutable chain segments into cold storage. -func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) { - kvdb, err := leveldb.New(file, cache, handles, namespace) +func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { + kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } - frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace) + frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly) if err != nil { kvdb.Close() return nil, err diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 5744b0cbb3..4e5ae4284e 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -35,6 +35,10 @@ import ( ) var ( + // errReadOnly is returned if the freezer is opened in read only mode. All the + // mutations are disallowed. + errReadOnly = errors.New("read only") + // errUnknownTable is returned if the user attempts to read from a table that is // not tracked by the freezer. errUnknownTable = errors.New("unknown table") @@ -73,6 +77,7 @@ type freezer struct { frozen uint64 // Number of blocks already frozen threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) + readonly bool tables map[string]*freezerTable // Data tables for storing everything instanceLock fileutil.Releaser // File-system lock to prevent double opens @@ -84,7 +89,7 @@ type freezer struct { // newFreezer creates a chain freezer that moves ancient chain data into // append-only flat file containers. -func newFreezer(datadir string, namespace string) (*freezer, error) { +func newFreezer(datadir string, namespace string, readonly bool) (*freezer, error) { // Create the initial freezer object var ( readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) @@ -106,6 +111,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) { } // Open all the supported data tables freezer := &freezer{ + readonly: readonly, threshold: params.FullImmutabilityThreshold, tables: make(map[string]*freezerTable), instanceLock: lock, @@ -130,7 +136,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) { lock.Release() return nil, err } - log.Info("Opened ancient database", "database", datadir) + log.Info("Opened ancient database", "database", datadir, "readonly", readonly) return freezer, nil } @@ -138,7 +144,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) { func (f *freezer) Close() error { var errs []error f.closeOnce.Do(func() { - f.quit <- struct{}{} + close(f.quit) for _, table := range f.tables { if err := table.Close(); err != nil { errs = append(errs, err) @@ -191,6 +197,9 @@ func (f *freezer) AncientSize(kind string) (uint64, error) { // injection will be rejected. But if two injections with same number happen at // the same time, we can get into the trouble. func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td []byte) (err error) { + if f.readonly { + return errReadOnly + } // Ensure the binary blobs we are appending is continuous with freezer. if atomic.LoadUint64(&f.frozen) != number { return errOutOrderInsertion @@ -233,6 +242,9 @@ func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td // TruncateAncients discards any recent data above the provided threshold number. func (f *freezer) TruncateAncients(items uint64) error { + if f.readonly { + return errReadOnly + } if atomic.LoadUint64(&f.frozen) <= items { return nil } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 530a348540..62cc7b0120 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -85,8 +85,12 @@ type Pruner struct { } // NewPruner creates the pruner instance. -func NewPruner(db ethdb.Database, headHeader *types.Header, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) { - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headHeader.Root, false, false, false) +func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) { + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return nil, errors.New("Failed to load head block") + } + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -104,7 +108,7 @@ func NewPruner(db ethdb.Database, headHeader *types.Header, datadir, trieCachePa stateBloom: stateBloom, datadir: datadir, trieCachePath: trieCachePath, - headHeader: headHeader, + headHeader: headBlock.Header(), snaptree: snaptree, }, nil } @@ -350,9 +354,9 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err if stateBloomPath == "" { return nil // nothing to recover } - headHeader, err := getHeadHeader(db) - if err != nil { - return err + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return errors.New("Failed to load head block") } // Initialize the snapshot tree in recovery mode to handle this special case: // - Users run the `prune-state` command multiple times @@ -362,7 +366,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headHeader.Root, false, false, true) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, true) if err != nil { return err // The relevant snapshot(s) might not exist } @@ -382,7 +386,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err // otherwise the dangling state will be left. var ( found bool - layers = snaptree.Snapshots(headHeader.Root, 128, true) + layers = snaptree.Snapshots(headBlock.Root(), 128, true) middleRoots = make(map[common.Hash]struct{}) ) for _, layer := range layers { @@ -506,22 +510,6 @@ func findBloomFilter(datadir string) (string, common.Hash, error) { return stateBloomPath, stateBloomRoot, nil } -func getHeadHeader(db ethdb.Database) (*types.Header, error) { - headHeaderHash := rawdb.ReadHeadBlockHash(db) - if headHeaderHash == (common.Hash{}) { - return nil, errors.New("empty head block hash") - } - headHeaderNumber := rawdb.ReadHeaderNumber(db, headHeaderHash) - if headHeaderNumber == nil { - return nil, errors.New("empty head block number") - } - headHeader := rawdb.ReadHeader(db, headHeaderHash, *headHeaderNumber) - if headHeader == nil { - return nil, errors.New("empty head header") - } - return headHeader, nil -} - const warningLog = ` WARNING! diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 6beb944e07..ccde2fc094 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -524,7 +524,7 @@ func TestDiskSeek(t *testing.T) { t.Fatal(err) } else { defer os.RemoveAll(dir) - diskdb, err := leveldb.New(dir, 256, 0, "") + diskdb, err := leveldb.New(dir, 256, 0, "", false) if err != nil { t.Fatal(err) } diff --git a/eth/backend.go b/eth/backend.go index 76ce5137f4..6e45d27501 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -122,7 +122,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/") + chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) if err != nil { return nil, err } diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 4f70d6d04a..020db070e5 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -65,7 +65,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" b.Log("Running bloombits benchmark section size:", sectionSize) - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } @@ -126,7 +126,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() - db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") + db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) backend = &testBackend{db: db, sections: cnt} } var addr common.Address @@ -157,7 +157,7 @@ func clearBloomBits(db ethdb.Database) { func BenchmarkNoBloomBits(b *testing.B) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" b.Log("Running benchmark without bloombits") - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index f45720d5a9..3fc77bbc4d 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -49,7 +49,7 @@ func BenchmarkFilters(b *testing.B) { defer os.RemoveAll(dir) var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) backend = &testBackend{db: db} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -103,7 +103,7 @@ func TestFilters(t *testing.T) { defer os.RemoveAll(dir) var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) backend = &testBackend{db: db} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index d92f3ea837..7910c9b735 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -178,7 +178,7 @@ func TestEth66Messages(t *testing.T) { // init the receipts { receipts = []*types.Receipt{ - &types.Receipt{ + { Status: types.ReceiptStatusFailed, CumulativeGasUsed: 1, Logs: []*types.Log{ diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 70ac7a91ac..d3011212aa 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -83,7 +83,7 @@ type Database struct { // New returns a wrapped LevelDB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string) (*Database, error) { +func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { return NewCustom(file, namespace, func(options *opt.Options) { // Ensure we have some minimal caching and file guarantees if cache < minCache { @@ -96,6 +96,9 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro options.OpenFilesCacheCapacity = handles options.BlockCacheCapacity = cache / 2 * opt.MiB options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally + if readonly { + options.ReadOnly = true + } }) } diff --git a/les/client.go b/les/client.go index 99f79bb20c..7534eb3ea0 100644 --- a/les/client.go +++ b/les/client.go @@ -80,11 +80,11 @@ type LightEthereum struct { // New creates an instance of the light client. func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { - chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/") + chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false) if err != nil { return nil, err } - lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/") + lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/", false) if err != nil { return nil, err } diff --git a/les/server.go b/les/server.go index 0351bdd801..be64dfe190 100644 --- a/les/server.go +++ b/les/server.go @@ -87,7 +87,7 @@ type LesServer struct { } func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*LesServer, error) { - lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/") + lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/", false) if err != nil { return nil, err } diff --git a/node/node.go b/node/node.go index 2ed4c31f60..1e65fff1c2 100644 --- a/node/node.go +++ b/node/node.go @@ -547,7 +547,7 @@ func (n *Node) EventMux() *event.TypeMux { // OpenDatabase opens an existing database with the given name (or creates one if no // previous can be found) from within the node's instance directory. If the node is // ephemeral, a memory database is returned. -func (n *Node) OpenDatabase(name string, cache, handles int, namespace string) (ethdb.Database, error) { +func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -559,7 +559,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string) ( if n.config.DataDir == "" { db = rawdb.NewMemoryDatabase() } else { - db, err = rawdb.NewLevelDBDatabase(n.ResolvePath(name), cache, handles, namespace) + db, err = rawdb.NewLevelDBDatabase(n.ResolvePath(name), cache, handles, namespace, readonly) } if err == nil { @@ -573,7 +573,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string) ( // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -592,7 +592,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, case !filepath.IsAbs(freezer): freezer = n.ResolvePath(freezer) } - db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly) } if err == nil { diff --git a/node/node_test.go b/node/node_test.go index 6731dbac1f..e104630600 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -158,7 +158,7 @@ func TestNodeCloseClosesDB(t *testing.T) { stack, _ := New(testNodeConfig()) defer stack.Close() - db, err := stack.OpenDatabase("mydb", 0, 0, "") + db, err := stack.OpenDatabase("mydb", 0, 0, "", false) if err != nil { t.Fatal("can't open DB:", err) } @@ -181,7 +181,7 @@ func TestNodeOpenDatabaseFromLifecycleStart(t *testing.T) { var err error stack.RegisterLifecycle(&InstrumentedService{ startHook: func() { - db, err = stack.OpenDatabase("mydb", 0, 0, "") + db, err = stack.OpenDatabase("mydb", 0, 0, "", false) if err != nil { t.Fatal("can't open DB:", err) } @@ -202,7 +202,7 @@ func TestNodeOpenDatabaseFromLifecycleStop(t *testing.T) { stack.RegisterLifecycle(&InstrumentedService{ stopHook: func() { - db, err := stack.OpenDatabase("mydb", 0, 0, "") + db, err := stack.OpenDatabase("mydb", 0, 0, "", false) if err != nil { t.Fatal("can't open DB:", err) } diff --git a/trie/trie_test.go b/trie/trie_test.go index 87bce9abca..3aa4098d14 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -1060,7 +1060,7 @@ func tempDB() (string, *Database) { if err != nil { panic(fmt.Sprintf("can't create temporary directory: %v", err)) } - diskdb, err := leveldb.New(dir, 256, 0, "") + diskdb, err := leveldb.New(dir, 256, 0, "", false) if err != nil { panic(fmt.Sprintf("can't create temporary database: %v", err)) } From 8d6cc1674252371c7441279fd9d482d9416dc8aa Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Mon, 22 Mar 2021 19:11:10 +0100 Subject: [PATCH 223/709] cmd/geth: check block range against chain head in export cmd (#22387) Check the input parameters against the actual head block, exit on error --- cmd/geth/chaincmd.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 61401dd59f..3cae32aa9e 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -322,6 +322,9 @@ func exportChain(ctx *cli.Context) error { if first < 0 || last < 0 { utils.Fatalf("Export error: block number must be greater than 0\n") } + if head := chain.CurrentFastBlock(); uint64(last) > head.NumberU64() { + utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.NumberU64()) + } err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last)) } From a31f6d54dfd49e277b78017c3c5e34d544687f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 22 Mar 2021 22:41:28 +0200 Subject: [PATCH 224/709] core/state/snapshot: fix panic on missing parent --- core/state/snapshot/snapshot.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index aa5f5900b0..810f1354e9 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -283,11 +283,11 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m return errSnapshotCycle } // Generate a new snapshot on top of the parent - parent := t.Snapshot(parentRoot).(snapshot) + parent := t.Snapshot(parentRoot) if parent == nil { return fmt.Errorf("parent [%#x] snapshot missing", parentRoot) } - snap := parent.Update(blockRoot, destructs, accounts, storage) + snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage) // Save the new snapshot for later t.lock.Lock() From e862cbff95ef76daf49779cff94aecaa54538aa4 Mon Sep 17 00:00:00 2001 From: meowsbits Date: Tue, 23 Mar 2021 04:41:23 -0500 Subject: [PATCH 225/709] internal/web3ext, node: migrate node admin API (Start|Stop)RPC->HTTP (#22461) * internal/web3ext,node: migrate node admin API (Start|Stop)RPC->HTTP Corresponding CLI flags --rpc have been moved to --http. This moves the admin module HTTP RPC start/stop methods to an equivalent namespace. Rel https://github.com/ethereum/go-ethereum/pull/22263 Date: 2021-03-08 08:00:11-06:00 Signed-off-by: meows * internal/web3ext: fix startRPC/HTTP param count (4->5) Date: 2021-03-16 06:13:23-05:00 Signed-off-by: meows --- internal/web3ext/web3ext.go | 16 ++++++++++++++-- node/api.go | 23 +++++++++++++++++++---- node/api_test.go | 20 ++++++++++---------- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 6fcf4b8380..aba262699c 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -186,12 +186,24 @@ web3._extend({ call: 'admin_sleepBlocks', params: 2 }), + new web3._extend.Method({ + name: 'startHTTP', + call: 'admin_startHTTP', + params: 5, + inputFormatter: [null, null, null, null, null] + }), + new web3._extend.Method({ + name: 'stopHTTP', + call: 'admin_stopHTTP' + }), + // This method is deprecated. new web3._extend.Method({ name: 'startRPC', call: 'admin_startRPC', - params: 4, - inputFormatter: [null, null, null, null] + params: 5, + inputFormatter: [null, null, null, null, null] }), + // This method is deprecated. new web3._extend.Method({ name: 'stopRPC', call: 'admin_stopRPC' diff --git a/node/api.go b/node/api.go index 083784f4e4..be20b89d95 100644 --- a/node/api.go +++ b/node/api.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rpc" @@ -162,8 +163,8 @@ func (api *privateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, return rpcSub, nil } -// StartRPC starts the HTTP RPC API server. -func (api *privateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { +// StartHTTP starts the HTTP RPC API server. +func (api *privateAdminAPI) StartHTTP(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() @@ -216,12 +217,26 @@ func (api *privateAdminAPI) StartRPC(host *string, port *int, cors *string, apis return true, nil } -// StopRPC shuts down the HTTP server. -func (api *privateAdminAPI) StopRPC() (bool, error) { +// StartRPC starts the HTTP RPC API server. +// This method is deprecated. Use StartHTTP instead. +func (api *privateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { + log.Warn("Deprecation warning", "method", "admin.StartRPC", "use-instead", "admin.StartHTTP") + return api.StartHTTP(host, port, cors, apis, vhosts) +} + +// StopHTTP shuts down the HTTP server. +func (api *privateAdminAPI) StopHTTP() (bool, error) { api.node.http.stop() return true, nil } +// StopRPC shuts down the HTTP server. +// This method is deprecated. Use StopHTTP instead. +func (api *privateAdminAPI) StopRPC() (bool, error) { + log.Warn("Deprecation warning", "method", "admin.StopRPC", "use-instead", "admin.StopHTTP") + return api.StopHTTP() +} + // StartWS starts the websocket RPC API server. func (api *privateAdminAPI) StartWS(host *string, port *int, allowedOrigins *string, apis *string) (bool, error) { api.node.lock.Lock() diff --git a/node/api_test.go b/node/api_test.go index 9c3fa3a31d..9549adf9c2 100644 --- a/node/api_test.go +++ b/node/api_test.go @@ -69,7 +69,7 @@ func TestStartRPC(t *testing.T) { name: "rpc enabled through API", cfg: Config{}, fn: func(t *testing.T, n *Node, api *privateAdminAPI) { - _, err := api.StartRPC(sp("127.0.0.1"), ip(0), nil, nil, nil) + _, err := api.StartHTTP(sp("127.0.0.1"), ip(0), nil, nil, nil) assert.NoError(t, err) }, wantReachable: true, @@ -90,14 +90,14 @@ func TestStartRPC(t *testing.T) { port := listener.Addr().(*net.TCPAddr).Port // Now try to start RPC on that port. This should fail. - _, err = api.StartRPC(sp("127.0.0.1"), ip(port), nil, nil, nil) + _, err = api.StartHTTP(sp("127.0.0.1"), ip(port), nil, nil, nil) if err == nil { - t.Fatal("StartRPC should have failed on port", port) + t.Fatal("StartHTTP should have failed on port", port) } // Try again after unblocking the port. It should work this time. listener.Close() - _, err = api.StartRPC(sp("127.0.0.1"), ip(port), nil, nil, nil) + _, err = api.StartHTTP(sp("127.0.0.1"), ip(port), nil, nil, nil) assert.NoError(t, err) }, wantReachable: true, @@ -109,7 +109,7 @@ func TestStartRPC(t *testing.T) { name: "rpc stopped through API", cfg: Config{HTTPHost: "127.0.0.1"}, fn: func(t *testing.T, n *Node, api *privateAdminAPI) { - _, err := api.StopRPC() + _, err := api.StopHTTP() assert.NoError(t, err) }, wantReachable: false, @@ -121,10 +121,10 @@ func TestStartRPC(t *testing.T) { name: "rpc stopped twice", cfg: Config{HTTPHost: "127.0.0.1"}, fn: func(t *testing.T, n *Node, api *privateAdminAPI) { - _, err := api.StopRPC() + _, err := api.StopHTTP() assert.NoError(t, err) - _, err = api.StopRPC() + _, err = api.StopHTTP() assert.NoError(t, err) }, wantReachable: false, @@ -211,14 +211,14 @@ func TestStartRPC(t *testing.T) { { name: "rpc stopped with ws enabled", fn: func(t *testing.T, n *Node, api *privateAdminAPI) { - _, err := api.StartRPC(sp("127.0.0.1"), ip(0), nil, nil, nil) + _, err := api.StartHTTP(sp("127.0.0.1"), ip(0), nil, nil, nil) assert.NoError(t, err) wsport := n.http.port _, err = api.StartWS(sp("127.0.0.1"), ip(wsport), nil, nil) assert.NoError(t, err) - _, err = api.StopRPC() + _, err = api.StopHTTP() assert.NoError(t, err) }, wantReachable: false, @@ -233,7 +233,7 @@ func TestStartRPC(t *testing.T) { assert.NoError(t, err) wsport := n.http.port - _, err = api.StartRPC(sp("127.0.0.1"), ip(wsport), nil, nil, nil) + _, err = api.StartHTTP(sp("127.0.0.1"), ip(wsport), nil, nil, nil) assert.NoError(t, err) }, wantReachable: true, From 5129cdc4f04889d3f21cadba5a246d62391301a6 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 24 Mar 2021 12:32:39 +0100 Subject: [PATCH 226/709] cmd/devp2p: skip ENR field tails properly in nodeset filter (#22565) In Geth v1.10, we changed the structure of the "les" ENR entry. As a result, the DHT crawler that creates the DNS lists no longer recognizes the les nodes, which is fixed in this commit. * cmd/devp2p: skip ENR field tails properly in nodeset filter * cmd/devp2p: fix tail decoder for snap as well * les: fix tail decoding in "eth" ENR entry --- cmd/devp2p/nodesetcmd.go | 6 +++--- les/enr_entry.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/devp2p/nodesetcmd.go b/cmd/devp2p/nodesetcmd.go index ba97405abc..33de1fdf31 100644 --- a/cmd/devp2p/nodesetcmd.go +++ b/cmd/devp2p/nodesetcmd.go @@ -173,7 +173,7 @@ func ethFilter(args []string) (nodeFilter, error) { f := func(n nodeJSON) bool { var eth struct { ForkID forkid.ID - _ []rlp.RawValue `rlp:"tail"` + Tail []rlp.RawValue `rlp:"tail"` } if n.N.Load(enr.WithEntry("eth", ð)) != nil { return false @@ -186,7 +186,7 @@ func ethFilter(args []string) (nodeFilter, error) { func lesFilter(args []string) (nodeFilter, error) { f := func(n nodeJSON) bool { var les struct { - _ []rlp.RawValue `rlp:"tail"` + Tail []rlp.RawValue `rlp:"tail"` } return n.N.Load(enr.WithEntry("les", &les)) == nil } @@ -196,7 +196,7 @@ func lesFilter(args []string) (nodeFilter, error) { func snapFilter(args []string) (nodeFilter, error) { f := func(n nodeJSON) bool { var snap struct { - _ []rlp.RawValue `rlp:"tail"` + Tail []rlp.RawValue `rlp:"tail"` } return n.N.Load(enr.WithEntry("snap", &snap)) == nil } diff --git a/les/enr_entry.go b/les/enr_entry.go index 892c3530d3..307313fb10 100644 --- a/les/enr_entry.go +++ b/les/enr_entry.go @@ -35,7 +35,7 @@ func (lesEntry) ENRKey() string { return "les" } // ethEntry is the "eth" ENR entry. This is redeclared here to avoid depending on package eth. type ethEntry struct { ForkID forkid.ID - _ []rlp.RawValue `rlp:"tail"` + Tail []rlp.RawValue `rlp:"tail"` } func (ethEntry) ENRKey() string { return "eth" } From 15e6c27f8bcc6a30b47b956701d70f5eb14c1b7c Mon Sep 17 00:00:00 2001 From: Chen Quan Date: Wed, 24 Mar 2021 20:18:29 +0800 Subject: [PATCH 227/709] p2p: fix minor typo and remove fd parameter in checkInboundConn (#22547) --- p2p/server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/server.go b/p2p/server.go index fc71548554..f70ebf7216 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -876,8 +876,8 @@ func (srv *Server) listenLoop() { } remoteIP := netutil.AddrIP(fd.RemoteAddr()) - if err := srv.checkInboundConn(fd, remoteIP); err != nil { - srv.log.Debug("Rejected inbound connnection", "addr", fd.RemoteAddr(), "err", err) + if err := srv.checkInboundConn(remoteIP); err != nil { + srv.log.Debug("Rejected inbound connection", "addr", fd.RemoteAddr(), "err", err) fd.Close() slots <- struct{}{} continue @@ -897,7 +897,7 @@ func (srv *Server) listenLoop() { } } -func (srv *Server) checkInboundConn(fd net.Conn, remoteIP net.IP) error { +func (srv *Server) checkInboundConn(remoteIP net.IP) error { if remoteIP == nil { return nil } From ab8fd4d005bd8d3fdf677f12c3b59f636c06f1a8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 24 Mar 2021 13:37:20 +0100 Subject: [PATCH 228/709] p2p/dnsdisc: rate limit resolving before checking cache (#22566) This makes the rate limit apply regardless of whether the node is already cached. --- p2p/dnsdisc/client.go | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index d3e8111ab5..3e4b50aadd 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -37,9 +37,10 @@ import ( // Client discovers nodes by querying DNS servers. type Client struct { - cfg Config - clock mclock.Clock - entries *lru.Cache + cfg Config + clock mclock.Clock + entries *lru.Cache + ratelimit *rate.Limiter } // Config holds configuration options for the client. @@ -97,8 +98,12 @@ func NewClient(cfg Config) *Client { panic(err) } rlimit := rate.NewLimiter(rate.Limit(cfg.RateLimit), 10) - cfg.Resolver = &rateLimitResolver{cfg.Resolver, rlimit} - return &Client{cfg: cfg, entries: cache, clock: mclock.System{}} + return &Client{ + cfg: cfg, + entries: cache, + clock: mclock.System{}, + ratelimit: rlimit, + } } // SyncTree downloads the entire node tree at the given URL. @@ -157,6 +162,13 @@ func parseAndVerifyRoot(txt string, loc *linkEntry) (rootEntry, error) { // resolveEntry retrieves an entry from the cache or fetches it from the network // if it isn't cached. func (c *Client) resolveEntry(ctx context.Context, domain, hash string) (entry, error) { + // The rate limit always applies, even when the result might be cached. This is + // important because it avoids hot-spinning in consumers of node iterators created on + // this client. + if err := c.ratelimit.Wait(ctx); err != nil { + return nil, err + } + cacheKey := truncateHash(hash) if e, ok := c.entries.Get(cacheKey); ok { return e.(entry), nil @@ -196,19 +208,6 @@ func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry return nil, nameError{name, errNoEntry} } -// rateLimitResolver applies a rate limit to a Resolver. -type rateLimitResolver struct { - r Resolver - limiter *rate.Limiter -} - -func (r *rateLimitResolver) LookupTXT(ctx context.Context, domain string) ([]string, error) { - if err := r.limiter.Wait(ctx); err != nil { - return nil, err - } - return r.r.LookupTXT(ctx, domain) -} - // randomIterator traverses a set of trees and returns nodes found in them. type randomIterator struct { cur *enode.Node From c5df05b9a9529cb17b4416e23cd24583335c6a9d Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 24 Mar 2021 22:33:34 +0800 Subject: [PATCH 229/709] eth/protocols/snap: fix the flaws in the snap sync (#22553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/protocols/snap: fix snap sync * eth/protocols/snap: fix tests * eth: fix tiny * eth: update tests * eth: update tests * core/state/snapshot: testcase for #22534 * eth/protocols/snap: fix boundary loss on full-but-proven range * core/state/snapshot: lintfix * eth: address comment * eth: fix handler Co-authored-by: Martin Holst Swende Co-authored-by: Péter Szilágyi --- eth/protocols/snap/handler.go | 9 +- eth/protocols/snap/sync.go | 24 +- eth/protocols/snap/sync_test.go | 845 +++++++++++++++++++++++++------- 3 files changed, 695 insertions(+), 183 deletions(-) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index b9515b8a39..37e84839ab 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -256,8 +256,13 @@ func handleMessage(backend Backend, peer *Peer) error { var ( storage []*StorageData last common.Hash + abort bool ) - for it.Next() && size < hardLimit { + for it.Next() { + if size >= hardLimit { + abort = true + break + } hash, slot := it.Hash(), common.CopyBytes(it.Slot()) // Track the returned interval for the Merkle proofs @@ -280,7 +285,7 @@ func handleMessage(backend Backend, peer *Peer) error { // Generate the Merkle proofs for the first and last storage slot, but // only if the response was capped. If the entire storage trie included // in the response, no need for any proofs. - if origin != (common.Hash{}) || size >= hardLimit { + if origin != (common.Hash{}) || abort { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs accTrie, err := trie.New(req.Root, backend.Chain().StateCache().TrieDB()) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 0303e65eda..fa2dc16c39 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -1551,7 +1551,14 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { // Ensure that the response doesn't overflow into the subsequent task last := res.task.Last.Big() for i, hash := range res.hashes { - if hash.Big().Cmp(last) > 0 { + // Mark the range complete if the last is already included. + // Keep iteration to delete the extra states if exists. + cmp := hash.Big().Cmp(last) + if cmp == 0 { + res.cont = false + continue + } + if cmp > 0 { // Chunk overflown, cut off excess, but also update the boundary nodes for j := i; j < len(res.hashes); j++ { if err := res.trie.Prove(res.hashes[j][:], 0, res.overflow); err != nil { @@ -1758,7 +1765,14 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // Ensure the response doesn't overflow into the subsequent task last := res.subTask.Last.Big() for k, hash := range res.hashes[i] { - if hash.Big().Cmp(last) > 0 { + // Mark the range complete if the last is already included. + // Keep iteration to delete the extra states if exists. + cmp := hash.Big().Cmp(last) + if cmp == 0 { + res.cont = false + continue + } + if cmp > 0 { // Chunk overflown, cut off excess, but also update the boundary for l := k; l < len(res.hashes[i]); l++ { if err := res.tries[i].Prove(res.hashes[i][l][:], 0, res.overflow); err != nil { @@ -1785,11 +1799,15 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { it := res.nodes[i].NewIterator(nil, nil) for it.Next() { // Boundary nodes are not written for the last result, since they are incomplete - if i == len(res.hashes)-1 { + if i == len(res.hashes)-1 && res.subTask != nil { if _, ok := res.bounds[common.BytesToHash(it.Key())]; ok { skipped++ continue } + if _, err := res.overflow.Get(it.Key()); err == nil { + skipped++ + continue + } } // Node is not a boundary, persist to disk batch.Put(it.Key(), it.Value()) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 49dff7bb3a..3e9778dbc7 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -23,6 +23,7 @@ import ( "fmt" "math/big" "sort" + "sync" "testing" "time" @@ -30,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -111,10 +113,12 @@ func BenchmarkHashing(b *testing.B) { }) } -type storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error -type accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error -type trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error -type codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error +type ( + accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error + storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error + trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error + codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error +) type testPeer struct { id string @@ -130,10 +134,10 @@ type testPeer struct { storageRequestHandler storageHandlerFunc trieRequestHandler trieHandlerFunc codeRequestHandler codeHandlerFunc - cancelCh chan struct{} + term func() } -func newTestPeer(id string, t *testing.T, cancelCh chan struct{}) *testPeer { +func newTestPeer(id string, t *testing.T, term func()) *testPeer { peer := &testPeer{ id: id, test: t, @@ -142,12 +146,11 @@ func newTestPeer(id string, t *testing.T, cancelCh chan struct{}) *testPeer { trieRequestHandler: defaultTrieRequestHandler, storageRequestHandler: defaultStorageRequestHandler, codeRequestHandler: defaultCodeRequestHandler, - cancelCh: cancelCh, + term: term, } //stderrHandler := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) //peer.logger.SetHandler(stderrHandler) return peer - } func (t *testPeer) ID() string { return t.id } @@ -155,7 +158,7 @@ func (t *testPeer) Log() log.Logger { return t.logger } func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { t.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) - go t.accountRequestHandler(t, id, root, origin, bytes) + go t.accountRequestHandler(t, id, root, origin, limit, bytes) return nil } @@ -211,20 +214,21 @@ func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, } // defaultAccountRequestHandler is a well-behaving handler for AccountRangeRequests -func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, cap uint64) error { - keys, vals, proofs := createAccountRequestResponse(t, root, origin, cap) +func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + keys, vals, proofs := createAccountRequestResponse(t, root, origin, limit, cap) if err := t.remote.OnAccounts(t, id, keys, vals, proofs); err != nil { - t.logger.Error("remote error on delivery", "error", err) t.test.Errorf("Remote side rejected our delivery: %v", err) - t.remote.Unregister(t.id) - close(t.cancelCh) + t.term() return err } return nil } -func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, cap uint64) (keys []common.Hash, vals [][]byte, proofs [][]byte) { +func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) (keys []common.Hash, vals [][]byte, proofs [][]byte) { var size uint64 + if limit == (common.Hash{}) { + limit = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + } for _, entry := range t.accountValues { if size > cap { break @@ -234,20 +238,22 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H vals = append(vals, entry.v) size += uint64(32 + len(entry.v)) } + // If we've exceeded the request threshold, abort + if bytes.Compare(entry.k, limit[:]) >= 0 { + break + } } // Unless we send the entire trie, we need to supply proofs - // Actually, we need to supply proofs either way! This seems tob be an implementation + // Actually, we need to supply proofs either way! This seems to be an implementation // quirk in go-ethereum proof := light.NewNodeSet() if err := t.accountTrie.Prove(origin[:], 0, proof); err != nil { - t.logger.Error("Could not prove inexistence of origin", "origin", origin, - "error", err) + t.logger.Error("Could not prove inexistence of origin", "origin", origin, "error", err) } if len(keys) > 0 { lastK := (keys[len(keys)-1])[:] if err := t.accountTrie.Prove(lastK, 0, proof); err != nil { - t.logger.Error("Could not prove last item", - "error", err) + t.logger.Error("Could not prove last item", "error", err) } } for _, blob := range proof.NodeList() { @@ -260,9 +266,8 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) error { hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, bOrigin, bLimit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { - t.logger.Error("remote error on delivery", "error", err) t.test.Errorf("Remote side rejected our delivery: %v", err) - close(t.cancelCh) + t.term() } return nil } @@ -270,58 +275,112 @@ func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Has func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { var bytecodes [][]byte for _, h := range hashes { - bytecodes = append(bytecodes, getCode(h)) + bytecodes = append(bytecodes, getCodeByHash(h)) } if err := t.remote.OnByteCodes(t, id, bytecodes); err != nil { - t.logger.Error("remote error on delivery", "error", err) t.test.Errorf("Remote side rejected our delivery: %v", err) - close(t.cancelCh) + t.term() } return nil } -func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { - var ( - size uint64 - limit = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") - ) - if len(bLimit) > 0 { - limit = common.BytesToHash(bLimit) +func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var size uint64 + for _, account := range accounts { + // The first account might start from a different origin and end sooner + var originHash common.Hash + if len(origin) > 0 { + originHash = common.BytesToHash(origin) + } + var limitHash = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + if len(limit) > 0 { + limitHash = common.BytesToHash(limit) + } + var ( + keys []common.Hash + vals [][]byte + abort bool + ) + for _, entry := range t.storageValues[account] { + if size >= max { + abort = true + break + } + if bytes.Compare(entry.k, originHash[:]) < 0 { + continue + } + keys = append(keys, common.BytesToHash(entry.k)) + vals = append(vals, entry.v) + size += uint64(32 + len(entry.v)) + if bytes.Compare(entry.k, limitHash[:]) >= 0 { + break + } + } + hashes = append(hashes, keys) + slots = append(slots, vals) + + // Generate the Merkle proofs for the first and last storage slot, but + // only if the response was capped. If the entire storage trie included + // in the response, no need for any proofs. + if originHash != (common.Hash{}) || abort { + // If we're aborting, we need to prove the first and last item + // This terminates the response (and thus the loop) + proof := light.NewNodeSet() + stTrie := t.storageTries[account] + + // Here's a potential gotcha: when constructing the proof, we cannot + // use the 'origin' slice directly, but must use the full 32-byte + // hash form. + if err := stTrie.Prove(originHash[:], 0, proof); err != nil { + t.logger.Error("Could not prove inexistence of origin", "origin", originHash, "error", err) + } + if len(keys) > 0 { + lastK := (keys[len(keys)-1])[:] + if err := stTrie.Prove(lastK, 0, proof); err != nil { + t.logger.Error("Could not prove last item", "error", err) + } + } + for _, blob := range proof.NodeList() { + proofs = append(proofs, blob) + } + break + } } + return hashes, slots, proofs +} + +// the createStorageRequestResponseAlwaysProve tests a cornercase, where it always +// supplies the proof for the last account, even if it is 'complete'.h +func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var size uint64 + max = max * 3 / 4 + var origin common.Hash if len(bOrigin) > 0 { origin = common.BytesToHash(bOrigin) } - - var limitExceeded bool - var incomplete bool - for _, account := range accounts { - + var exit bool + for i, account := range accounts { var keys []common.Hash var vals [][]byte for _, entry := range t.storageValues[account] { - if limitExceeded { - incomplete = true - break - } if bytes.Compare(entry.k, origin[:]) < 0 { - incomplete = true - continue + exit = true } keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) size += uint64(32 + len(entry.v)) - if bytes.Compare(entry.k, limit[:]) >= 0 { - limitExceeded = true - } if size > max { - limitExceeded = true + exit = true } } + if i == len(accounts)-1 { + exit = true + } hashes = append(hashes, keys) slots = append(slots, vals) - if incomplete { + if exit { // If we're aborting, we need to prove the first and last item // This terminates the response (and thus the loop) proof := light.NewNodeSet() @@ -350,21 +409,17 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm } // emptyRequestAccountRangeFn is a rejects AccountRangeRequests -func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { - var proofs [][]byte - var keys []common.Hash - var vals [][]byte - t.remote.OnAccounts(t, requestId, keys, vals, proofs) +func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + t.remote.OnAccounts(t, requestId, nil, nil, nil) return nil } -func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { +func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { return nil } func emptyTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { - var nodes [][]byte - t.remote.OnTrieNodes(t, requestId, nodes) + t.remote.OnTrieNodes(t, requestId, nil) return nil } @@ -373,10 +428,7 @@ func nonResponsiveTrieRequestHandler(t *testPeer, requestId uint64, root common. } func emptyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { - var hashes [][]common.Hash - var slots [][][]byte - var proofs [][]byte - t.remote.OnStorage(t, requestId, hashes, slots, proofs) + t.remote.OnStorage(t, requestId, nil, nil, nil) return nil } @@ -384,6 +436,15 @@ func nonResponsiveStorageRequestHandler(t *testPeer, requestId uint64, root comm return nil } +func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + hashes, slots, proofs := createStorageRequestResponseAlwaysProve(t, root, accounts, origin, limit, max) + if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { + t.test.Errorf("Remote side rejected our delivery: %v", err) + t.term() + } + return nil +} + //func emptyCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { // var bytecodes [][]byte // t.remote.OnByteCodes(t, id, bytecodes) @@ -397,7 +458,7 @@ func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max bytecodes = append(bytecodes, h[:]) } if err := t.remote.OnByteCodes(t, id, bytecodes); err != nil { - t.logger.Error("remote error on delivery", "error", err) + t.logger.Info("remote error on delivery (as expected)", "error", err) // Mimic the real-life handler, which drops a peer on errors t.remote.Unregister(t.id) } @@ -407,12 +468,12 @@ func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { var bytecodes [][]byte for _, h := range hashes[:1] { - bytecodes = append(bytecodes, getCode(h)) + bytecodes = append(bytecodes, getCodeByHash(h)) } + // Missing bytecode can be retrieved again, no error expected if err := t.remote.OnByteCodes(t, id, bytecodes); err != nil { - t.logger.Error("remote error on delivery", "error", err) - // Mimic the real-life handler, which drops a peer on errors - t.remote.Unregister(t.id) + t.test.Errorf("Remote side rejected our delivery: %v", err) + t.term() } return nil } @@ -422,16 +483,16 @@ func starvingStorageRequestHandler(t *testPeer, requestId uint64, root common.Ha return defaultStorageRequestHandler(t, requestId, root, accounts, origin, limit, 500) } -func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { - return defaultAccountRequestHandler(t, requestId, root, origin, 500) +func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + return defaultAccountRequestHandler(t, requestId, root, origin, limit, 500) } //func misdeliveringAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { // return defaultAccountRequestHandler(t, requestId-1, root, origin, 500) //} -func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { - hashes, accounts, proofs := createAccountRequestResponse(t, root, origin, cap) +func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + hashes, accounts, proofs := createAccountRequestResponse(t, root, origin, limit, cap) if len(proofs) > 0 { proofs = proofs[1:] } @@ -473,23 +534,36 @@ func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Has func TestSyncBloatedProof(t *testing.T) { t.Parallel() + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(100) - cancel := make(chan struct{}) - source := newTestPeer("source", t, cancel) + source := newTestPeer("source", t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems - source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, cap uint64) error { - var proofs [][]byte - var keys []common.Hash - var vals [][]byte - + source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + var ( + proofs [][]byte + keys []common.Hash + vals [][]byte + ) // The values for _, entry := range t.accountValues { - if bytes.Compare(origin[:], entry.k) <= 0 { - keys = append(keys, common.BytesToHash(entry.k)) - vals = append(vals, entry.v) + if bytes.Compare(entry.k, origin[:]) < 0 { + continue + } + if bytes.Compare(entry.k, limit[:]) > 0 { + continue } + keys = append(keys, common.BytesToHash(entry.k)) + vals = append(vals, entry.v) } // The proofs proof := light.NewNodeSet() @@ -511,9 +585,9 @@ func TestSyncBloatedProof(t *testing.T) { proofs = append(proofs, blob) } if err := t.remote.OnAccounts(t, requestId, keys, vals, proofs); err != nil { - t.logger.Info("remote error on delivery", "error", err) + t.logger.Info("remote error on delivery (as expected)", "error", err) + t.term() // This is actually correct, signal to exit the test successfully - close(t.cancelCh) } return nil } @@ -537,20 +611,28 @@ func setupSyncer(peers ...*testPeer) *Syncer { func TestSync(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(100) mkSource := func(name string) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems return source } - - syncer := setupSyncer(mkSource("sourceA")) + syncer := setupSyncer(mkSource("source")) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a @@ -558,56 +640,79 @@ func TestSync(t *testing.T) { func TestSyncTinyTriePanic(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(1) mkSource := func(name string) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems return source } - - syncer := setupSyncer( - mkSource("nice-a"), - ) - done := checkStall(t, cancel) + syncer := setupSyncer(mkSource("source")) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSync tests a basic sync with multiple peers func TestMultiSync(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(100) mkSource := func(name string) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems return source } - syncer := setupSyncer(mkSource("sourceA"), mkSource("sourceB")) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorage tests basic sync using accounts + storage + code func TestSyncWithStorage(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true, false) mkSource := func(name string) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries @@ -615,33 +720,43 @@ func TestSyncWithStorage(t *testing.T) { return source } syncer := setupSyncer(mkSource("sourceA")) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all func TestMultiSyncManyUseless(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) - mkSource := func(name string, a, b, c bool) *testPeer { - source := newTestPeer(name, t, cancel) + mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries source.storageValues = storageElems - if !a { + if !noAccount { source.accountRequestHandler = emptyRequestAccountRangeFn } - if !b { + if !noStorage { source.storageRequestHandler = emptyStorageRequestHandler } - if !c { + if !noTrieNode { source.trieRequestHandler = emptyTrieRequestHandler } return source @@ -653,9 +768,12 @@ func TestMultiSyncManyUseless(t *testing.T) { mkSource("noStorage", true, false, true), mkSource("noTrie", true, true, false), ) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all @@ -666,24 +784,31 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { defer func(old time.Duration) { requestTimeout = old }(requestTimeout) requestTimeout = time.Millisecond - cancel := make(chan struct{}) - - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) - mkSource := func(name string, a, b, c bool) *testPeer { - source := newTestPeer(name, t, cancel) + mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries source.storageValues = storageElems - if !a { + if !noAccount { source.accountRequestHandler = emptyRequestAccountRangeFn } - if !b { + if !noStorage { source.storageRequestHandler = emptyStorageRequestHandler } - if !c { + if !noTrieNode { source.trieRequestHandler = emptyTrieRequestHandler } return source @@ -695,9 +820,12 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { mkSource("noStorage", true, false, true), mkSource("noTrie", true, true, false), ) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all @@ -706,24 +834,31 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { defer func(old time.Duration) { requestTimeout = old }(requestTimeout) requestTimeout = time.Millisecond - cancel := make(chan struct{}) - - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) - mkSource := func(name string, a, b, c bool) *testPeer { - source := newTestPeer(name, t, cancel) + mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries source.storageValues = storageElems - if !a { + if !noAccount { source.accountRequestHandler = nonResponsiveRequestAccountRangeFn } - if !b { + if !noStorage { source.storageRequestHandler = nonResponsiveStorageRequestHandler } - if !c { + if !noTrieNode { source.trieRequestHandler = nonResponsiveTrieRequestHandler } return source @@ -735,18 +870,21 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { mkSource("noStorage", true, false, true), mkSource("noTrie", true, true, false), ) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } -func checkStall(t *testing.T, cancel chan struct{}) chan struct{} { +func checkStall(t *testing.T, term func()) chan struct{} { testDone := make(chan struct{}) go func() { select { case <-time.After(time.Minute): // TODO(karalabe): Make tests smaller, this is too much t.Log("Sync stalled") - close(cancel) + term() case <-testDone: return } @@ -754,17 +892,58 @@ func checkStall(t *testing.T, cancel chan struct{}) chan struct{} { return testDone } +// TestSyncBoundaryAccountTrie tests sync against a few normal peers, but the +// account trie has a few boundary elements. +func TestSyncBoundaryAccountTrie(t *testing.T) { + t.Parallel() + + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems := makeBoundaryAccountTrie(3000) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, term) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + return source + } + syncer := setupSyncer( + mkSource("peer-a"), + mkSource("peer-b"), + ) + done := checkStall(t, term) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) +} + // TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is // consistently returning very small results func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) mkSource := func(name string, slow bool) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems @@ -780,11 +959,12 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { mkSource("nice-c", false), mkSource("capped", true), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver @@ -792,12 +972,19 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.codeRequestHandler = codeFn @@ -811,22 +998,30 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { mkSource("capped", cappedCodeRequestHandler), mkSource("corrupt", corruptCodeRequestHandler), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) mkSource := func(name string, accFn accountHandlerFunc) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.accountRequestHandler = accFn @@ -840,11 +1035,12 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { mkSource("capped", defaultAccountRequestHandler), mkSource("corrupt", corruptAccountRequestHandler), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes @@ -852,12 +1048,19 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.codeRequestHandler = codeFn @@ -872,7 +1075,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { return cappedCodeRequestHandler(t, id, hashes, max) }), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } @@ -885,6 +1088,43 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { if threshold := 100; counter > threshold { t.Fatalf("Error, expected < %d invocations, got %d", threshold, counter) } + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) +} + +// TestSyncBoundaryStorageTrie tests sync against a few normal peers, but the +// storage trie has a few boundary elements. +func TestSyncBoundaryStorageTrie(t *testing.T) { + t.Parallel() + + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(10, 1000, false, true) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, term) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + return source + } + syncer := setupSyncer( + mkSource("peer-a"), + mkSource("peer-b"), + ) + done := checkStall(t, term) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is @@ -892,12 +1132,19 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false, false) mkSource := func(name string, slow bool) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries @@ -913,11 +1160,12 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { mkSource("nice-a", false), mkSource("slow", true), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is @@ -925,12 +1173,19 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { func TestSyncWithStorageAndCorruptPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries @@ -945,22 +1200,30 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) { mkSource("nice-c", defaultStorageRequestHandler), mkSource("corrupt", corruptStorageRequestHandler), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { t.Parallel() - cancel := make(chan struct{}) - - sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true) + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { - source := newTestPeer(name, t, cancel) + source := newTestPeer(name, t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems source.storageTries = storageTries @@ -968,23 +1231,55 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { source.storageRequestHandler = handler return source } - syncer := setupSyncer( mkSource("nice-a", defaultStorageRequestHandler), mkSource("nice-b", defaultStorageRequestHandler), mkSource("nice-c", defaultStorageRequestHandler), mkSource("corrupt", noProofStorageRequestHandler), ) - done := checkStall(t, cancel) + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } close(done) + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) +} + +// TestSyncWithStorage tests basic sync using accounts + storage + code, against +// a peer who insists on delivering full storage sets _and_ proofs. This triggered +// an error, where the recipient erroneously clipped the boundary nodes, but +// did not mark the account for healing. +func TestSyncWithStorageMisbehavingProve(t *testing.T) { + t.Parallel() + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(10, 30, false) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, term) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + source.storageTries = storageTries + source.storageValues = storageElems + source.storageRequestHandler = proofHappyStorageRequestHandler + return source + } + syncer := setupSyncer(mkSource("sourceA")) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) } type kv struct { k, v []byte - t bool } // Some helpers for sorting @@ -1013,14 +1308,14 @@ var ( } ) -// getACodeHash returns a pseudo-random code hash -func getACodeHash(i uint64) []byte { +// getCodeHash returns a pseudo-random code hash +func getCodeHash(i uint64) []byte { h := codehashes[int(i)%len(codehashes)] return common.CopyBytes(h[:]) } -// convenience function to lookup the code from the code hash -func getCode(hash common.Hash) []byte { +// getCodeByHash convenience function to lookup the code from the code hash +func getCodeByHash(hash common.Hash) []byte { if hash == emptyCode { return nil } @@ -1042,23 +1337,77 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { Nonce: i, Balance: big.NewInt(int64(i)), Root: emptyRoot, - CodeHash: getACodeHash(i), + CodeHash: getCodeHash(i), }) key := key32(i) - elem := &kv{key, value, false} + elem := &kv{key, value} accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } sort.Sort(entries) - // Push to disk layer accTrie.Commit(nil) return accTrie, entries } -// makeAccountTrieWithStorage spits out a trie, along with the leafs -func makeAccountTrieWithStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, - map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +// makeBoundaryAccountTrie constructs an account trie. Instead of filling +// accounts normally, this function will fill a few accounts which have +// boundary hash. +func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { + var ( + entries entrySlice + boundaries []common.Hash + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + trie, _ = trie.New(common.Hash{}, db) + ) + // Initialize boundaries + var next common.Hash + step := new(big.Int).Sub( + new(big.Int).Div( + new(big.Int).Exp(common.Big2, common.Big256, nil), + big.NewInt(accountConcurrency), + ), common.Big1, + ) + for i := 0; i < accountConcurrency; i++ { + last := common.BigToHash(new(big.Int).Add(next.Big(), step)) + if i == accountConcurrency-1 { + last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + } + boundaries = append(boundaries, last) + next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) + } + // Fill boundary accounts + for i := 0; i < len(boundaries); i++ { + value, _ := rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(int64(i)), + Root: emptyRoot, + CodeHash: getCodeHash(uint64(i)), + }) + elem := &kv{boundaries[i].Bytes(), value} + trie.Update(elem.k, elem.v) + entries = append(entries, elem) + } + // Fill other accounts if required + for i := uint64(1); i <= uint64(n); i++ { + value, _ := rlp.EncodeToBytes(state.Account{ + Nonce: i, + Balance: big.NewInt(int64(i)), + Root: emptyRoot, + CodeHash: getCodeHash(i), + }) + elem := &kv{key32(i), value} + trie.Update(elem.k, elem.v) + entries = append(entries, elem) + } + sort.Sort(entries) + trie.Commit(nil) + return trie, entries +} + +// makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts +// has a unique storage set. +func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie, _ = trie.New(common.Hash{}, db) @@ -1066,16 +1415,63 @@ func makeAccountTrieWithStorage(accounts, slots int, code bool) (*trie.Trie, ent storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) ) + // Create n accounts in the trie + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + codehash := emptyCode[:] + if code { + codehash = getCodeHash(i) + } + // Create a storage trie + stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) + stRoot := stTrie.Hash() + stTrie.Commit(nil) + value, _ := rlp.EncodeToBytes(state.Account{ + Nonce: i, + Balance: big.NewInt(int64(i)), + Root: stRoot, + CodeHash: codehash, + }) + elem := &kv{key, value} + accTrie.Update(elem.k, elem.v) + entries = append(entries, elem) + + storageTries[common.BytesToHash(key)] = stTrie + storageEntries[common.BytesToHash(key)] = stEntries + } + sort.Sort(entries) + + accTrie.Commit(nil) + return accTrie, entries, storageTries, storageEntries +} +// makeAccountTrieWithStorage spits out a trie, along with the leafs +func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { + var ( + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie, _ = trie.New(common.Hash{}, db) + entries entrySlice + storageTries = make(map[common.Hash]*trie.Trie) + storageEntries = make(map[common.Hash]entrySlice) + ) // Make a storage trie which we reuse for the whole lot - stTrie, stEntries := makeStorageTrie(slots, db) + var ( + stTrie *trie.Trie + stEntries entrySlice + ) + if boundary { + stTrie, stEntries = makeBoundaryStorageTrie(slots, db) + } else { + stTrie, stEntries = makeStorageTrieWithSeed(uint64(slots), 0, db) + } stRoot := stTrie.Hash() + // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) codehash := emptyCode[:] if code { - codehash = getACodeHash(i) + codehash = getCodeHash(i) } value, _ := rlp.EncodeToBytes(state.Account{ Nonce: i, @@ -1083,7 +1479,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code bool) (*trie.Trie, ent Root: stRoot, CodeHash: codehash, }) - elem := &kv{key, value, false} + elem := &kv{key, value} accTrie.Update(elem.k, elem.v) entries = append(entries, elem) // we reuse the same one for all accounts @@ -1096,23 +1492,116 @@ func makeAccountTrieWithStorage(accounts, slots int, code bool) (*trie.Trie, ent return accTrie, entries, storageTries, storageEntries } -// makeStorageTrie fills a storage trie with n items, returning the -// not-yet-committed trie and the sorted entries -func makeStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { +// makeStorageTrieWithSeed fills a storage trie with n items, returning the +// not-yet-committed trie and the sorted entries. The seeds can be used to ensure +// that tries are unique. +func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { trie, _ := trie.New(common.Hash{}, db) var entries entrySlice - for i := uint64(1); i <= uint64(n); i++ { - // store 'i' at slot 'i' - slotValue := key32(i) + for i := uint64(1); i <= n; i++ { + // store 'x' at slot 'x' + slotValue := key32(i + seed) rlpSlotValue, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(slotValue[:])) slotKey := key32(i) key := crypto.Keccak256Hash(slotKey[:]) - elem := &kv{key[:], rlpSlotValue, false} + elem := &kv{key[:], rlpSlotValue} + trie.Update(elem.k, elem.v) + entries = append(entries, elem) + } + sort.Sort(entries) + trie.Commit(nil) + return trie, entries +} + +// makeBoundaryStorageTrie constructs a storage trie. Instead of filling +// storage slots normally, this function will fill a few slots which have +// boundary hash. +func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { + var ( + entries entrySlice + boundaries []common.Hash + trie, _ = trie.New(common.Hash{}, db) + ) + // Initialize boundaries + var next common.Hash + step := new(big.Int).Sub( + new(big.Int).Div( + new(big.Int).Exp(common.Big2, common.Big256, nil), + big.NewInt(accountConcurrency), + ), common.Big1, + ) + for i := 0; i < accountConcurrency; i++ { + last := common.BigToHash(new(big.Int).Add(next.Big(), step)) + if i == accountConcurrency-1 { + last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + } + boundaries = append(boundaries, last) + next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) + } + // Fill boundary slots + for i := 0; i < len(boundaries); i++ { + key := boundaries[i] + val := []byte{0xde, 0xad, 0xbe, 0xef} + + elem := &kv{key[:], val} + trie.Update(elem.k, elem.v) + entries = append(entries, elem) + } + // Fill other slots if required + for i := uint64(1); i <= uint64(n); i++ { + slotKey := key32(i) + key := crypto.Keccak256Hash(slotKey[:]) + + slotValue := key32(i) + rlpSlotValue, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(slotValue[:])) + + elem := &kv{key[:], rlpSlotValue} trie.Update(elem.k, elem.v) entries = append(entries, elem) } sort.Sort(entries) + trie.Commit(nil) return trie, entries } + +func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { + t.Helper() + triedb := trie.NewDatabase(db) + accTrie, err := trie.New(root, triedb) + if err != nil { + t.Fatal(err) + } + accounts, slots := 0, 0 + accIt := trie.NewIterator(accTrie.NodeIterator(nil)) + for accIt.Next() { + var acc struct { + Nonce uint64 + Balance *big.Int + Root common.Hash + CodeHash []byte + } + if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil { + log.Crit("Invalid account encountered during snapshot creation", "err", err) + } + accounts++ + if acc.Root != emptyRoot { + storeTrie, err := trie.NewSecure(acc.Root, triedb) + if err != nil { + t.Fatal(err) + } + storeIt := trie.NewIterator(storeTrie.NodeIterator(nil)) + for storeIt.Next() { + slots++ + } + if err := storeIt.Err; err != nil { + t.Fatal(err) + } + } + } + if err := accIt.Err; err != nil { + t.Fatal(err) + } + t.Logf("accounts: %d, slots: %d", accounts, slots) +} From 0fda25e471aa0e061396050c3d5e59fbaaf1e7b0 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 25 Mar 2021 10:13:14 +0100 Subject: [PATCH 230/709] eth/tracers, core: use scopecontext in tracers, provide statedb in capturestart (#22333) Fixes the CaptureStart api to include the EVM, thus being able to set the statedb early on. This pr also exposes the struct we used internally in the interpreter to encapsulate the contract, mem, stack, rstack, so we pass it as a single struct to the tracer, and removes the error returns on the capture methods. --- core/vm/eips.go | 10 +- core/vm/evm.go | 6 +- core/vm/instructions.go | 424 ++++++++++++++++---------------- core/vm/instructions_test.go | 16 +- core/vm/interpreter.go | 24 +- core/vm/jump_table.go | 2 +- core/vm/logger.go | 44 ++-- core/vm/logger_json.go | 23 +- core/vm/logger_test.go | 16 +- core/vm/runtime/runtime_test.go | 19 +- eth/tracers/tracer.go | 113 ++++----- eth/tracers/tracer_test.go | 52 +++- 12 files changed, 384 insertions(+), 365 deletions(-) diff --git a/core/vm/eips.go b/core/vm/eips.go index 0c8bf1792e..6bb941d5f9 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -76,9 +76,9 @@ func enable1884(jt *JumpTable) { } } -func opSelfBalance(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - balance, _ := uint256.FromBig(interpreter.evm.StateDB.GetBalance(callContext.contract.Address())) - callContext.stack.push(balance) +func opSelfBalance(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + balance, _ := uint256.FromBig(interpreter.evm.StateDB.GetBalance(scope.Contract.Address())) + scope.Stack.push(balance) return nil, nil } @@ -95,9 +95,9 @@ func enable1344(jt *JumpTable) { } // opChainID implements CHAINID opcode -func opChainID(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opChainID(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { chainId, _ := uint256.FromBig(interpreter.evm.chainConfig.ChainID) - callContext.stack.push(chainId) + scope.Stack.push(chainId) return nil, nil } diff --git a/core/vm/evm.go b/core/vm/evm.go index 7346d76e5b..6fac50f721 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -239,7 +239,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas if !isPrecompile && evm.chainRules.IsEIP158 && value.Sign() == 0 { // Calling a non existing account, don't do anything, but ping the tracer if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) + evm.vmConfig.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil) } return nil, gas, nil @@ -250,7 +250,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // Capture the tracer start/end events in debug mode if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) + evm.vmConfig.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters evm.vmConfig.Tracer.CaptureEnd(ret, startGas-gas, time.Since(startTime), err) }(gas, time.Now()) @@ -472,7 +472,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), address, true, codeAndHash.code, gas, value) + evm.vmConfig.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value) } start := time.Now() diff --git a/core/vm/instructions.go b/core/vm/instructions.go index f4ca0603ed..3277674ee8 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -24,68 +24,68 @@ import ( "golang.org/x/crypto/sha3" ) -func opAdd(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opAdd(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Add(&x, y) return nil, nil } -func opSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opSub(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Sub(&x, y) return nil, nil } -func opMul(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opMul(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Mul(&x, y) return nil, nil } -func opDiv(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opDiv(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Div(&x, y) return nil, nil } -func opSdiv(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opSdiv(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.SDiv(&x, y) return nil, nil } -func opMod(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opMod(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Mod(&x, y) return nil, nil } -func opSmod(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opSmod(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.SMod(&x, y) return nil, nil } -func opExp(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - base, exponent := callContext.stack.pop(), callContext.stack.peek() +func opExp(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + base, exponent := scope.Stack.pop(), scope.Stack.peek() exponent.Exp(&base, exponent) return nil, nil } -func opSignExtend(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - back, num := callContext.stack.pop(), callContext.stack.peek() +func opSignExtend(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + back, num := scope.Stack.pop(), scope.Stack.peek() num.ExtendSign(num, &back) return nil, nil } -func opNot(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x := callContext.stack.peek() +func opNot(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x := scope.Stack.peek() x.Not(x) return nil, nil } -func opLt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opLt(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() if x.Lt(y) { y.SetOne() } else { @@ -94,8 +94,8 @@ func opLt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte return nil, nil } -func opGt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opGt(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() if x.Gt(y) { y.SetOne() } else { @@ -104,8 +104,8 @@ func opGt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte return nil, nil } -func opSlt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opSlt(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() if x.Slt(y) { y.SetOne() } else { @@ -114,8 +114,8 @@ func opSlt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byt return nil, nil } -func opSgt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opSgt(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() if x.Sgt(y) { y.SetOne() } else { @@ -124,8 +124,8 @@ func opSgt(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byt return nil, nil } -func opEq(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opEq(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() if x.Eq(y) { y.SetOne() } else { @@ -134,8 +134,8 @@ func opEq(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte return nil, nil } -func opIszero(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x := callContext.stack.peek() +func opIszero(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x := scope.Stack.peek() if x.IsZero() { x.SetOne() } else { @@ -144,32 +144,32 @@ func opIszero(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([] return nil, nil } -func opAnd(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opAnd(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.And(&x, y) return nil, nil } -func opOr(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opOr(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Or(&x, y) return nil, nil } -func opXor(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y := callContext.stack.pop(), callContext.stack.peek() +func opXor(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y := scope.Stack.pop(), scope.Stack.peek() y.Xor(&x, y) return nil, nil } -func opByte(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - th, val := callContext.stack.pop(), callContext.stack.peek() +func opByte(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + th, val := scope.Stack.pop(), scope.Stack.peek() val.Byte(&th) return nil, nil } -func opAddmod(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y, z := callContext.stack.pop(), callContext.stack.pop(), callContext.stack.peek() +func opAddmod(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y, z := scope.Stack.pop(), scope.Stack.pop(), scope.Stack.peek() if z.IsZero() { z.Clear() } else { @@ -178,8 +178,8 @@ func opAddmod(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([] return nil, nil } -func opMulmod(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x, y, z := callContext.stack.pop(), callContext.stack.pop(), callContext.stack.peek() +func opMulmod(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x, y, z := scope.Stack.pop(), scope.Stack.pop(), scope.Stack.peek() z.MulMod(&x, &y, z) return nil, nil } @@ -187,9 +187,9 @@ func opMulmod(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([] // opSHL implements Shift Left // The SHL instruction (shift left) pops 2 values from the stack, first arg1 and then arg2, // and pushes on the stack arg2 shifted to the left by arg1 number of bits. -func opSHL(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opSHL(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards - shift, value := callContext.stack.pop(), callContext.stack.peek() + shift, value := scope.Stack.pop(), scope.Stack.peek() if shift.LtUint64(256) { value.Lsh(value, uint(shift.Uint64())) } else { @@ -201,9 +201,9 @@ func opSHL(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byt // opSHR implements Logical Shift Right // The SHR instruction (logical shift right) pops 2 values from the stack, first arg1 and then arg2, // and pushes on the stack arg2 shifted to the right by arg1 number of bits with zero fill. -func opSHR(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opSHR(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards - shift, value := callContext.stack.pop(), callContext.stack.peek() + shift, value := scope.Stack.pop(), scope.Stack.peek() if shift.LtUint64(256) { value.Rsh(value, uint(shift.Uint64())) } else { @@ -215,8 +215,8 @@ func opSHR(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byt // opSAR implements Arithmetic Shift Right // The SAR instruction (arithmetic shift right) pops 2 values from the stack, first arg1 and then arg2, // and pushes on the stack arg2 shifted to the right by arg1 number of bits with sign extension. -func opSAR(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - shift, value := callContext.stack.pop(), callContext.stack.peek() +func opSAR(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + shift, value := scope.Stack.pop(), scope.Stack.peek() if shift.GtUint64(256) { if value.Sign() >= 0 { value.Clear() @@ -231,9 +231,9 @@ func opSAR(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byt return nil, nil } -func opSha3(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - offset, size := callContext.stack.pop(), callContext.stack.peek() - data := callContext.memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) +func opSha3(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + offset, size := scope.Stack.pop(), scope.Stack.peek() + data := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) if interpreter.hasher == nil { interpreter.hasher = sha3.NewLegacyKeccak256().(keccakState) @@ -251,37 +251,37 @@ func opSha3(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]by size.SetBytes(interpreter.hasherBuf[:]) return nil, nil } -func opAddress(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetBytes(callContext.contract.Address().Bytes())) +func opAddress(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Address().Bytes())) return nil, nil } -func opBalance(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - slot := callContext.stack.peek() +func opBalance(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + slot := scope.Stack.peek() address := common.Address(slot.Bytes20()) slot.SetFromBig(interpreter.evm.StateDB.GetBalance(address)) return nil, nil } -func opOrigin(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetBytes(interpreter.evm.Origin.Bytes())) +func opOrigin(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetBytes(interpreter.evm.Origin.Bytes())) return nil, nil } -func opCaller(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetBytes(callContext.contract.Caller().Bytes())) +func opCaller(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Caller().Bytes())) return nil, nil } -func opCallValue(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - v, _ := uint256.FromBig(callContext.contract.value) - callContext.stack.push(v) +func opCallValue(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + v, _ := uint256.FromBig(scope.Contract.value) + scope.Stack.push(v) return nil, nil } -func opCallDataLoad(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - x := callContext.stack.peek() +func opCallDataLoad(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + x := scope.Stack.peek() if offset, overflow := x.Uint64WithOverflow(); !overflow { - data := getData(callContext.contract.Input, offset, 32) + data := getData(scope.Contract.Input, offset, 32) x.SetBytes(data) } else { x.Clear() @@ -289,16 +289,16 @@ func opCallDataLoad(pc *uint64, interpreter *EVMInterpreter, callContext *callCt return nil, nil } -func opCallDataSize(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetUint64(uint64(len(callContext.contract.Input)))) +func opCallDataSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(scope.Contract.Input)))) return nil, nil } -func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - memOffset = callContext.stack.pop() - dataOffset = callContext.stack.pop() - length = callContext.stack.pop() + memOffset = scope.Stack.pop() + dataOffset = scope.Stack.pop() + length = scope.Stack.pop() ) dataOffset64, overflow := dataOffset.Uint64WithOverflow() if overflow { @@ -307,21 +307,21 @@ func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCt // These values are checked for overflow during gas cost calculation memOffset64 := memOffset.Uint64() length64 := length.Uint64() - callContext.memory.Set(memOffset64, length64, getData(callContext.contract.Input, dataOffset64, length64)) + scope.Memory.Set(memOffset64, length64, getData(scope.Contract.Input, dataOffset64, length64)) return nil, nil } -func opReturnDataSize(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetUint64(uint64(len(interpreter.returnData)))) +func opReturnDataSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(interpreter.returnData)))) return nil, nil } -func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - memOffset = callContext.stack.pop() - dataOffset = callContext.stack.pop() - length = callContext.stack.pop() + memOffset = scope.Stack.pop() + dataOffset = scope.Stack.pop() + length = scope.Stack.pop() ) offset64, overflow := dataOffset.Uint64WithOverflow() @@ -335,42 +335,42 @@ func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, callContext *call if overflow || uint64(len(interpreter.returnData)) < end64 { return nil, ErrReturnDataOutOfBounds } - callContext.memory.Set(memOffset.Uint64(), length.Uint64(), interpreter.returnData[offset64:end64]) + scope.Memory.Set(memOffset.Uint64(), length.Uint64(), interpreter.returnData[offset64:end64]) return nil, nil } -func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - slot := callContext.stack.peek() +func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + slot := scope.Stack.peek() slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(slot.Bytes20()))) return nil, nil } -func opCodeSize(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { l := new(uint256.Int) - l.SetUint64(uint64(len(callContext.contract.Code))) - callContext.stack.push(l) + l.SetUint64(uint64(len(scope.Contract.Code))) + scope.Stack.push(l) return nil, nil } -func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - memOffset = callContext.stack.pop() - codeOffset = callContext.stack.pop() - length = callContext.stack.pop() + memOffset = scope.Stack.pop() + codeOffset = scope.Stack.pop() + length = scope.Stack.pop() ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { uint64CodeOffset = 0xffffffffffffffff } - codeCopy := getData(callContext.contract.Code, uint64CodeOffset, length.Uint64()) - callContext.memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) + codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64()) + scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) return nil, nil } -func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - stack = callContext.stack + stack = scope.Stack a = stack.pop() memOffset = stack.pop() codeOffset = stack.pop() @@ -382,7 +382,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx } addr := common.Address(a.Bytes20()) codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64()) - callContext.memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) + scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) return nil, nil } @@ -413,8 +413,8 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx // // (6) Caller tries to get the code hash for an account which is marked as deleted, // this account should be regarded as a non-existent account and zero should be returned. -func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - slot := callContext.stack.peek() +func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + slot := scope.Stack.peek() address := common.Address(slot.Bytes20()) if interpreter.evm.StateDB.Empty(address) { slot.Clear() @@ -424,14 +424,14 @@ func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx return nil, nil } -func opGasprice(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opGasprice(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { v, _ := uint256.FromBig(interpreter.evm.GasPrice) - callContext.stack.push(v) + scope.Stack.push(v) return nil, nil } -func opBlockhash(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - num := callContext.stack.peek() +func opBlockhash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + num := scope.Stack.peek() num64, overflow := num.Uint64WithOverflow() if overflow { num.Clear() @@ -452,88 +452,88 @@ func opBlockhash(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) return nil, nil } -func opCoinbase(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetBytes(interpreter.evm.Context.Coinbase.Bytes())) +func opCoinbase(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetBytes(interpreter.evm.Context.Coinbase.Bytes())) return nil, nil } -func opTimestamp(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opTimestamp(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { v, _ := uint256.FromBig(interpreter.evm.Context.Time) - callContext.stack.push(v) + scope.Stack.push(v) return nil, nil } -func opNumber(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opNumber(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { v, _ := uint256.FromBig(interpreter.evm.Context.BlockNumber) - callContext.stack.push(v) + scope.Stack.push(v) return nil, nil } -func opDifficulty(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opDifficulty(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { v, _ := uint256.FromBig(interpreter.evm.Context.Difficulty) - callContext.stack.push(v) + scope.Stack.push(v) return nil, nil } -func opGasLimit(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetUint64(interpreter.evm.Context.GasLimit)) +func opGasLimit(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetUint64(interpreter.evm.Context.GasLimit)) return nil, nil } -func opPop(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.pop() +func opPop(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.pop() return nil, nil } -func opMload(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - v := callContext.stack.peek() +func opMload(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + v := scope.Stack.peek() offset := int64(v.Uint64()) - v.SetBytes(callContext.memory.GetPtr(offset, 32)) + v.SetBytes(scope.Memory.GetPtr(offset, 32)) return nil, nil } -func opMstore(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opMstore(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { // pop value of the stack - mStart, val := callContext.stack.pop(), callContext.stack.pop() - callContext.memory.Set32(mStart.Uint64(), &val) + mStart, val := scope.Stack.pop(), scope.Stack.pop() + scope.Memory.Set32(mStart.Uint64(), &val) return nil, nil } -func opMstore8(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - off, val := callContext.stack.pop(), callContext.stack.pop() - callContext.memory.store[off.Uint64()] = byte(val.Uint64()) +func opMstore8(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + off, val := scope.Stack.pop(), scope.Stack.pop() + scope.Memory.store[off.Uint64()] = byte(val.Uint64()) return nil, nil } -func opSload(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - loc := callContext.stack.peek() +func opSload(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + loc := scope.Stack.peek() hash := common.Hash(loc.Bytes32()) - val := interpreter.evm.StateDB.GetState(callContext.contract.Address(), hash) + val := interpreter.evm.StateDB.GetState(scope.Contract.Address(), hash) loc.SetBytes(val.Bytes()) return nil, nil } -func opSstore(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - loc := callContext.stack.pop() - val := callContext.stack.pop() - interpreter.evm.StateDB.SetState(callContext.contract.Address(), +func opSstore(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + loc := scope.Stack.pop() + val := scope.Stack.pop() + interpreter.evm.StateDB.SetState(scope.Contract.Address(), loc.Bytes32(), val.Bytes32()) return nil, nil } -func opJump(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - pos := callContext.stack.pop() - if !callContext.contract.validJumpdest(&pos) { +func opJump(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + pos := scope.Stack.pop() + if !scope.Contract.validJumpdest(&pos) { return nil, ErrInvalidJump } *pc = pos.Uint64() return nil, nil } -func opJumpi(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - pos, cond := callContext.stack.pop(), callContext.stack.pop() +func opJumpi(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + pos, cond := scope.Stack.pop(), scope.Stack.pop() if !cond.IsZero() { - if !callContext.contract.validJumpdest(&pos) { + if !scope.Contract.validJumpdest(&pos) { return nil, ErrInvalidJump } *pc = pos.Uint64() @@ -543,31 +543,31 @@ func opJumpi(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]b return nil, nil } -func opJumpdest(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opJumpdest(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { return nil, nil } -func opPc(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetUint64(*pc)) +func opPc(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetUint64(*pc)) return nil, nil } -func opMsize(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetUint64(uint64(callContext.memory.Len()))) +func opMsize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetUint64(uint64(scope.Memory.Len()))) return nil, nil } -func opGas(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.push(new(uint256.Int).SetUint64(callContext.contract.Gas)) +func opGas(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(new(uint256.Int).SetUint64(scope.Contract.Gas)) return nil, nil } -func opCreate(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - value = callContext.stack.pop() - offset, size = callContext.stack.pop(), callContext.stack.pop() - input = callContext.memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) - gas = callContext.contract.Gas + value = scope.Stack.pop() + offset, size = scope.Stack.pop(), scope.Stack.pop() + input = scope.Memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) + gas = scope.Contract.Gas ) if interpreter.evm.chainRules.IsEIP150 { gas -= gas / 64 @@ -575,14 +575,14 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([] // reuse size int for stackvalue stackvalue := size - callContext.contract.UseGas(gas) + scope.Contract.UseGas(gas) //TODO: use uint256.Int instead of converting with toBig() var bigVal = big0 if !value.IsZero() { bigVal = value.ToBig() } - res, addr, returnGas, suberr := interpreter.evm.Create(callContext.contract, input, gas, bigVal) + res, addr, returnGas, suberr := interpreter.evm.Create(scope.Contract, input, gas, bigVal) // Push item on the stack based on the returned error. If the ruleset is // homestead we must check for CodeStoreOutOfGasError (homestead only // rule) and treat as an error, if the ruleset is frontier we must @@ -594,8 +594,8 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([] } else { stackvalue.SetBytes(addr.Bytes()) } - callContext.stack.push(&stackvalue) - callContext.contract.Gas += returnGas + scope.Stack.push(&stackvalue) + scope.Contract.Gas += returnGas if suberr == ErrExecutionReverted { return res, nil @@ -603,18 +603,18 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([] return nil, nil } -func opCreate2(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opCreate2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - endowment = callContext.stack.pop() - offset, size = callContext.stack.pop(), callContext.stack.pop() - salt = callContext.stack.pop() - input = callContext.memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) - gas = callContext.contract.Gas + endowment = scope.Stack.pop() + offset, size = scope.Stack.pop(), scope.Stack.pop() + salt = scope.Stack.pop() + input = scope.Memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) + gas = scope.Contract.Gas ) // Apply EIP150 gas -= gas / 64 - callContext.contract.UseGas(gas) + scope.Contract.UseGas(gas) // reuse size int for stackvalue stackvalue := size //TODO: use uint256.Int instead of converting with toBig() @@ -622,7 +622,7 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([ if !endowment.IsZero() { bigEndowment = endowment.ToBig() } - res, addr, returnGas, suberr := interpreter.evm.Create2(callContext.contract, input, gas, + res, addr, returnGas, suberr := interpreter.evm.Create2(scope.Contract, input, gas, bigEndowment, &salt) // Push item on the stack based on the returned error. if suberr != nil { @@ -630,8 +630,8 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([ } else { stackvalue.SetBytes(addr.Bytes()) } - callContext.stack.push(&stackvalue) - callContext.contract.Gas += returnGas + scope.Stack.push(&stackvalue) + scope.Contract.Gas += returnGas if suberr == ErrExecutionReverted { return res, nil @@ -639,8 +639,8 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([ return nil, nil } -func opCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - stack := callContext.stack +func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + stack := scope.Stack // Pop gas. The actual gas in interpreter.evm.callGasTemp. // We can use this as a temporary value temp := stack.pop() @@ -649,7 +649,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]by addr, value, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get the arguments from the memory. - args := callContext.memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) var bigVal = big0 //TODO: use uint256.Int instead of converting with toBig() @@ -660,7 +660,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]by bigVal = value.ToBig() } - ret, returnGas, err := interpreter.evm.Call(callContext.contract, toAddr, args, gas, bigVal) + ret, returnGas, err := interpreter.evm.Call(scope.Contract, toAddr, args, gas, bigVal) if err != nil { temp.Clear() @@ -669,16 +669,16 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]by } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - callContext.memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - callContext.contract.Gas += returnGas + scope.Contract.Gas += returnGas return ret, nil } -func opCallCode(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { // Pop gas. The actual gas is in interpreter.evm.callGasTemp. - stack := callContext.stack + stack := scope.Stack // We use it as a temporary value temp := stack.pop() gas := interpreter.evm.callGasTemp @@ -686,7 +686,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ( addr, value, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get arguments from the memory. - args := callContext.memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) //TODO: use uint256.Int instead of converting with toBig() var bigVal = big0 @@ -695,7 +695,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ( bigVal = value.ToBig() } - ret, returnGas, err := interpreter.evm.CallCode(callContext.contract, toAddr, args, gas, bigVal) + ret, returnGas, err := interpreter.evm.CallCode(scope.Contract, toAddr, args, gas, bigVal) if err != nil { temp.Clear() } else { @@ -703,15 +703,15 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ( } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - callContext.memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - callContext.contract.Gas += returnGas + scope.Contract.Gas += returnGas return ret, nil } -func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - stack := callContext.stack +func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + stack := scope.Stack // Pop gas. The actual gas is in interpreter.evm.callGasTemp. // We use it as a temporary value temp := stack.pop() @@ -720,9 +720,9 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCt addr, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get arguments from the memory. - args := callContext.memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) - ret, returnGas, err := interpreter.evm.DelegateCall(callContext.contract, toAddr, args, gas) + ret, returnGas, err := interpreter.evm.DelegateCall(scope.Contract, toAddr, args, gas) if err != nil { temp.Clear() } else { @@ -730,16 +730,16 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCt } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - callContext.memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - callContext.contract.Gas += returnGas + scope.Contract.Gas += returnGas return ret, nil } -func opStaticCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { // Pop gas. The actual gas is in interpreter.evm.callGasTemp. - stack := callContext.stack + stack := scope.Stack // We use it as a temporary value temp := stack.pop() gas := interpreter.evm.callGasTemp @@ -747,9 +747,9 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) addr, inOffset, inSize, retOffset, retSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.Address(addr.Bytes20()) // Get arguments from the memory. - args := callContext.memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) + args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) - ret, returnGas, err := interpreter.evm.StaticCall(callContext.contract, toAddr, args, gas) + ret, returnGas, err := interpreter.evm.StaticCall(scope.Contract, toAddr, args, gas) if err != nil { temp.Clear() } else { @@ -757,36 +757,36 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - callContext.memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } - callContext.contract.Gas += returnGas + scope.Contract.Gas += returnGas return ret, nil } -func opReturn(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - offset, size := callContext.stack.pop(), callContext.stack.pop() - ret := callContext.memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) +func opReturn(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + offset, size := scope.Stack.pop(), scope.Stack.pop() + ret := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) return ret, nil } -func opRevert(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - offset, size := callContext.stack.pop(), callContext.stack.pop() - ret := callContext.memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) +func opRevert(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + offset, size := scope.Stack.pop(), scope.Stack.pop() + ret := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) return ret, nil } -func opStop(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opStop(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { return nil, nil } -func opSuicide(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - beneficiary := callContext.stack.pop() - balance := interpreter.evm.StateDB.GetBalance(callContext.contract.Address()) +func opSuicide(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + beneficiary := scope.Stack.pop() + balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) - interpreter.evm.StateDB.Suicide(callContext.contract.Address()) + interpreter.evm.StateDB.Suicide(scope.Contract.Address()) return nil, nil } @@ -794,18 +794,18 @@ func opSuicide(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([ // make log instruction function func makeLog(size int) executionFunc { - return func(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { + return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { topics := make([]common.Hash, size) - stack := callContext.stack + stack := scope.Stack mStart, mSize := stack.pop(), stack.pop() for i := 0; i < size; i++ { addr := stack.pop() topics[i] = addr.Bytes32() } - d := callContext.memory.GetCopy(int64(mStart.Uint64()), int64(mSize.Uint64())) + d := scope.Memory.GetCopy(int64(mStart.Uint64()), int64(mSize.Uint64())) interpreter.evm.StateDB.AddLog(&types.Log{ - Address: callContext.contract.Address(), + Address: scope.Contract.Address(), Topics: topics, Data: d, // This is a non-consensus field, but assigned here because @@ -818,24 +818,24 @@ func makeLog(size int) executionFunc { } // opPush1 is a specialized version of pushN -func opPush1(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { +func opPush1(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { var ( - codeLen = uint64(len(callContext.contract.Code)) + codeLen = uint64(len(scope.Contract.Code)) integer = new(uint256.Int) ) *pc += 1 if *pc < codeLen { - callContext.stack.push(integer.SetUint64(uint64(callContext.contract.Code[*pc]))) + scope.Stack.push(integer.SetUint64(uint64(scope.Contract.Code[*pc]))) } else { - callContext.stack.push(integer.Clear()) + scope.Stack.push(integer.Clear()) } return nil, nil } // make push instruction function func makePush(size uint64, pushByteSize int) executionFunc { - return func(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - codeLen := len(callContext.contract.Code) + return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + codeLen := len(scope.Contract.Code) startMin := codeLen if int(*pc+1) < startMin { @@ -848,8 +848,8 @@ func makePush(size uint64, pushByteSize int) executionFunc { } integer := new(uint256.Int) - callContext.stack.push(integer.SetBytes(common.RightPadBytes( - callContext.contract.Code[startMin:endMin], pushByteSize))) + scope.Stack.push(integer.SetBytes(common.RightPadBytes( + scope.Contract.Code[startMin:endMin], pushByteSize))) *pc += size return nil, nil @@ -858,8 +858,8 @@ func makePush(size uint64, pushByteSize int) executionFunc { // make dup instruction function func makeDup(size int64) executionFunc { - return func(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.dup(int(size)) + return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.dup(int(size)) return nil, nil } } @@ -868,8 +868,8 @@ func makeDup(size int64) executionFunc { func makeSwap(size int64) executionFunc { // switch n + 1 otherwise n would be swapped with n size++ - return func(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { - callContext.stack.swap(int(size)) + return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + scope.Stack.swap(int(size)) return nil, nil } } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 55d876581c..14f9e181f9 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -104,7 +104,7 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu expected := new(uint256.Int).SetBytes(common.Hex2Bytes(test.Expected)) stack.push(x) stack.push(y) - opFn(&pc, evmInterpreter, &callCtx{nil, stack, nil}) + opFn(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", name, len(stack.data)) } @@ -219,7 +219,7 @@ func TestAddMod(t *testing.T) { stack.push(z) stack.push(y) stack.push(x) - opAddmod(&pc, evmInterpreter, &callCtx{nil, stack, nil}) + opAddmod(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) actual := stack.pop() if actual.Cmp(expected) != 0 { t.Errorf("Testcase %d, expected %x, got %x", i, expected, actual) @@ -241,7 +241,7 @@ func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcas y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) stack.push(x) stack.push(y) - opFn(&pc, interpreter, &callCtx{nil, stack, nil}) + opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) actual := stack.pop() result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} } @@ -299,7 +299,7 @@ func opBenchmark(bench *testing.B, op executionFunc, args ...string) { a.SetBytes(arg) stack.push(a) } - op(&pc, evmInterpreter, &callCtx{nil, stack, nil}) + op(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) stack.pop() } } @@ -525,12 +525,12 @@ func TestOpMstore(t *testing.T) { pc := uint64(0) v := "abcdef00000000000000abba000000000deaf000000c0de00100000000133700" stack.pushN(*new(uint256.Int).SetBytes(common.Hex2Bytes(v)), *new(uint256.Int)) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) if got := common.Bytes2Hex(mem.GetCopy(0, 32)); got != v { t.Fatalf("Mstore fail, got %v, expected %v", got, v) } stack.pushN(*new(uint256.Int).SetUint64(0x1), *new(uint256.Int)) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) if common.Bytes2Hex(mem.GetCopy(0, 32)) != "0000000000000000000000000000000000000000000000000000000000000001" { t.Fatalf("Mstore failed to overwrite previous value") } @@ -553,7 +553,7 @@ func BenchmarkOpMstore(bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { stack.pushN(*value, *memStart) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) } } @@ -572,7 +572,7 @@ func BenchmarkOpSHA3(bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { stack.pushN(*uint256.NewInt().SetUint64(32), *start) - opSha3(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opSha3(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) } } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 06a3b962b2..3b67ad6dea 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -62,12 +62,12 @@ type Interpreter interface { CanRun([]byte) bool } -// callCtx contains the things that are per-call, such as stack and memory, +// ScopeContext contains the things that are per-call, such as stack and memory, // but not transients like pc and gas -type callCtx struct { - memory *Memory - stack *Stack - contract *Contract +type ScopeContext struct { + Memory *Memory + Stack *Stack + Contract *Contract } // keccakState wraps sha3.state. In addition to the usual hash methods, it also supports @@ -163,10 +163,10 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( op OpCode // current opcode mem = NewMemory() // bound memory stack = newstack() // local stack - callContext = &callCtx{ - memory: mem, - stack: stack, - contract: contract, + callContext = &ScopeContext{ + Memory: mem, + Stack: stack, + Contract: contract, } // For optimisation reason we're using uint64 as the program counter. // It's theoretically possible to go above 2^64. The YP defines the PC @@ -191,9 +191,9 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer func() { if err != nil { if !logged { - in.cfg.Tracer.CaptureState(in.evm, pcCopy, op, gasCopy, cost, mem, stack, in.returnData, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pcCopy, op, gasCopy, cost, callContext, in.returnData, in.evm.depth, err) } else { - in.cfg.Tracer.CaptureFault(in.evm, pcCopy, op, gasCopy, cost, mem, stack, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureFault(in.evm, pcCopy, op, gasCopy, cost, callContext, in.evm.depth, err) } } }() @@ -275,7 +275,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } if in.cfg.Debug { - in.cfg.Tracer.CaptureState(in.evm, pc, op, gasCopy, cost, mem, stack, in.returnData, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pc, op, gasCopy, cost, callContext, in.returnData, in.evm.depth, err) logged = true } diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 7b61762456..bb1800ea91 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -21,7 +21,7 @@ import ( ) type ( - executionFunc func(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) + executionFunc func(pc *uint64, interpreter *EVMInterpreter, callContext *ScopeContext) ([]byte, error) gasFunc func(*EVM, *Contract, *Stack, *Memory, uint64) (uint64, error) // last parameter is the requested memory size as a uint64 // memorySizeFunc returns the required size, and whether the operation overflowed a uint64 memorySizeFunc func(*Stack) (size uint64, overflow bool) diff --git a/core/vm/logger.go b/core/vm/logger.go index 41ce00ed01..9ccaafc772 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -18,7 +18,6 @@ package vm import ( "encoding/hex" - "errors" "fmt" "io" "math/big" @@ -32,8 +31,6 @@ import ( "github.com/ethereum/go-ethereum/params" ) -var errTraceLimitReached = errors.New("the number of logs reached the specified limit") - // Storage represents a contract's storage. type Storage map[common.Hash]common.Hash @@ -107,10 +104,10 @@ func (s *StructLog) ErrorString() string { // Note that reference types are actual VM data structures; make copies // if you need to retain them beyond the current call. type Tracer interface { - CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error - CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error - CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error - CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error + CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) + CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) + CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error) + CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) } // StructLogger is an EVM state logger and implements Tracer. @@ -139,17 +136,19 @@ func NewStructLogger(cfg *LogConfig) *StructLogger { } // CaptureStart implements the Tracer interface to initialize the tracing operation. -func (l *StructLogger) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { - return nil +func (l *StructLogger) CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { } // CaptureState logs a new structured log message and pushes it out to the environment // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. -func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error { +func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) { + memory := scope.Memory + stack := scope.Stack + contract := scope.Contract // check if already accumulated the specified number of logs if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) { - return errTraceLimitReached + return } // Copy a snapshot of the current memory state to a new buffer var mem []byte @@ -199,17 +198,15 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui // create a new snapshot of the EVM. log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rdata, storage, depth, env.StateDB.GetRefund(), err} l.logs = append(l.logs, log) - return nil } // CaptureFault implements the Tracer interface to trace an execution fault // while running an opcode. -func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { - return nil +func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error) { } // CaptureEnd is called after the call finishes to finalize the tracing. -func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error { +func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) { l.output = output l.err = err if l.cfg.Debug { @@ -218,7 +215,6 @@ func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration fmt.Printf(" error: %v\n", err) } } - return nil } // StructLogs returns the captured log entries. @@ -292,7 +288,7 @@ func NewMarkdownLogger(cfg *LogConfig, writer io.Writer) *mdLogger { return l } -func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { +func (t *mdLogger) CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { if !create { fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), @@ -307,10 +303,11 @@ func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create b | Pc | Op | Cost | Stack | RStack | Refund | |-------|-------------|------|-----------|-----------|---------| `) - return nil } -func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error { +// CaptureState also tracks SLOAD/SSTORE ops to track storage change. +func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) { + stack := scope.Stack fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, op, cost) if !t.cfg.DisableStack { @@ -327,18 +324,13 @@ func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64 if err != nil { fmt.Fprintf(t.out, "Error: %v\n", err) } - return nil } -func (t *mdLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { - +func (t *mdLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error) { fmt.Fprintf(t.out, "\nError: at pc=%d, op=%v: %v\n", pc, op, err) - - return nil } -func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) error { +func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) { fmt.Fprintf(t.out, "\nOutput: `0x%x`\nConsumed gas: `%d`\nError: `%v`\n", output, gasUsed, err) - return nil } diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index a27c261ed8..e54be08596 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -41,12 +41,16 @@ func NewJSONLogger(cfg *LogConfig, writer io.Writer) *JSONLogger { return l } -func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { - return nil +func (l *JSONLogger) CaptureStart(env *EVM, from, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { } +func (l *JSONLogger) CaptureFault(*EVM, uint64, OpCode, uint64, uint64, *ScopeContext, int, error) {} + // CaptureState outputs state information on the logger. -func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rData []byte, contract *Contract, depth int, err error) error { +func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) { + memory := scope.Memory + stack := scope.Stack + log := StructLog{ Pc: pc, Op: op, @@ -72,16 +76,11 @@ func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint if !l.cfg.DisableReturnData { log.ReturnData = rData } - return l.encoder.Encode(log) -} - -// CaptureFault outputs state information on the logger. -func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { - return nil + l.encoder.Encode(log) } // CaptureEnd is triggered at end of execution. -func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error { +func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) { type endLog struct { Output string `json:"output"` GasUsed math.HexOrDecimal64 `json:"gasUsed"` @@ -89,7 +88,7 @@ func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, Err string `json:"error,omitempty"` } if err != nil { - return l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, err.Error()}) + l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, err.Error()}) } - return l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, ""}) + l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, ""}) } diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 5a5f42fd34..9c936af36a 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -53,16 +53,20 @@ func TestStoreCapture(t *testing.T) { var ( env = NewEVM(BlockContext{}, TxContext{}, &dummyStatedb{}, params.TestChainConfig, Config{}) logger = NewStructLogger(nil) - mem = NewMemory() - stack = newstack() contract = NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 0) + scope = &ScopeContext{ + Memory: NewMemory(), + Stack: newstack(), + Contract: contract, + } ) - stack.push(uint256.NewInt().SetUint64(1)) - stack.push(uint256.NewInt()) + scope.Stack.push(uint256.NewInt().SetUint64(1)) + scope.Stack.push(uint256.NewInt()) var index common.Hash - logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, nil, contract, 0, nil) + logger.CaptureState(env, 0, SSTORE, 0, 0, scope, nil, 0, nil) if len(logger.storage[contract.Address()]) == 0 { - t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.storage[contract.Address()])) + t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), + len(logger.storage[contract.Address()])) } exp := common.BigToHash(big.NewInt(1)) if logger.storage[contract.Address()][index] != exp { diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 6e0434c2ca..2692755324 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -326,23 +326,18 @@ type stepCounter struct { steps int } -func (s *stepCounter) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { - return nil +func (s *stepCounter) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { } -func (s *stepCounter) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rData []byte, contract *vm.Contract, depth int, err error) error { - s.steps++ - // Enable this for more output - //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) - return nil +func (s *stepCounter) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { } -func (s *stepCounter) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { - return nil -} +func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {} -func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error { - return nil +func (s *stepCounter) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { + s.steps++ + // Enable this for more output + //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) } // benchmarkNonModifyingCode benchmarks code, but if the code modifies the diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index 4c398edf34..ba65925373 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -287,8 +287,6 @@ func (cw *contractWrapper) pushObject(vm *duktape.Context) { // Tracer provides an implementation of Tracer that evaluates a Javascript // function for each VM execution step. type Tracer struct { - inited bool // Flag whether the context was already inited from the EVM - vm *duktape.Context // Javascript VM instance tracerObject int // Stack index of the tracer JavaScript object @@ -529,7 +527,7 @@ func wrapError(context string, err error) error { } // CaptureStart implements the Tracer interface to initialize the tracing operation. -func (jst *Tracer) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { +func (jst *Tracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { jst.ctx["type"] = "CALL" if create { jst.ctx["type"] = "CREATE" @@ -540,77 +538,67 @@ func (jst *Tracer) CaptureStart(from common.Address, to common.Address, create b jst.ctx["gas"] = gas jst.ctx["value"] = value - return nil + // Initialize the context + jst.ctx["block"] = env.Context.BlockNumber.Uint64() + jst.dbWrapper.db = env.StateDB + // Compute intrinsic gas + isHomestead := env.ChainConfig().IsHomestead(env.Context.BlockNumber) + isIstanbul := env.ChainConfig().IsIstanbul(env.Context.BlockNumber) + intrinsicGas, err := core.IntrinsicGas(input, nil, jst.ctx["type"] == "CREATE", isHomestead, isIstanbul) + if err != nil { + return + } + jst.ctx["intrinsicGas"] = intrinsicGas } // CaptureState implements the Tracer interface to trace a single step of VM execution. -func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rdata []byte, contract *vm.Contract, depth int, err error) error { - if jst.err == nil { - // Initialize the context if it wasn't done yet - if !jst.inited { - jst.ctx["block"] = env.Context.BlockNumber.Uint64() - // Compute intrinsic gas - isHomestead := env.ChainConfig().IsHomestead(env.Context.BlockNumber) - isIstanbul := env.ChainConfig().IsIstanbul(env.Context.BlockNumber) - var input []byte - if data, ok := jst.ctx["input"].([]byte); ok { - input = data - } - intrinsicGas, err := core.IntrinsicGas(input, nil, jst.ctx["type"] == "CREATE", isHomestead, isIstanbul) - if err != nil { - return err - } - jst.ctx["intrinsicGas"] = intrinsicGas - jst.inited = true - } - // If tracing was interrupted, set the error and stop - if atomic.LoadUint32(&jst.interrupt) > 0 { - jst.err = jst.reason - return nil - } - jst.opWrapper.op = op - jst.stackWrapper.stack = stack - jst.memoryWrapper.memory = memory - jst.contractWrapper.contract = contract - jst.dbWrapper.db = env.StateDB - - *jst.pcValue = uint(pc) - *jst.gasValue = uint(gas) - *jst.costValue = uint(cost) - *jst.depthValue = uint(depth) - *jst.refundValue = uint(env.StateDB.GetRefund()) - - jst.errorValue = nil - if err != nil { - jst.errorValue = new(string) - *jst.errorValue = err.Error() - } - _, err := jst.call("step", "log", "db") - if err != nil { - jst.err = wrapError("step", err) - } +func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { + if jst.err != nil { + return + } + // If tracing was interrupted, set the error and stop + if atomic.LoadUint32(&jst.interrupt) > 0 { + jst.err = jst.reason + return + } + jst.opWrapper.op = op + jst.stackWrapper.stack = scope.Stack + jst.memoryWrapper.memory = scope.Memory + jst.contractWrapper.contract = scope.Contract + + *jst.pcValue = uint(pc) + *jst.gasValue = uint(gas) + *jst.costValue = uint(cost) + *jst.depthValue = uint(depth) + *jst.refundValue = uint(env.StateDB.GetRefund()) + + jst.errorValue = nil + if err != nil { + jst.errorValue = new(string) + *jst.errorValue = err.Error() + } + + if _, err := jst.call("step", "log", "db"); err != nil { + jst.err = wrapError("step", err) } - return nil } // CaptureFault implements the Tracer interface to trace an execution fault -// while running an opcode. -func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { - if jst.err == nil { - // Apart from the error, everything matches the previous invocation - jst.errorValue = new(string) - *jst.errorValue = err.Error() +func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { + if jst.err != nil { + return + } + // Apart from the error, everything matches the previous invocation + jst.errorValue = new(string) + *jst.errorValue = err.Error() - _, err := jst.call("fault", "log", "db") - if err != nil { - jst.err = wrapError("fault", err) - } + if _, err := jst.call("fault", "log", "db"); err != nil { + jst.err = wrapError("fault", err) } - return nil } // CaptureEnd is called after the call finishes to finalize the tracing. -func (jst *Tracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error { +func (jst *Tracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) { jst.ctx["output"] = output jst.ctx["time"] = t.String() jst.ctx["gasUsed"] = gasUsed @@ -618,7 +606,6 @@ func (jst *Tracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, er if err != nil { jst.ctx["error"] = err.Error() } - return nil } // GetResult calls the Javascript 'result' function and returns its value, or any accumulated error diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index d96030385c..7cda2e5330 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -47,7 +47,8 @@ type dummyStatedb struct { state.StateDB } -func (*dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetBalance(addr common.Address) *big.Int { return new(big.Int) } type vmContext struct { blockCtx vm.BlockContext @@ -67,7 +68,7 @@ func runTrace(tracer *Tracer, vmctx *vmContext) (json.RawMessage, error) { contract := vm.NewContract(account{}, account{}, value, startGas) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} - tracer.CaptureStart(contract.Caller(), contract.Address(), false, []byte{}, startGas, value) + tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, value) ret, err := env.Interpreter().Run(contract, []byte{}, false) tracer.CaptureEnd(ret, startGas-contract.Gas, 1, err) if err != nil { @@ -150,14 +151,55 @@ func TestHaltBetweenSteps(t *testing.T) { t.Fatal(err) } env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) - contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) + scope := &vm.ScopeContext{ + Contract: vm.NewContract(&account{}, &account{}, big.NewInt(0), 0), + } - tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, scope, nil, 0, nil) timeout := errors.New("stahp") tracer.Stop(timeout) - tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, scope, nil, 0, nil) if _, err := tracer.GetResult(); err.Error() != timeout.Error() { t.Errorf("Expected timeout error, got %v", err) } } + +// TestNoStepExec tests a regular value transfer (no exec), and accessing the statedb +// in 'result' +func TestNoStepExec(t *testing.T) { + runEmptyTrace := func(tracer *Tracer, vmctx *vmContext) (json.RawMessage, error) { + env := vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) + startGas := uint64(10000) + contract := vm.NewContract(account{}, account{}, big.NewInt(0), startGas) + tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, big.NewInt(0)) + tracer.CaptureEnd(nil, startGas-contract.Gas, 1, nil) + return tracer.GetResult() + } + execTracer := func(code string) []byte { + t.Helper() + ctx := &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} + tracer, err := New(code, ctx.txCtx) + if err != nil { + t.Fatal(err) + } + ret, err := runEmptyTrace(tracer, ctx) + if err != nil { + t.Fatal(err) + } + return ret + } + for i, tt := range []struct { + code string + want string + }{ + { // tests that we don't panic on accessing the db methods + code: "{depths: [], step: function() {}, fault: function() {}, result: function(ctx, db){ return db.getBalance(ctx.to)} }", + want: `"0"`, + }, + } { + if have := execTracer(tt.code); tt.want != string(have) { + t.Errorf("testcase %d: expected return value to be %s got %s\n\tcode: %v", i, tt.want, string(have), tt.code) + } + } +} From 497448bf90226eb088a41d1d102ffbb6cb2d1d6f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 25 Mar 2021 11:50:14 +0100 Subject: [PATCH 231/709] core: fix condition on header verification --- core/headerchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/headerchain.go b/core/headerchain.go index dcd3644cd1..9aee660eba 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -323,7 +323,7 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) seals := make([]bool, len(chain)) if checkFreq != 0 { // In case of checkFreq == 0 all seals are left false. - for i := 0; i < len(seals)/checkFreq; i++ { + for i := 0; i <= len(seals)/checkFreq; i++ { index := i*checkFreq + hc.rand.Intn(checkFreq) if index >= len(seals) { index = len(seals) - 1 From bed74b38d985e48d1e8b4b8c132e97a8987e07ac Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 25 Mar 2021 12:32:32 +0100 Subject: [PATCH 232/709] cmd/devp2p: fix comparison of TXT record value (#22572) * cmd/devp2p: fix comparison of TXT record value The AWS API returns quoted DNS strings, so we must encode the new value before comparing it against the existing record content. * cmd/devp2p: add test * cmd/devp2p: fix typo and rename val -> newValue --- cmd/devp2p/dns_route53.go | 35 +++++++++++++++++----------------- cmd/devp2p/dns_route53_test.go | 24 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 010913060a..2b6a30e60f 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -131,7 +131,7 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { } } - // wait for all change batches to propagate + // Wait for all change batches to propagate. for _, change := range changesToCheck { log.Info(fmt.Sprintf("Waiting for change request %s", *change.ChangeInfo.Id)) wreq := &route53.GetChangeInput{Id: change.ChangeInfo.Id} @@ -196,24 +196,29 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e records = lrecords var changes []types.Change - for path, val := range records { + for path, newValue := range records { + prevRecords, exists := existing[path] + prevValue := strings.Join(prevRecords.values, "") + + // prevValue contains quoted strings, encode newValue to compare. + newValue = splitTXT(newValue) + + // Assign TTL. ttl := int64(rootTTL) if path != name { ttl = int64(treeNodeTTL) } - prevRecords, exists := existing[path] - prevValue := strings.Join(prevRecords.values, "") if !exists { // Entry is unknown, push a new one - log.Info(fmt.Sprintf("Creating %s = %q", path, val)) - changes = append(changes, newTXTChange("CREATE", path, ttl, splitTXT(val))) - } else if prevValue != val || prevRecords.ttl != ttl { + log.Info(fmt.Sprintf("Creating %s = %s", path, newValue)) + changes = append(changes, newTXTChange("CREATE", path, ttl, newValue)) + } else if prevValue != newValue || prevRecords.ttl != ttl { // Entry already exists, only change its content. - log.Info(fmt.Sprintf("Updating %s from %q to %q", path, prevValue, val)) - changes = append(changes, newTXTChange("UPSERT", path, ttl, splitTXT(val))) + log.Info(fmt.Sprintf("Updating %s from %s to %s", path, prevValue, newValue)) + changes = append(changes, newTXTChange("UPSERT", path, ttl, newValue)) } else { - log.Info(fmt.Sprintf("Skipping %s = %q", path, val)) + log.Debug(fmt.Sprintf("Skipping %s = %s", path, newValue)) } } @@ -288,21 +293,19 @@ func changeCount(ch types.Change) int { // collectRecords collects all TXT records below the given name. func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) { - log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID)) var req route53.ListResourceRecordSetsInput req.HostedZoneId = &c.zoneID existing := make(map[string]recordSet) - for { + for page := 0; ; page++ { + log.Info("Loading existing TXT records", "name", name, "zone", c.zoneID, "page", page) resp, err := c.api.ListResourceRecordSets(context.TODO(), &req) if err != nil { return existing, err } - for _, set := range resp.ResourceRecordSets { if !isSubdomain(*set.Name, name) || set.Type != types.RRTypeTxt { continue } - s := recordSet{ttl: *set.TTL} for _, rec := range set.ResourceRecords { s.values = append(s.values, *rec.Value) @@ -314,14 +317,12 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error if !resp.IsTruncated { break } - - // Set the cursor to the next batc. From the AWS docs: + // Set the cursor to the next batch. From the AWS docs: // // To display the next page of results, get the values of NextRecordName, // NextRecordType, and NextRecordIdentifier (if any) from the response. Then submit // another ListResourceRecordSets request, and specify those values for // StartRecordName, StartRecordType, and StartRecordIdentifier. - req.StartRecordIdentifier = resp.NextRecordIdentifier req.StartRecordName = resp.NextRecordName req.StartRecordType = resp.NextRecordType diff --git a/cmd/devp2p/dns_route53_test.go b/cmd/devp2p/dns_route53_test.go index 600c281a28..e6eb516e6b 100644 --- a/cmd/devp2p/dns_route53_test.go +++ b/cmd/devp2p/dns_route53_test.go @@ -162,5 +162,29 @@ func TestRoute53ChangeSort(t *testing.T) { } } +// This test checks that computeChanges compares the quoted value of the records correctly. +func TestRoute53NoChange(t *testing.T) { + // Existing record set. + testTree0 := map[string]recordSet{ + "n": {ttl: rootTTL, values: []string{ + `"enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA"`, + }}, + "2xs2367yhaxjfglzhvawlqd4zy.n": {ttl: treeNodeTTL, values: []string{ + `"enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA"`, + }}, + } + // New set. + testTree1 := map[string]string{ + "n": "enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA", + "2XS2367YHAXJFGLZHVAWLQD4ZY.n": "enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA", + } + + var client route53Client + changes := client.computeChanges("n", testTree1, testTree0) + if len(changes) > 0 { + t.Fatalf("wrong changes (got %d, want 0)", len(changes)) + } +} + func sp(s string) *string { return &s } func ip(i int64) *int64 { return &i } From 54c0d573d75ab9baa239db3f071d6cb4d1ec6aad Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 25 Mar 2021 15:37:51 +0100 Subject: [PATCH 233/709] eth: dump rpc gas cap and tx fee cap (#22574) --- eth/ethconfig/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 5d0eece067..6be767aaf4 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -187,11 +187,11 @@ type Config struct { EVMInterpreter string // RPCGasCap is the global gas cap for eth-call variants. - RPCGasCap uint64 `toml:",omitempty"` + RPCGasCap uint64 // RPCTxFeeCap is the global transaction fee(price * gaslimit) cap for // send-transction variants. The unit is ether. - RPCTxFeeCap float64 `toml:",omitempty"` + RPCTxFeeCap float64 // Checkpoint is a hardcoded checkpoint which can be nil. Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` From 6d7ff6acea32b19b7c23c411748925a24873de9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 26 Mar 2021 14:00:06 +0200 Subject: [PATCH 234/709] eth/protocols, metrics, p2p: add handler performance metrics --- eth/protocols/eth/handler.go | 10 +++++++++- eth/protocols/snap/handler.go | 10 ++++++++++ metrics/histogram.go | 9 +++++++++ p2p/metrics.go | 11 ++++++++++- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 64648ed419..de5a38dcce 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -241,7 +242,14 @@ func handleMessage(backend Backend, peer *Peer) error { } else if peer.Version() >= ETH66 { handlers = eth66 } - + // Track the emount of time it takes to serve the request and run the handler + if metrics.Enabled { + h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) + defer func(start time.Time) { + sampler := func() metrics.Sample { return metrics.NewExpDecaySample(1028, 0.015) } + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) + }(time.Now()) + } if handler := handlers[msg.Code]; handler != nil { return handler(backend, msg, peer) } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 37e84839ab..6622cd8718 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -19,12 +19,14 @@ package snap import ( "bytes" "fmt" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -128,6 +130,14 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() + // Track the emount of time it takes to serve the request and run the handler + if metrics.Enabled { + h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) + defer func(start time.Time) { + sampler := func() metrics.Sample { return metrics.NewExpDecaySample(1028, 0.015) } + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) + }(time.Now()) + } // Handle the message depending on its contents switch { case msg.Code == GetAccountRangeMsg: diff --git a/metrics/histogram.go b/metrics/histogram.go index 46f3bbd2f1..2c54ce8b40 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -26,6 +26,15 @@ func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { return r.GetOrRegister(name, func() Histogram { return NewHistogram(s) }).(Histogram) } +// GetOrRegisterHistogramLazy returns an existing Histogram or constructs and +// registers a new StandardHistogram. +func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() Histogram { return NewHistogram(s()) }).(Histogram) +} + // NewHistogram constructs a new StandardHistogram from a Sample. func NewHistogram(s Sample) Histogram { if !Enabled { diff --git a/p2p/metrics.go b/p2p/metrics.go index 44946473fa..be0d2f495e 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -25,8 +25,17 @@ import ( ) const ( + // ingressMeterName is the prefix of the per-packet inbound metrics. ingressMeterName = "p2p/ingress" - egressMeterName = "p2p/egress" + + // egressMeterName is the prefix of the per-packet outbound metrics. + egressMeterName = "p2p/egress" + + // HandleHistName is the prefix of the per-packet serving time histograms. + HandleHistName = "p2p/handle" + + // WaitHistName is the prefix of the per-packet (req only) waiting time histograms. + WaitHistName = "p2p/wait" ) var ( From 2550e46269b05aa89127b06c1e18d51ebadd9da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 26 Mar 2021 16:14:12 +0200 Subject: [PATCH 235/709] eth/protocols, metrics: use resetting histograms for rare packets --- eth/protocols/eth/handler.go | 6 +++++- eth/protocols/snap/handler.go | 6 +++++- metrics/resetting_sample.go | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 metrics/resetting_sample.go diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index de5a38dcce..0dc3de9898 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -246,7 +246,11 @@ func handleMessage(backend Backend, peer *Peer) error { if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) defer func(start time.Time) { - sampler := func() metrics.Sample { return metrics.NewExpDecaySample(1028, 0.015) } + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) + } metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) }(time.Now()) } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 6622cd8718..4169306c25 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -134,7 +134,11 @@ func handleMessage(backend Backend, peer *Peer) error { if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) defer func(start time.Time) { - sampler := func() metrics.Sample { return metrics.NewExpDecaySample(1028, 0.015) } + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) + } metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) }(time.Now()) } diff --git a/metrics/resetting_sample.go b/metrics/resetting_sample.go new file mode 100644 index 0000000000..43c1129cd0 --- /dev/null +++ b/metrics/resetting_sample.go @@ -0,0 +1,24 @@ +package metrics + +// ResettingSample converts an ordinary sample into one that resets whenever its +// snapshot is retrieved. This will break for multi-monitor systems, but when only +// a single metric is being pushed out, this ensure that low-frequency events don't +// skew th charts indefinitely. +func ResettingSample(sample Sample) Sample { + return &resettingSample{ + Sample: sample, + } +} + +// resettingSample is a simple wrapper around a sample that resets it upon the +// snapshot retrieval. +type resettingSample struct { + Sample +} + +// Snapshot returns a read-only copy of the sample with the original reset. +func (rs *resettingSample) Snapshot() Sample { + s := rs.Sample.Snapshot() + rs.Sample.Clear() + return s +} From 955727181be3d4f6d919440f9709e3bf736ff8a4 Mon Sep 17 00:00:00 2001 From: Zou Guangxian Date: Sat, 27 Mar 2021 01:06:25 +0800 Subject: [PATCH 236/709] eth: fix corner case in sync head determination (#21695) This avoids synchronisation failures when the local header is ahead of the local full block. --- eth/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/sync.go b/eth/sync.go index dc72e88388..4520ec6879 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -289,8 +289,8 @@ func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { } } // Nope, we're really full syncing - head := cs.handler.chain.CurrentHeader() - td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64()) + head := cs.handler.chain.CurrentBlock() + td := cs.handler.chain.GetTd(head.Hash(), head.NumberU64()) return downloader.FullSync, td } From cae6b5527e34befb05d21e2c3d635d82d8c32582 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 26 Mar 2021 18:30:10 +0100 Subject: [PATCH 237/709] cmd/geth, consensus/ethash: add support for --miner.notify.full flag (#22558) The PR implements the --miner.notify.full flag that enables full pending block notifications. When this flag is used, the block notifications sent to mining endpoints contain the complete block header JSON instead of a work package array. Co-authored-by: AlexSSD7 Co-authored-by: Martin Holst Swende --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 5 ++ consensus/ethash/algorithm_test.go | 8 ++- consensus/ethash/ethash.go | 28 +++++---- consensus/ethash/ethash_test.go | 9 ++- consensus/ethash/sealer.go | 11 +++- consensus/ethash/sealer_test.go | 94 ++++++++++++++++++++++++++++++ eth/backend.go | 6 +- eth/ethconfig/config.go | 30 +++++----- miner/miner.go | 17 +++--- 11 files changed, 171 insertions(+), 39 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 5559835499..556ad0dbf3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -152,6 +152,7 @@ var ( utils.GpoMaxGasPriceFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, + utils.MinerNotifyFullFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index daea0afc4e..d7f8fd7ab9 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -180,6 +180,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MiningEnabledFlag, utils.MinerThreadsFlag, utils.MinerNotifyFlag, + utils.MinerNotifyFullFlag, utils.MinerGasPriceFlag, utils.MinerGasTargetFlag, utils.MinerGasLimitFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 00b28bddf6..a602d5a35e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -427,6 +427,10 @@ var ( Name: "miner.notify", Usage: "Comma separated HTTP URL list to notify of new work packages", } + MinerNotifyFullFlag = cli.BoolFlag{ + Name: "miner.notify.full", + Usage: "Notify with pending block headers instead of work packages", + } MinerGasTargetFlag = cli.Uint64Flag{ Name: "miner.gastarget", Usage: "Target gas floor for mined blocks", @@ -1359,6 +1363,7 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") } + cfg.NotifyFull = ctx.GlobalBool(MinerNotifyFullFlag.Name) if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index 663687b81c..9cc9d535d4 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -726,10 +726,14 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { for i := 0; i < 3; i++ { pend.Add(1) - go func(idx int) { defer pend.Done() - ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false) + + config := Config{ + CacheDir: cachedir, + CachesOnDisk: 1, + } + ethash := New(config, nil, false) defer ethash.Close() if err := ethash.verifySeal(nil, block.Header(), false); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 1afdc9381a..d922be7773 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -48,7 +48,7 @@ var ( two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) // sharedEthash is a full instance that can be shared between multiple users. - sharedEthash = New(Config{"", 3, 0, false, "", 1, 0, false, ModeNormal, nil}, nil, false) + sharedEthash *Ethash // algorithmRevision is the data structure version used for file naming. algorithmRevision = 23 @@ -57,6 +57,15 @@ var ( dumpMagic = []uint32{0xbaddcafe, 0xfee1dead} ) +func init() { + sharedConfig := Config{ + PowMode: ModeNormal, + CachesInMem: 3, + DatasetsInMem: 1, + } + sharedEthash = New(sharedConfig, nil, false) +} + // isLittleEndian returns whether the local system is running in little or big // endian byte order. func isLittleEndian() bool { @@ -411,6 +420,10 @@ type Config struct { DatasetsLockMmap bool PowMode Mode + // When set, notifications sent by the remote sealer will + // be block header JSON objects instead of work package arrays. + NotifyFull bool + Log log.Logger `toml:"-"` } @@ -462,6 +475,9 @@ func New(config Config, notify []string, noverify bool) *Ethash { update: make(chan struct{}), hashrate: metrics.NewMeterForced(), } + if config.PowMode == ModeShared { + ethash.shared = sharedEthash + } ethash.remote = startRemoteSealer(ethash, notify, noverify) return ethash } @@ -469,15 +485,7 @@ func New(config Config, notify []string, noverify bool) *Ethash { // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. func NewTester(notify []string, noverify bool) *Ethash { - ethash := &Ethash{ - config: Config{PowMode: ModeTest, Log: log.Root()}, - caches: newlru("cache", 1, newCache), - datasets: newlru("dataset", 1, newDataset), - update: make(chan struct{}), - hashrate: metrics.NewMeterForced(), - } - ethash.remote = startRemoteSealer(ethash, notify, noverify) - return ethash + return New(Config{PowMode: ModeTest}, notify, noverify) } // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 2639707eb2..eb3850b3b0 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -62,7 +62,14 @@ func TestCacheFileEvict(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpdir) - e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false) + + config := Config{ + CachesInMem: 3, + CachesOnDisk: 10, + CacheDir: tmpdir, + PowMode: ModeTest, + } + e := New(config, nil, false) defer e.Close() workers := 8 diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 2053534028..1830e672b1 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -358,7 +358,16 @@ func (s *remoteSealer) makeWork(block *types.Block) { // new work to be processed. func (s *remoteSealer) notifyWork() { work := s.currentWork - blob, _ := json.Marshal(work) + + // Encode the JSON payload of the notification. When NotifyFull is set, + // this is the complete block header, otherwise it is a JSON array. + var blob []byte + if s.ethash.config.NotifyFull { + blob, _ = json.Marshal(s.currentBlock.Header()) + } else { + blob, _ = json.Marshal(work) + } + s.reqWG.Add(len(s.notifyURLs)) for _, url := range s.notifyURLs { go s.sendNotification(s.notifyCtx, url, blob, work) diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 20ed2a4184..c34e76aec2 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -22,6 +22,7 @@ import ( "math/big" "net/http" "net/http/httptest" + "strconv" "testing" "time" @@ -74,6 +75,50 @@ func TestRemoteNotify(t *testing.T) { } } +// Tests whether remote HTTP servers are correctly notified of new work. (Full pending block body / --miner.notify.full) +func TestRemoteNotifyFull(t *testing.T) { + // Start a simple web server to capture notifications. + sink := make(chan map[string]interface{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("failed to read miner notification: %v", err) + } + var work map[string]interface{} + if err := json.Unmarshal(blob, &work); err != nil { + t.Errorf("failed to unmarshal miner notification: %v", err) + } + sink <- work + })) + defer server.Close() + + // Create the custom ethash engine. + config := Config{ + PowMode: ModeTest, + NotifyFull: true, + Log: testlog.Logger(t, log.LvlWarn), + } + ethash := New(config, []string{server.URL}, false) + defer ethash.Close() + + // Stream a work task and ensure the notification bubbles out. + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) + + ethash.Seal(nil, block, nil, nil) + select { + case work := <-sink: + if want := "0x" + strconv.FormatUint(header.Number.Uint64(), 16); work["number"] != want { + t.Errorf("pending block number mismatch: have %v, want %v", work["number"], want) + } + if want := "0x" + header.Difficulty.Text(16); work["difficulty"] != want { + t.Errorf("pending block difficulty mismatch: have %s, want %s", work["difficulty"], want) + } + case <-time.After(3 * time.Second): + t.Fatalf("notification timed out") + } +} + // Tests that pushing work packages fast to the miner doesn't cause any data race // issues in the notifications. func TestRemoteMultiNotify(t *testing.T) { @@ -119,6 +164,55 @@ func TestRemoteMultiNotify(t *testing.T) { } } +// Tests that pushing work packages fast to the miner doesn't cause any data race +// issues in the notifications. Full pending block body / --miner.notify.full) +func TestRemoteMultiNotifyFull(t *testing.T) { + // Start a simple web server to capture notifications. + sink := make(chan map[string]interface{}, 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("failed to read miner notification: %v", err) + } + var work map[string]interface{} + if err := json.Unmarshal(blob, &work); err != nil { + t.Errorf("failed to unmarshal miner notification: %v", err) + } + sink <- work + })) + defer server.Close() + + // Create the custom ethash engine. + config := Config{ + PowMode: ModeTest, + NotifyFull: true, + Log: testlog.Logger(t, log.LvlWarn), + } + ethash := New(config, []string{server.URL}, false) + defer ethash.Close() + + // Provide a results reader. + // Otherwise the unread results will be logged asynchronously + // and this can happen after the test is finished, causing a panic. + results := make(chan *types.Block, cap(sink)) + + // Stream a lot of work task and ensure all the notifications bubble out. + for i := 0; i < cap(sink); i++ { + header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) + ethash.Seal(nil, block, results, nil) + } + + for i := 0; i < cap(sink); i++ { + select { + case <-sink: + <-results + case <-time.After(10 * time.Second): + t.Fatalf("notification %d timed out", i) + } + } +} + // Tests whether stale solutions are correctly processed. func TestStaleSubmission(t *testing.T) { ethash := NewTester(nil, true) diff --git a/eth/backend.go b/eth/backend.go index 6e45d27501..9cf8b85663 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -121,6 +121,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) + // Transfer mining-related config to the ethash config. + ethashConfig := config.Ethash + ethashConfig.NotifyFull = config.Miner.NotifyFull + // Assemble the Ethereum object chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) if err != nil { @@ -140,7 +144,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, config.Miner.Notify, config.Miner.Noverify, chainDb), + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 6be767aaf4..0c6eb0bdd7 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -213,25 +213,23 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co switch config.PowMode { case ethash.ModeFake: log.Warn("Ethash used in fake mode") - return ethash.NewFaker() case ethash.ModeTest: log.Warn("Ethash used in test mode") - return ethash.NewTester(nil, noverify) case ethash.ModeShared: log.Warn("Ethash used in shared mode") - return ethash.NewShared() - default: - engine := ethash.New(ethash.Config{ - CacheDir: stack.ResolvePath(config.CacheDir), - CachesInMem: config.CachesInMem, - CachesOnDisk: config.CachesOnDisk, - CachesLockMmap: config.CachesLockMmap, - DatasetDir: config.DatasetDir, - DatasetsInMem: config.DatasetsInMem, - DatasetsOnDisk: config.DatasetsOnDisk, - DatasetsLockMmap: config.DatasetsLockMmap, - }, notify, noverify) - engine.SetThreads(-1) // Disable CPU mining - return engine } + engine := ethash.New(ethash.Config{ + PowMode: config.PowMode, + CacheDir: stack.ResolvePath(config.CacheDir), + CachesInMem: config.CachesInMem, + CachesOnDisk: config.CachesOnDisk, + CachesLockMmap: config.CachesLockMmap, + DatasetDir: config.DatasetDir, + DatasetsInMem: config.DatasetsInMem, + DatasetsOnDisk: config.DatasetsOnDisk, + DatasetsLockMmap: config.DatasetsLockMmap, + NotifyFull: config.NotifyFull, + }, notify, noverify) + engine.SetThreads(-1) // Disable CPU mining + return engine } diff --git a/miner/miner.go b/miner/miner.go index 20169f5007..4d71e307a6 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -42,14 +42,15 @@ type Backend interface { // Config is the configuration parameters of mining. type Config struct { - Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) - Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages(only useful in ethash). - ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner - GasFloor uint64 // Target gas floor for mined blocks. - GasCeil uint64 // Target gas ceiling for mined blocks. - GasPrice *big.Int // Minimum gas price for mining a transaction - Recommit time.Duration // The time interval for miner to re-create mining work. - Noverify bool // Disable remote mining solution verification(only useful in ethash). + Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) + Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages (only useful in ethash). + NotifyFull bool `toml:",omitempty"` // Notify with pending block headers instead of work packages + ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner + GasFloor uint64 // Target gas floor for mined blocks. + GasCeil uint64 // Target gas ceiling for mined blocks. + GasPrice *big.Int // Minimum gas price for mining a transaction + Recommit time.Duration // The time interval for miner to re-create mining work. + Noverify bool // Disable remote mining solution verification(only useful in ethash). } // Miner creates blocks and searches for proof-of-work values. From 62379f02c61f459020a1fb8d7fdea4935a1d8701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 26 Mar 2021 21:13:52 +0200 Subject: [PATCH 238/709] metrics/influxdb: don't push empty histograms, no measurement != 0 --- metrics/influxdb/influxdb.go | 43 +++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index d4fc478e7b..52d0091034 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -162,26 +162,29 @@ func (r *reporter) send() error { }) case metrics.Histogram: ms := metric.Snapshot() - ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) - pts = append(pts, client.Point{ - Measurement: fmt.Sprintf("%s%s.histogram", namespace, name), - Tags: r.tags, - Fields: map[string]interface{}{ - "count": ms.Count(), - "max": ms.Max(), - "mean": ms.Mean(), - "min": ms.Min(), - "stddev": ms.StdDev(), - "variance": ms.Variance(), - "p50": ps[0], - "p75": ps[1], - "p95": ps[2], - "p99": ps[3], - "p999": ps[4], - "p9999": ps[5], - }, - Time: now, - }) + + if ms.Count() > 0 { + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.histogram", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + }, + Time: now, + }) + } case metrics.Meter: ms := metric.Snapshot() pts = append(pts, client.Point{ From 099be0410048c65f885b9112f09238523fe19b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 26 Mar 2021 22:29:22 +0200 Subject: [PATCH 239/709] eth/protocols/snap: add peer id and req id to the timeout logs --- eth/protocols/snap/sync.go | 40 ++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index fa2dc16c39..af581df07c 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -811,6 +811,8 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { if idle == "" { return } + peer := s.peers[idle] + // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 for { @@ -834,14 +836,14 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { task: task, } req.timeout = time.AfterFunc(requestTimeout, func() { - log.Debug("Account range request timed out") + peer.Log().Debug("Account range request timed out", "reqid", reqid) s.scheduleRevertAccountRequest(req) }) s.accountReqs[reqid] = req delete(s.accountIdlers, idle) s.pend.Add(1) - go func(peer SyncPeer, root common.Hash) { + go func(root common.Hash) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -849,7 +851,7 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { peer.Log().Debug("Failed to request account range", "err", err) s.scheduleRevertAccountRequest(req) } - }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists + }(s.root) // Inject the request into the task to block further assignments task.req = req @@ -891,6 +893,8 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { if idle == "" { return } + peer := s.peers[idle] + // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 for { @@ -921,14 +925,14 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { task: task, } req.timeout = time.AfterFunc(requestTimeout, func() { - log.Debug("Bytecode request timed out") + peer.Log().Debug("Bytecode request timed out", "reqid", reqid) s.scheduleRevertBytecodeRequest(req) }) s.bytecodeReqs[reqid] = req delete(s.bytecodeIdlers, idle) s.pend.Add(1) - go func(peer SyncPeer) { + go func() { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -936,7 +940,7 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { log.Debug("Failed to request bytecodes", "err", err) s.scheduleRevertBytecodeRequest(req) } - }(s.peers[idle]) // We're in the lock, peers[id] surely exists + }() } } @@ -976,6 +980,8 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { if idle == "" { return } + peer := s.peers[idle] + // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 for { @@ -1045,14 +1051,14 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { req.limit = subtask.Last } req.timeout = time.AfterFunc(requestTimeout, func() { - log.Debug("Storage request timed out") + peer.Log().Debug("Storage request timed out", "reqid", reqid) s.scheduleRevertStorageRequest(req) }) s.storageReqs[reqid] = req delete(s.storageIdlers, idle) s.pend.Add(1) - go func(peer SyncPeer, root common.Hash) { + go func(root common.Hash) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -1064,7 +1070,7 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { log.Debug("Failed to request storage", "err", err) s.scheduleRevertStorageRequest(req) } - }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists + }(s.root) // Inject the request into the subtask to block further assignments if subtask != nil { @@ -1121,6 +1127,8 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { if idle == "" { return } + peer := s.peers[idle] + // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 for { @@ -1160,14 +1168,14 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { task: s.healer, } req.timeout = time.AfterFunc(requestTimeout, func() { - log.Debug("Trienode heal request timed out") + peer.Log().Debug("Trienode heal request timed out", "reqid", reqid) s.scheduleRevertTrienodeHealRequest(req) }) s.trienodeHealReqs[reqid] = req delete(s.trienodeHealIdlers, idle) s.pend.Add(1) - go func(peer SyncPeer, root common.Hash) { + go func(root common.Hash) { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -1175,7 +1183,7 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { log.Debug("Failed to request trienode healers", "err", err) s.scheduleRevertTrienodeHealRequest(req) } - }(s.peers[idle], s.root) // We're in the lock, peers[id] surely exists + }(s.root) } } @@ -1227,6 +1235,8 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { if idle == "" { return } + peer := s.peers[idle] + // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 for { @@ -1258,14 +1268,14 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { task: s.healer, } req.timeout = time.AfterFunc(requestTimeout, func() { - log.Debug("Bytecode heal request timed out") + peer.Log().Debug("Bytecode heal request timed out", "reqid", reqid) s.scheduleRevertBytecodeHealRequest(req) }) s.bytecodeHealReqs[reqid] = req delete(s.bytecodeHealIdlers, idle) s.pend.Add(1) - go func(peer SyncPeer) { + go func() { defer s.pend.Done() // Attempt to send the remote request and revert if it fails @@ -1273,7 +1283,7 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { log.Debug("Failed to request bytecode healers", "err", err) s.scheduleRevertBytecodeHealRequest(req) } - }(s.peers[idle]) // We're in the lock, peers[id] surely exists + }() } } From 27056f62e57e55ae91cfac95d7af24d1f707fb99 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 26 Mar 2021 22:15:20 +0100 Subject: [PATCH 240/709] cmd/devp2p: update to newer cloudflare API client (#22588) This upgrades the cloudflare client dependency to v0.14.0. The new version changes the API because all methods now require a context parameter. This change also reduces the log level of the 'Skipping...' message to debug, following a similar change in the AWS deployer. --- cmd/devp2p/dns_cloudflare.go | 14 ++++++++------ go.mod | 8 ++++---- go.sum | 30 ++++++++++++++++++------------ 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/cmd/devp2p/dns_cloudflare.go b/cmd/devp2p/dns_cloudflare.go index a4d10dcfdd..596254df91 100644 --- a/cmd/devp2p/dns_cloudflare.go +++ b/cmd/devp2p/dns_cloudflare.go @@ -17,6 +17,7 @@ package main import ( + "context" "fmt" "strings" @@ -79,7 +80,7 @@ func (c *cloudflareClient) checkZone(name string) error { c.zoneID = id } log.Info(fmt.Sprintf("Checking Permissions on zone %s", c.zoneID)) - zone, err := c.ZoneDetails(c.zoneID) + zone, err := c.ZoneDetails(context.Background(), c.zoneID) if err != nil { return err } @@ -112,7 +113,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) records = lrecords log.Info(fmt.Sprintf("Retrieving existing TXT records on %s", name)) - entries, err := c.DNSRecords(c.zoneID, cloudflare.DNSRecord{Type: "TXT"}) + entries, err := c.DNSRecords(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT"}) if err != nil { return err } @@ -134,14 +135,15 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) if path != name { ttl = treeNodeTTL // Max TTL permitted by Cloudflare } - _, err = c.CreateDNSRecord(c.zoneID, cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl}) + record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl} + _, err = c.CreateDNSRecord(context.Background(), c.zoneID, record) } else if old.Content != val { // Entry already exists, only change its content. log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val)) old.Content = val - err = c.UpdateDNSRecord(c.zoneID, old.ID, old) + err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old) } else { - log.Info(fmt.Sprintf("Skipping %s = %q", path, val)) + log.Debug(fmt.Sprintf("Skipping %s = %q", path, val)) } if err != nil { return fmt.Errorf("failed to publish %s: %v", path, err) @@ -155,7 +157,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) } // Stale entry, nuke it. log.Info(fmt.Sprintf("Deleting %s = %q", path, entry.Content)) - if err := c.DeleteDNSRecord(c.zoneID, entry.ID); err != nil { + if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil { return fmt.Errorf("failed to delete %s: %v", path, err) } } diff --git a/go.mod b/go.mod index dc020f9a65..0b0e8bf373 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1 github.com/btcsuite/btcd v0.20.1-beta github.com/cespare/cp v0.1.0 - github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 + github.com/cloudflare/cloudflare-go v0.14.0 github.com/consensys/gurvy v0.3.8 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea @@ -40,7 +40,7 @@ require ( github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 - github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c + github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 @@ -52,8 +52,8 @@ require ( github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c - golang.org/x/text v0.3.3 - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/text v0.3.4 + golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 diff --git a/go.sum b/go.sum index 813aa8e27d..1b7212e420 100644 --- a/go.sum +++ b/go.sum @@ -60,7 +60,6 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1: github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.2.0 h1:BS+UYpbsElC82gB+2E2jiCBg36i8HlubTB/dO/moQ9c= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= @@ -108,8 +107,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/cloudflare/cloudflare-go v0.14.0 h1:gFqGlGl/5f9UGXAaKapCGUfaTCgRKKnzu2VvzMZlOFA= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/consensys/bavard v0.1.8-0.20210105233146-c16790d2aa8b/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/goff v0.3.10/go.mod h1:xTldOBEHmFiYS0gPXd3NsaEqZWlnmeWcRLWgD3ba3xc= github.com/consensys/gurvy v0.3.8 h1:H2hvjvT2OFMgdMn5ZbhXqHt+F8DJ2clZW7Vmc0kFFxc= @@ -209,7 +209,6 @@ github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= @@ -284,11 +283,8 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -339,8 +335,9 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -365,8 +362,9 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -390,8 +388,9 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssy github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -464,6 +463,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -537,8 +537,9 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d h1:1aflnvSoWWLI2k/dMUAl5lvU1YO4Mb4hz0gh+1rjcxU= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -578,17 +579,21 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -686,6 +691,7 @@ gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHO gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= From 76700ac892f6fe3d7865fbe2f724626c72a04227 Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 29 Mar 2021 17:09:29 +0800 Subject: [PATCH 241/709] core/state/pruner: move the compaction out of the pruning procedure (#22579) The main idea behind it is: the range compaction is very expensive which can take a few hours to finish. During this long procedure, a lot of exceptions can occur, e.g. - Geth is killed manually - Geth is killed because of machine crash - etc In order to minimize the effect of the exceptions, the compaction is moved out of the pruning. So that even the compaction is not finished, the pruning is regarded as done. --- core/state/pruner/pruner.go | 68 ++++++++++++------------------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 62cc7b0120..4d6e415511 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -113,7 +113,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint6 }, nil } -func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[common.Hash]struct{}, start time.Time) error { +func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error { // Delete all stale trie nodes in the disk. With the help of state bloom // the trie nodes(and codes) belong to the active state will be filtered // out. A very small part of stale tries will also be filtered because of @@ -186,6 +186,25 @@ func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[c iter.Release() log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + // Pruning is done, now drop the "useless" layers from the snapshot. + // Firstly, flushing the target layer into the disk. After that all + // diff layers below the target will all be merged into the disk. + if err := snaptree.Cap(root, 0); err != nil { + return err + } + // Secondly, flushing the snapshot journal into the disk. All diff + // layers upon are dropped silently. Eventually the entire snapshot + // tree is converted into a single disk layer with the pruning target + // as the root. + if _, err := snaptree.Journal(root); err != nil { + return err + } + // Delete the state bloom, it marks the entire pruning procedure is + // finished. If any crashes or manual exit happens before this, + // `RecoverPruning` will pick it up in the next restarts to redo all + // the things. + os.RemoveAll(bloomPath) + // Start compactions, will remove the deleted data from the disk immediately. // Note for small pruning, the compaction is skipped. if count >= rangeCompactionThreshold { @@ -314,29 +333,7 @@ func (p *Pruner) Prune(root common.Hash) error { return err } log.Info("State bloom filter committed", "name", filterName) - - if err := prune(p.db, p.stateBloom, middleRoots, start); err != nil { - return err - } - // Pruning is done, now drop the "useless" layers from the snapshot. - // Firstly, flushing the target layer into the disk. After that all - // diff layers below the target will all be merged into the disk. - if err := p.snaptree.Cap(root, 0); err != nil { - return err - } - // Secondly, flushing the snapshot journal into the disk. All diff - // layers upon the target layer are dropped silently. Eventually the - // entire snapshot tree is converted into a single disk layer with - // the pruning target as the root. - if _, err := p.snaptree.Journal(root); err != nil { - return err - } - // Delete the state bloom, it marks the entire pruning procedure is - // finished. If any crashes or manual exit happens before this, - // `RecoverPruning` will pick it up in the next restarts to redo all - // the things. - os.RemoveAll(filterName) - return nil + return prune(p.snaptree, root, p.db, p.stateBloom, filterName, middleRoots, start) } // RecoverPruning will resume the pruning procedure during the system restart. @@ -400,28 +397,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err log.Error("Pruning target state is not existent") return errors.New("non-existent target state") } - if err := prune(db, stateBloom, middleRoots, time.Now()); err != nil { - return err - } - // Pruning is done, now drop the "useless" layers from the snapshot. - // Firstly, flushing the target layer into the disk. After that all - // diff layers below the target will all be merged into the disk. - if err := snaptree.Cap(stateBloomRoot, 0); err != nil { - return err - } - // Secondly, flushing the snapshot journal into the disk. All diff - // layers upon are dropped silently. Eventually the entire snapshot - // tree is converted into a single disk layer with the pruning target - // as the root. - if _, err := snaptree.Journal(stateBloomRoot); err != nil { - return err - } - // Delete the state bloom, it marks the entire pruning procedure is - // finished. If any crashes or manual exit happens before this, - // `RecoverPruning` will pick it up in the next restarts to redo all - // the things. - os.RemoveAll(stateBloomPath) - return nil + return prune(snaptree, stateBloomRoot, db, stateBloom, stateBloomPath, middleRoots, time.Now()) } // extractGenesis loads the genesis state and commits all the state entries From 7644795950481432f6ba230421d41349b69774e8 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 29 Mar 2021 14:17:35 +0200 Subject: [PATCH 242/709] eth/protocols/snap: try to prevent requests timing out --- eth/protocols/snap/handler.go | 13 +++++++++---- eth/protocols/snap/sync.go | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 4169306c25..f7939964f3 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -50,6 +50,11 @@ const ( // maxTrieNodeLookups is the maximum number of state trie nodes to serve. This // number is there to limit the number of disk lookups. maxTrieNodeLookups = 1024 + + // maxTrieNodeTimeSpent is the maximum time we should spend on looking up trie nodes. + // If we spend too much time, then it's a fairly high chance of timing out + // at the remote side, which means all the work is in vain. + maxTrieNodeTimeSpent = 5 * time.Second ) // Handler is a callback to invoke from an outside runner after the boilerplate @@ -129,7 +134,7 @@ func handleMessage(backend Backend, peer *Peer) error { return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) } defer msg.Discard() - + start := time.Now() // Track the emount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) @@ -140,7 +145,7 @@ func handleMessage(backend Backend, peer *Peer) error { ) } metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) - }(time.Now()) + }(start) } // Handle the message depending on its contents switch { @@ -470,13 +475,13 @@ func handleMessage(backend Backend, peer *Peer) error { bytes += uint64(len(blob)) // Sanity check limits to avoid DoS on the store trie loads - if bytes > req.Bytes || loads > maxTrieNodeLookups { + if bytes > req.Bytes || loads > maxTrieNodeLookups || time.Since(start) > maxTrieNodeTimeSpent { break } } } // Abort request processing if we've exceeded our limits - if bytes > req.Bytes || loads > maxTrieNodeLookups { + if bytes > req.Bytes || loads > maxTrieNodeLookups || time.Since(start) > maxTrieNodeTimeSpent { break } } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index af581df07c..2924fa0802 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -85,7 +85,7 @@ const ( var ( // requestTimeout is the maximum time a peer is allowed to spend on serving // a single network request. - requestTimeout = 10 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? + requestTimeout = 15 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? ) // ErrCancelled is returned from snap syncing if the operation was prematurely From b6912c10476bf24c25b780b91a892a71fba268b6 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 29 Mar 2021 19:54:49 +0100 Subject: [PATCH 243/709] core: add BlockGen.GetBalance method (#22589) --- core/chain_makers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/chain_makers.go b/core/chain_makers.go index 33f253d9e8..e058e5a78e 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -111,6 +111,11 @@ func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) { b.receipts = append(b.receipts, receipt) } +// GetBalance returns the balance of the given address at the generated block. +func (b *BlockGen) GetBalance(addr common.Address) *big.Int { + return b.statedb.GetBalance(addr) +} + // AddUncheckedTx forcefully adds a transaction to the block without any // validation. // From 24588bacfd81bb991f9832f6557a34ae5078b06e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 29 Mar 2021 20:58:58 +0200 Subject: [PATCH 244/709] cmd/puppeth: specify working directory for nodejs 15 (#22549) --- cmd/puppeth/module_dashboard.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index be8b6ec600..a76ee19a06 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -518,6 +518,8 @@ var dashboardMascot = []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x01s\x var dashboardDockerfile = ` FROM mhart/alpine-node:latest +WORKDIR /usr/app + RUN \ npm install connect serve-static && \ \ From 59ac3c9fd30eb4134da60d0549cfd84b27a3a3f7 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 30 Mar 2021 13:57:21 +0200 Subject: [PATCH 245/709] cmd/geth: add db dumptrie command (#22563) Adds the command "geth db dumptrie ", to better help investigate the trie data --- cmd/geth/dbcmd.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 078cad53b4..db1fb0b801 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "time" "github.com/ethereum/go-ethereum/cmd/utils" @@ -29,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie" "gopkg.in/urfave/cli.v1" ) @@ -57,6 +59,7 @@ Remove blockchain and state databases`, dbGetCmd, dbDeleteCmd, dbPutCmd, + dbGetSlotsCmd, }, } dbInspectCmd = cli.Command{ @@ -158,6 +161,22 @@ WARNING: This is a low-level operation which may cause database corruption!`, Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } + dbGetSlotsCmd = cli.Command{ + Action: utils.MigrateFlags(dbDumpTrie), + Name: "dumptrie", + Usage: "Show the storage key/values of a given storage trie", + ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, + Description: "This command looks up the specified database key from the database.", + } ) func removeDB(ctx *cli.Context) error { @@ -380,3 +399,53 @@ func dbPut(ctx *cli.Context) error { } return db.Put(key, value) } + +// dbDumpTrie shows the key-value slots of a given storage trie +func dbDumpTrie(ctx *cli.Context) error { + if ctx.NArg() < 1 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + var ( + root []byte + start []byte + max = int64(-1) + err error + ) + if root, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { + log.Info("Could not decode the root", "error", err) + return err + } + stRoot := common.BytesToHash(root) + if ctx.NArg() >= 2 { + if start, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { + log.Info("Could not decode the seek position", "error", err) + return err + } + } + if ctx.NArg() >= 3 { + if max, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { + log.Info("Could not decode the max count", "error", err) + return err + } + } + theTrie, err := trie.New(stRoot, trie.NewDatabase(db)) + if err != nil { + return err + } + var count int64 + it := trie.NewIterator(theTrie.NodeIterator(start)) + for it.Next() { + if max > 0 && count == max { + fmt.Printf("Exiting after %d values\n", count) + break + } + fmt.Printf(" %d. key %#x: %#x\n", count, it.Key, it.Value) + count++ + } + return it.Err +} From 44fe466999ce7f8b02de7e83de955f8847342309 Mon Sep 17 00:00:00 2001 From: nebojsa94 Date: Tue, 30 Mar 2021 15:38:53 +0200 Subject: [PATCH 246/709] core/vm: fix Byzantium address list (#22603) --- core/vm/contracts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 4e99a51618..a3ceece0e9 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -118,7 +118,7 @@ func init() { PrecompiledAddressesHomestead = append(PrecompiledAddressesHomestead, k) } for k := range PrecompiledContractsByzantium { - PrecompiledAddressesHomestead = append(PrecompiledAddressesByzantium, k) + PrecompiledAddressesByzantium = append(PrecompiledAddressesByzantium, k) } for k := range PrecompiledContractsIstanbul { PrecompiledAddressesIstanbul = append(PrecompiledAddressesIstanbul, k) From 3faae5defc0066a95c234da84462535aed1db463 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 30 Mar 2021 15:52:03 +0200 Subject: [PATCH 247/709] ethstats: avoid creating subscriptions on background goroutine (#22587) This fixes an issue where the ethstats service could crash if geth was started and then immediately stopped due to an internal error. The cause of the crash was a nil subscription being returned by the backend, because the background goroutine creating them was scheduled after the backend had already shut down. Moving the creation of subscriptions into the Start method, which runs synchronously during startup of the node, means the returned subscriptions can never be 'nil'. Co-authored-by: Felix Lange Co-authored-by: Martin Holst Swende --- ethstats/ethstats.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index e0f4f95ff3..0f386c02fd 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -95,6 +95,8 @@ type Service struct { pongCh chan struct{} // Pong notifications are fed into this channel histCh chan []uint64 // History request block numbers are fed into this channel + headSub event.Subscription + txSub event.Subscription } // connWrapper is a wrapper to prevent concurrent-write or concurrent-read on the @@ -167,7 +169,12 @@ func New(node *node.Node, backend backend, engine consensus.Engine, url string) // Start implements node.Lifecycle, starting up the monitoring and reporting daemon. func (s *Service) Start() error { - go s.loop() + // Subscribe to chain events to execute updates on + chainHeadCh := make(chan core.ChainHeadEvent, chainHeadChanSize) + s.headSub = s.backend.SubscribeChainHeadEvent(chainHeadCh) + txEventCh := make(chan core.NewTxsEvent, txChanSize) + s.txSub = s.backend.SubscribeNewTxsEvent(txEventCh) + go s.loop(chainHeadCh, txEventCh) log.Info("Stats daemon started") return nil @@ -175,22 +182,15 @@ func (s *Service) Start() error { // Stop implements node.Lifecycle, terminating the monitoring and reporting daemon. func (s *Service) Stop() error { + s.headSub.Unsubscribe() + s.txSub.Unsubscribe() log.Info("Stats daemon stopped") return nil } // loop keeps trying to connect to the netstats server, reporting chain events // until termination. -func (s *Service) loop() { - // Subscribe to chain events to execute updates on - chainHeadCh := make(chan core.ChainHeadEvent, chainHeadChanSize) - headSub := s.backend.SubscribeChainHeadEvent(chainHeadCh) - defer headSub.Unsubscribe() - - txEventCh := make(chan core.NewTxsEvent, txChanSize) - txSub := s.backend.SubscribeNewTxsEvent(txEventCh) - defer txSub.Unsubscribe() - +func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent) { // Start a goroutine that exhausts the subscriptions to avoid events piling up var ( quitCh = make(chan struct{}) @@ -223,9 +223,9 @@ func (s *Service) loop() { } // node stopped - case <-txSub.Err(): + case <-s.txSub.Err(): break HandleLoop - case <-headSub.Err(): + case <-s.headSub.Err(): break HandleLoop } } From 61ff3e86b2d33f32566cae4f9bd1be973d84d757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 30 Mar 2021 19:04:22 +0300 Subject: [PATCH 248/709] core/state/snapshot, ethdb: track deletions more accurately (#22582) * core/state/snapshot, ethdb: track deletions more accurately * core/state/snapshot: don't reset the iterator, leveldb's screwy * ethdb: don't mess with the insert batches for now --- core/state/snapshot/snapshot.go | 21 ++++++++++++++++++++- ethdb/leveldb/leveldb.go | 2 +- ethdb/memorydb/memorydb.go | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 810f1354e9..eccf377264 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -484,8 +484,17 @@ func diffToDisk(bottom *diffLayer) *diskLayer { if key := it.Key(); len(key) == 65 { // TODO(karalabe): Yuck, we should move this into the iterator batch.Delete(key) base.cache.Del(key[1:]) - snapshotFlushStorageItemMeter.Mark(1) + + // Ensure we don't delete too much data blindly (contract can be + // huge). It's ok to flush, the root will go missing in case of a + // crash and we'll detect and regenerate the snapshot. + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Crit("Failed to write storage deletions", "err", err) + } + batch.Reset() + } } } it.Release() @@ -503,6 +512,16 @@ func diffToDisk(bottom *diffLayer) *diskLayer { snapshotFlushAccountItemMeter.Mark(1) snapshotFlushAccountSizeMeter.Mark(int64(len(data))) + + // Ensure we don't write too much data blindly. It's ok to flush, the + // root will go missing in case of a crash and we'll detect and regen + // the snapshot. + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Crit("Failed to write storage deletions", "err", err) + } + batch.Reset() + } } // Push all the storage slots into the database for accountHash, storage := range bottom.storageData { diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index d3011212aa..5d19cc3577 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -461,7 +461,7 @@ func (b *batch) Put(key, value []byte) error { // Delete inserts the a key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { b.b.Delete(key) - b.size++ + b.size += len(key) return nil } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 4c5e1a84de..fedc9e326c 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -211,7 +211,7 @@ func (b *batch) Put(key, value []byte) error { // Delete inserts the a key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true}) - b.size += 1 + b.size += len(key) return nil } From 4a37ae510eb02bb7daf1fa5982f326e92a401c60 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 30 Mar 2021 20:09:30 +0200 Subject: [PATCH 249/709] rpc: tighter shutdown synchronization in client subscription (#22597) This fixes a rare issue where the client subscription forwarding loop would attempt send on the subscription's channel after Unsubscribe has returned, leading to a panic if the subscription channel was already closed by the user. Example: sub, _ := client.Subscribe(..., channel, ...) sub.Unsubscribe() close(channel) The race occurred because Unsubscribe called quitWithServer to tell the forwarding loop to stop sending on sub.channel, but did not wait for the loop to actually come down. This is fixed by adding an additional channel to track the shutdown, on which Unsubscribe now waits. Fixes #22322 --- rpc/client_test.go | 88 ++++++++++++++++++++++++++++++ rpc/handler.go | 4 +- rpc/subscription.go | 121 +++++++++++++++++++++++++++++------------- rpc/websocket_test.go | 4 ++ 4 files changed, 177 insertions(+), 40 deletions(-) diff --git a/rpc/client_test.go b/rpc/client_test.go index 5d301a07a7..224eb0c5c8 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -18,6 +18,7 @@ package rpc import ( "context" + "encoding/json" "fmt" "math/rand" "net" @@ -376,6 +377,93 @@ func TestClientCloseUnsubscribeRace(t *testing.T) { } } +// unsubscribeRecorder collects the subscription IDs of *_unsubscribe calls. +type unsubscribeRecorder struct { + ServerCodec + unsubscribes map[string]bool +} + +func (r *unsubscribeRecorder) readBatch() ([]*jsonrpcMessage, bool, error) { + if r.unsubscribes == nil { + r.unsubscribes = make(map[string]bool) + } + + msgs, batch, err := r.ServerCodec.readBatch() + for _, msg := range msgs { + if msg.isUnsubscribe() { + var params []string + if err := json.Unmarshal(msg.Params, ¶ms); err != nil { + panic("unsubscribe decode error: " + err.Error()) + } + r.unsubscribes[params[0]] = true + } + } + return msgs, batch, err +} + +// This checks that Client calls the _unsubscribe method on the server when Unsubscribe is +// called on a subscription. +func TestClientSubscriptionUnsubscribeServer(t *testing.T) { + t.Parallel() + + // Create the server. + srv := NewServer() + srv.RegisterName("nftest", new(notificationTestService)) + p1, p2 := net.Pipe() + recorder := &unsubscribeRecorder{ServerCodec: NewCodec(p1)} + go srv.ServeCodec(recorder, OptionMethodInvocation|OptionSubscriptions) + defer srv.Stop() + + // Create the client on the other end of the pipe. + client, _ := newClient(context.Background(), func(context.Context) (ServerCodec, error) { + return NewCodec(p2), nil + }) + defer client.Close() + + // Create the subscription. + ch := make(chan int) + sub, err := client.Subscribe(context.Background(), "nftest", ch, "someSubscription", 1, 1) + if err != nil { + t.Fatal(err) + } + + // Unsubscribe and check that unsubscribe was called. + sub.Unsubscribe() + if !recorder.unsubscribes[sub.subid] { + t.Fatal("client did not call unsubscribe method") + } + if _, open := <-sub.Err(); open { + t.Fatal("subscription error channel not closed after unsubscribe") + } +} + +// This checks that the subscribed channel can be closed after Unsubscribe. +// It is the reproducer for https://github.com/ethereum/go-ethereum/issues/22322 +func TestClientSubscriptionChannelClose(t *testing.T) { + t.Parallel() + + var ( + srv = NewServer() + httpsrv = httptest.NewServer(srv.WebsocketHandler(nil)) + wsURL = "ws:" + strings.TrimPrefix(httpsrv.URL, "http:") + ) + defer srv.Stop() + defer httpsrv.Close() + + srv.RegisterName("nftest", new(notificationTestService)) + client, _ := Dial(wsURL) + + for i := 0; i < 100; i++ { + ch := make(chan int, 100) + sub, err := client.Subscribe(context.Background(), "nftest", ch, "someSubscription", maxClientSubscriptionBuffer-1, 1) + if err != nil { + t.Fatal(err) + } + sub.Unsubscribe() + close(ch) + } +} + // This test checks that Client doesn't lock up when a single subscriber // doesn't read subscription events. func TestClientNotificationStorm(t *testing.T) { diff --git a/rpc/handler.go b/rpc/handler.go index 23023eaca1..85f774a1b7 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -189,7 +189,7 @@ func (h *handler) cancelAllRequests(err error, inflightReq *requestOp) { } for id, sub := range h.clientSubs { delete(h.clientSubs, id) - sub.quitWithError(false, err) + sub.close(err) } } @@ -281,7 +281,7 @@ func (h *handler) handleResponse(msg *jsonrpcMessage) { return } if op.err = json.Unmarshal(msg.Result, &op.sub.subid); op.err == nil { - go op.sub.start() + go op.sub.run() h.clientSubs[op.sub.subid] = op.sub } } diff --git a/rpc/subscription.go b/rpc/subscription.go index 233215d792..942e764e5d 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -208,23 +208,37 @@ type ClientSubscription struct { channel reflect.Value namespace string subid string - in chan json.RawMessage - quitOnce sync.Once // ensures quit is closed once - quit chan struct{} // quit is closed when the subscription exits - errOnce sync.Once // ensures err is closed once - err chan error + // The in channel receives notification values from client dispatcher. + in chan json.RawMessage + + // The error channel receives the error from the forwarding loop. + // It is closed by Unsubscribe. + err chan error + errOnce sync.Once + + // Closing of the subscription is requested by sending on 'quit'. This is handled by + // the forwarding loop, which closes 'forwardDone' when it has stopped sending to + // sub.channel. Finally, 'unsubDone' is closed after unsubscribing on the server side. + quit chan error + forwardDone chan struct{} + unsubDone chan struct{} } +// This is the sentinel value sent on sub.quit when Unsubscribe is called. +var errUnsubscribed = errors.New("unsubscribed") + func newClientSubscription(c *Client, namespace string, channel reflect.Value) *ClientSubscription { sub := &ClientSubscription{ - client: c, - namespace: namespace, - etype: channel.Type().Elem(), - channel: channel, - quit: make(chan struct{}), - err: make(chan error, 1), - in: make(chan json.RawMessage), + client: c, + namespace: namespace, + etype: channel.Type().Elem(), + channel: channel, + in: make(chan json.RawMessage), + quit: make(chan error), + forwardDone: make(chan struct{}), + unsubDone: make(chan struct{}), + err: make(chan error, 1), } return sub } @@ -232,9 +246,9 @@ func newClientSubscription(c *Client, namespace string, channel reflect.Value) * // Err returns the subscription error channel. The intended use of Err is to schedule // resubscription when the client connection is closed unexpectedly. // -// The error channel receives a value when the subscription has ended due -// to an error. The received error is nil if Close has been called -// on the underlying client and no other error has occurred. +// The error channel receives a value when the subscription has ended due to an error. The +// received error is nil if Close has been called on the underlying client and no other +// error has occurred. // // The error channel is closed when Unsubscribe is called on the subscription. func (sub *ClientSubscription) Err() <-chan error { @@ -244,41 +258,63 @@ func (sub *ClientSubscription) Err() <-chan error { // Unsubscribe unsubscribes the notification and closes the error channel. // It can safely be called more than once. func (sub *ClientSubscription) Unsubscribe() { - sub.quitWithError(true, nil) - sub.errOnce.Do(func() { close(sub.err) }) -} - -func (sub *ClientSubscription) quitWithError(unsubscribeServer bool, err error) { - sub.quitOnce.Do(func() { - // The dispatch loop won't be able to execute the unsubscribe call - // if it is blocked on deliver. Close sub.quit first because it - // unblocks deliver. - close(sub.quit) - if unsubscribeServer { - sub.requestUnsubscribe() - } - if err != nil { - if err == ErrClientQuit { - err = nil // Adhere to subscription semantics. - } - sub.err <- err + sub.errOnce.Do(func() { + select { + case sub.quit <- errUnsubscribed: + <-sub.unsubDone + case <-sub.unsubDone: } + close(sub.err) }) } +// deliver is called by the client's message dispatcher to send a notification value. func (sub *ClientSubscription) deliver(result json.RawMessage) (ok bool) { select { case sub.in <- result: return true - case <-sub.quit: + case <-sub.forwardDone: return false } } -func (sub *ClientSubscription) start() { - sub.quitWithError(sub.forward()) +// close is called by the client's message dispatcher when the connection is closed. +func (sub *ClientSubscription) close(err error) { + select { + case sub.quit <- err: + case <-sub.forwardDone: + } +} + +// run is the forwarding loop of the subscription. It runs in its own goroutine and +// is launched by the client's handler after the subscription has been created. +func (sub *ClientSubscription) run() { + defer close(sub.unsubDone) + + unsubscribe, err := sub.forward() + + // The client's dispatch loop won't be able to execute the unsubscribe call if it is + // blocked in sub.deliver() or sub.close(). Closing forwardDone unblocks them. + close(sub.forwardDone) + + // Call the unsubscribe method on the server. + if unsubscribe { + sub.requestUnsubscribe() + } + + // Send the error. + if err != nil { + if err == ErrClientQuit { + // ErrClientQuit gets here when Client.Close is called. This is reported as a + // nil error because it's not an error, but we can't close sub.err here. + err = nil + } + sub.err <- err + } } +// forward is the forwarding loop. It takes in RPC notifications and sends them +// on the subscription channel. func (sub *ClientSubscription) forward() (unsubscribeServer bool, err error) { cases := []reflect.SelectCase{ {Dir: reflect.SelectRecv, Chan: reflect.ValueOf(sub.quit)}, @@ -286,7 +322,7 @@ func (sub *ClientSubscription) forward() (unsubscribeServer bool, err error) { {Dir: reflect.SelectSend, Chan: sub.channel}, } buffer := list.New() - defer buffer.Init() + for { var chosen int var recv reflect.Value @@ -301,7 +337,15 @@ func (sub *ClientSubscription) forward() (unsubscribeServer bool, err error) { switch chosen { case 0: // <-sub.quit - return false, nil + if !recv.IsNil() { + err = recv.Interface().(error) + } + if err == errUnsubscribed { + // Exiting because Unsubscribe was called, unsubscribe on server. + return true, nil + } + return false, err + case 1: // <-sub.in val, err := sub.unmarshal(recv.Interface().(json.RawMessage)) if err != nil { @@ -311,6 +355,7 @@ func (sub *ClientSubscription) forward() (unsubscribeServer bool, err error) { return true, ErrSubscriptionQueueOverflow } buffer.PushBack(val) + case 2: // sub.channel<- cases[2].Send = reflect.Value{} // Don't hold onto the value. buffer.Remove(buffer.Front()) diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index 37ed19476f..4976853baf 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -129,11 +129,15 @@ func TestClientWebsocketPing(t *testing.T) { if err != nil { t.Fatalf("client dial error: %v", err) } + defer client.Close() + resultChan := make(chan int) sub, err := client.EthSubscribe(ctx, resultChan, "foo") if err != nil { t.Fatalf("client subscribe error: %v", err) } + // Note: Unsubscribe is not called on this subscription because the mockup + // server can't handle the request. // Wait for the context's deadline to be reached before proceeding. // This is important for reproducing https://github.com/ethereum/go-ethereum/issues/19798 From 55300d4fdbfda751a7a3d4baadd9db65c50b22b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 31 Mar 2021 10:56:51 +0300 Subject: [PATCH 250/709] all: fix miner hashRate -> hashrate on API calls --- consensus/ethash/api.go | 2 +- consensus/ethash/ethash_test.go | 6 +++--- eth/api.go | 10 ---------- ethstats/ethstats.go | 2 +- internal/web3ext/web3ext.go | 4 ++-- miner/miner.go | 2 +- 6 files changed, 8 insertions(+), 18 deletions(-) diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go index 68b3a84b09..f4d3802e0b 100644 --- a/consensus/ethash/api.go +++ b/consensus/ethash/api.go @@ -89,7 +89,7 @@ func (api *API) SubmitWork(nonce types.BlockNonce, hash, digest common.Hash) boo // // It accepts the miner hash rate and an identifier which must be unique // between nodes. -func (api *API) SubmitHashRate(rate hexutil.Uint64, id common.Hash) bool { +func (api *API) SubmitHashrate(rate hexutil.Uint64, id common.Hash) bool { if api.ethash.remote == nil { return false } diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index eb3850b3b0..382eefeecf 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -135,7 +135,7 @@ func TestRemoteSealer(t *testing.T) { } } -func TestHashRate(t *testing.T) { +func TestHashrate(t *testing.T) { var ( hashrate = []hexutil.Uint64{100, 200, 300} expect uint64 @@ -150,7 +150,7 @@ func TestHashRate(t *testing.T) { api := &API{ethash} for i := 0; i < len(hashrate); i += 1 { - if res := api.SubmitHashRate(hashrate[i], ids[i]); !res { + if res := api.SubmitHashrate(hashrate[i], ids[i]); !res { t.Error("remote miner submit hashrate failed") } expect += uint64(hashrate[i]) @@ -170,7 +170,7 @@ func TestClosedRemoteSealer(t *testing.T) { t.Error("expect to return an error to indicate ethash is stopped") } - if res := api.SubmitHashRate(hexutil.Uint64(100), common.HexToHash("a")); res { + if res := api.SubmitHashrate(hexutil.Uint64(100), common.HexToHash("a")); res { t.Error("expect to return false when submit hashrate to a stopped ethash") } } diff --git a/eth/api.go b/eth/api.go index 53ef91392b..c2f72a1897 100644 --- a/eth/api.go +++ b/eth/api.go @@ -61,11 +61,6 @@ func (api *PublicEthereumAPI) Coinbase() (common.Address, error) { return api.Etherbase() } -// Hashrate returns the POW hashrate -func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { - return hexutil.Uint64(api.e.Miner().HashRate()) -} - // ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. func (api *PublicEthereumAPI) ChainId() (hexutil.Uint64, error) { // if current block is at or past the EIP-155 replay-protection fork block, return chainID from config @@ -149,11 +144,6 @@ func (api *PrivateMinerAPI) SetRecommitInterval(interval int) { api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) } -// GetHashrate returns the current hashrate of the miner. -func (api *PrivateMinerAPI) GetHashrate() uint64 { - return api.e.miner.HashRate() -} - // PrivateAdminAPI is the collection of Ethereum full node-related APIs // exposed over the private admin endpoint. type PrivateAdminAPI struct { diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 0f386c02fd..c7acb9481c 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -754,7 +754,7 @@ func (s *Service) reportStats(conn *connWrapper) error { fullBackend, ok := s.backend.(fullNodeBackend) if ok { mining = fullBackend.Miner().Mining() - hashrate = int(fullBackend.Miner().HashRate()) + hashrate = int(fullBackend.Miner().Hashrate()) sync := fullBackend.Downloader().Progress() syncing = fullBackend.CurrentHeader().Number.Uint64() >= sync.HighestBlock diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index aba262699c..e1f20ad72a 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -138,8 +138,8 @@ web3._extend({ params: 3, }), new web3._extend.Method({ - name: 'submitHashRate', - call: 'ethash_submitHashRate', + name: 'submitHashrate', + call: 'ethash_submitHashrate', params: 2, }), ] diff --git a/miner/miner.go b/miner/miner.go index 4d71e307a6..00c3d0cb5c 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -160,7 +160,7 @@ func (miner *Miner) Mining() bool { return miner.worker.isRunning() } -func (miner *Miner) HashRate() uint64 { +func (miner *Miner) Hashrate() uint64 { if pow, ok := miner.engine.(consensus.PoW); ok { return uint64(pow.Hashrate()) } From c79fc209cdcdd3ec28e4643b12c954fdbeeba452 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Tue, 6 Apr 2021 04:57:00 -0400 Subject: [PATCH 251/709] core/state/snapshot: fix data race in diff layer (#22540) --- core/state/snapshot/difflayer.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 9c86a679d1..ee88938b77 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -296,13 +296,17 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { if !hit { hit = dl.diffed.Contains(destructBloomHasher(hash)) } + var origin *diskLayer + if !hit { + origin = dl.origin // extract origin while holding the lock + } dl.lock.RUnlock() // If the bloom filter misses, don't even bother with traversing the memory // diff layers, reach straight into the bottom persistent disk layer - if !hit { + if origin != nil { snapshotBloomAccountMissMeter.Mark(1) - return dl.origin.AccountRLP(hash) + return origin.AccountRLP(hash) } // The bloom filter hit, start poking in the internal maps return dl.accountRLP(hash, 0) @@ -358,13 +362,17 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro if !hit { hit = dl.diffed.Contains(destructBloomHasher(accountHash)) } + var origin *diskLayer + if !hit { + origin = dl.origin // extract origin while holding the lock + } dl.lock.RUnlock() // If the bloom filter misses, don't even bother with traversing the memory // diff layers, reach straight into the bottom persistent disk layer - if !hit { + if origin != nil { snapshotBloomStorageMissMeter.Mark(1) - return dl.origin.Storage(accountHash, storageHash) + return origin.Storage(accountHash, storageHash) } // The bloom filter hit, start poking in the internal maps return dl.storage(accountHash, storageHash, 0) From 706683ea72c1b711bd3b56327fe6c08e70bd37e3 Mon Sep 17 00:00:00 2001 From: piersy Date: Tue, 6 Apr 2021 11:24:39 +0100 Subject: [PATCH 252/709] internal/ethapi: fix eth_chainId method (#22243) This removes the duplicated definition of eth_chainID in package eth and updates the definition in internal/ethapi to treat chain ID as a bigint. Co-authored-by: Felix Lange --- eth/api.go | 9 --------- internal/ethapi/api.go | 10 +++++++--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/eth/api.go b/eth/api.go index c2f72a1897..fb51e78c1a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -61,15 +61,6 @@ func (api *PublicEthereumAPI) Coinbase() (common.Address, error) { return api.Etherbase() } -// ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. -func (api *PublicEthereumAPI) ChainId() (hexutil.Uint64, error) { - // if current block is at or past the EIP-155 replay-protection fork block, return chainID from config - if config := api.e.blockchain.Config(); config.IsEIP155(api.e.blockchain.CurrentBlock().Number()) { - return (hexutil.Uint64)(config.ChainID.Uint64()), nil - } - return hexutil.Uint64(0), fmt.Errorf("chain not synced beyond EIP-155 replay-protection fork block") -} - // PublicMinerAPI provides an API to control the miner. // It offers only methods that operate on data that pose no security risk when it is publicly accessible. type PublicMinerAPI struct { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 622063cf64..861d427851 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -529,9 +529,13 @@ func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { return &PublicBlockChainAPI{b} } -// ChainId returns the chainID value for transaction replay protection. -func (s *PublicBlockChainAPI) ChainId() *hexutil.Big { - return (*hexutil.Big)(s.b.ChainConfig().ChainID) +// ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. +func (api *PublicBlockChainAPI) ChainId() (*hexutil.Big, error) { + // if current block is at or past the EIP-155 replay-protection fork block, return chainID from config + if config := api.b.ChainConfig(); config.IsEIP155(api.b.CurrentBlock().Number()) { + return (*hexutil.Big)(config.ChainID), nil + } + return nil, fmt.Errorf("chain not synced beyond EIP-155 replay-protection fork block") } // BlockNumber returns the block number of the chain head. From adf09aeab1fc091e4ab56c04852b18b4b924eb50 Mon Sep 17 00:00:00 2001 From: AmitBRD <60668103+AmitBRD@users.noreply.github.com> Date: Tue, 6 Apr 2021 08:58:36 -0500 Subject: [PATCH 253/709] graphql: add support for tx types and tx access lists (#22491) This adds support for EIP-2718 access list transactions in the GraphQL API. Co-authored-by: Amit Shah Co-authored-by: Felix Lange --- graphql/graphql.go | 39 ++++++++++++ graphql/graphql_test.go | 136 ++++++++++++++++++++++++++++++++++++++-- graphql/schema.go | 9 +++ 3 files changed, 180 insertions(+), 4 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 2374beb8e1..b1af7675bd 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -151,6 +151,20 @@ func (l *Log) Data(ctx context.Context) hexutil.Bytes { return l.log.Data } +// AccessTuple represents EIP-2930 +type AccessTuple struct { + address common.Address + storageKeys *[]common.Hash +} + +func (at *AccessTuple) Address(ctx context.Context) common.Address { + return at.address +} + +func (at *AccessTuple) StorageKeys(ctx context.Context) *[]common.Hash { + return at.storageKeys +} + // Transaction represents an Ethereum transaction. // backend and hash are mandatory; all others will be fetched when required. type Transaction struct { @@ -342,6 +356,31 @@ func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { return &ret, nil } +func (t *Transaction) Type(ctx context.Context) (*int32, error) { + tx, err := t.resolve(ctx) + if err != nil { + return nil, err + } + txType := int32(tx.Type()) + return &txType, nil +} + +func (t *Transaction) AccessList(ctx context.Context) (*[]*AccessTuple, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return nil, err + } + accessList := tx.AccessList() + ret := make([]*AccessTuple, 0, len(accessList)) + for _, al := range accessList { + ret = append(ret, &AccessTuple{ + address: al.Address, + storageKeys: &al.StorageKeys, + }) + } + return &ret, nil +} + func (t *Transaction) R(ctx context.Context) (hexutil.Big, error) { tx, err := t.resolve(ctx) if err != nil || tx == nil { diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 2f3b230329..2404ff45fa 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -25,8 +25,12 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" @@ -55,7 +59,7 @@ func TestBuildSchema(t *testing.T) { // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint func TestGraphQLBlockSerialization(t *testing.T) { - stack := createNode(t, true) + stack := createNode(t, true, false) defer stack.Close() // start node if err := stack.Start(); err != nil { @@ -157,9 +161,45 @@ func TestGraphQLBlockSerialization(t *testing.T) { } } +func TestGraphQLBlockSerializationEIP2718(t *testing.T) { + stack := createNode(t, true, true) + defer stack.Close() + // start node + if err := stack.Start(); err != nil { + t.Fatalf("could not start node: %v", err) + } + + for i, tt := range []struct { + body string + want string + code int + }{ + { + body: `{"query": "{block {number transactions { from { address } to { address } value hash type accessList { address storageKeys } index}}}"}`, + want: `{"data":{"block":{"number":1,"transactions":[{"from":{"address":"0x71562b71999873db5b286df957af199ec94617f7"},"to":{"address":"0x0000000000000000000000000000000000000dad"},"value":"0x64","hash":"0x4f7b8d718145233dcf7f29e34a969c63dd4de8715c054ea2af022b66c4f4633e","type":0,"accessList":[],"index":0},{"from":{"address":"0x71562b71999873db5b286df957af199ec94617f7"},"to":{"address":"0x0000000000000000000000000000000000000dad"},"value":"0x32","hash":"0x9c6c2c045b618fe87add0e49ba3ca00659076ecae00fd51de3ba5d4ccf9dbf40","type":1,"accessList":[{"address":"0x0000000000000000000000000000000000000dad","storageKeys":["0x0000000000000000000000000000000000000000000000000000000000000000"]}],"index":1}]}}}`, + code: 200, + }, + } { + resp, err := http.Post(fmt.Sprintf("%s/graphql", stack.HTTPEndpoint()), "application/json", strings.NewReader(tt.body)) + if err != nil { + t.Fatalf("could not post: %v", err) + } + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("could not read from response body: %v", err) + } + if have := string(bodyBytes); have != tt.want { + t.Errorf("testcase %d %s,\nhave:\n%v\nwant:\n%v", i, tt.body, have, tt.want) + } + if tt.code != resp.StatusCode { + t.Errorf("testcase %d %s,\nwrong statuscode, have: %v, want: %v", i, tt.body, resp.StatusCode, tt.code) + } + } +} + // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { - stack := createNode(t, false) + stack := createNode(t, false, false) defer stack.Close() if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -173,7 +213,7 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } -func createNode(t *testing.T, gqlEnabled bool) *node.Node { +func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", HTTPPort: 0, @@ -186,7 +226,11 @@ func createNode(t *testing.T, gqlEnabled bool) *node.Node { if !gqlEnabled { return stack } - createGQLService(t, stack) + if !txEnabled { + createGQLService(t, stack) + } else { + createGQLServiceWithTransactions(t, stack) + } return stack } @@ -226,3 +270,87 @@ func createGQLService(t *testing.T, stack *node.Node) { t.Fatalf("could not create graphql service: %v", err) } } + +func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { + // create backend + key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address := crypto.PubkeyToAddress(key.PublicKey) + funds := big.NewInt(1000000000) + dad := common.HexToAddress("0x0000000000000000000000000000000000000dad") + + ethConf := ðconfig.Config{ + Genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + address: {Balance: funds}, + // The address 0xdad sloads 0x00 and 0x01 + dad: { + Code: []byte{ + byte(vm.PC), + byte(vm.PC), + byte(vm.SLOAD), + byte(vm.SLOAD), + }, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + }, + Ethash: ethash.Config{ + PowMode: ethash.ModeFake, + }, + NetworkId: 1337, + TrieCleanCache: 5, + TrieCleanCacheJournal: "triecache", + TrieCleanCacheRejournal: 60 * time.Minute, + TrieDirtyCache: 5, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 5, + } + + ethBackend, err := eth.New(stack, ethConf) + if err != nil { + t.Fatalf("could not create eth backend: %v", err) + } + signer := types.LatestSigner(ethConf.Genesis.Config) + + legacyTx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: uint64(0), + To: &dad, + Value: big.NewInt(100), + Gas: 50000, + GasPrice: big.NewInt(1), + }) + envelopTx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: ethConf.Genesis.Config.ChainID, + Nonce: uint64(1), + To: &dad, + Gas: 30000, + GasPrice: big.NewInt(1), + Value: big.NewInt(50), + AccessList: types.AccessList{{ + Address: dad, + StorageKeys: []common.Hash{{0}}, + }}, + }) + + // Create some blocks and import them + chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), + ethash.NewFaker(), ethBackend.ChainDb(), 1, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + b.AddTx(legacyTx) + b.AddTx(envelopTx) + }) + + _, err = ethBackend.BlockChain().InsertChain(chain) + if err != nil { + t.Fatalf("could not create import blocks: %v", err) + } + // create gql service + err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + if err != nil { + t.Fatalf("could not create graphql service: %v", err) + } +} diff --git a/graphql/schema.go b/graphql/schema.go index 6ea63db636..d792bb9153 100644 --- a/graphql/schema.go +++ b/graphql/schema.go @@ -69,6 +69,12 @@ const schema string = ` transaction: Transaction! } + #EIP-2718 + type AccessTuple{ + address: Address! + storageKeys : [Bytes32!] + } + # Transaction is an Ethereum transaction. type Transaction { # Hash is the hash of this transaction. @@ -118,6 +124,9 @@ const schema string = ` r: BigInt! s: BigInt! v: BigInt! + #Envelope transaction support + type: Int + accessList: [AccessTuple!] } # BlockFilterCriteria encapsulates log filter criteria for a filter applied From 5338ce44475191eb8708509a44d92f1219a8a798 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 6 Apr 2021 18:39:51 +0430 Subject: [PATCH 254/709] internal/debug: add JSON log format and rename logging flags (#22341) This change adds support for logging JSON records when the --log.json flag is given. The --debug and --backtrace flags are deprecated and replaced by --log.debug and --log.backtrace. While changing this, it was noticed that the --memprofilerate and --blockprofilerate were ineffective (they were always overridden even if --pprof.memprofilerate was not set). This is also fixed. Co-authored-by: Felix Lange --- internal/debug/flags.go | 124 ++++++++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 18 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 2c92b19de6..126ee09a7c 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -41,22 +41,22 @@ var ( Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", Value: 3, } - logjsonFlag = cli.BoolFlag{ - Name: "log.json", - Usage: "Format logs with JSON", - } vmoduleFlag = cli.StringFlag{ Name: "vmodule", Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", Value: "", } + logjsonFlag = cli.BoolFlag{ + Name: "log.json", + Usage: "Format logs with JSON", + } backtraceAtFlag = cli.StringFlag{ - Name: "backtrace", + Name: "log.backtrace", Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\")", Value: "", } debugFlag = cli.BoolFlag{ - Name: "debug", + Name: "log.debug", Usage: "Prepends log messages with call-site location (file and line number)", } pprofFlag = cli.BoolFlag{ @@ -90,18 +90,69 @@ var ( Name: "trace", Usage: "Write execution trace to the given file", } + // (Deprecated April 2020) + legacyPprofPortFlag = cli.IntFlag{ + Name: "pprofport", + Usage: "pprof HTTP server listening port (deprecated, use --pprof.port)", + Value: 6060, + } + legacyPprofAddrFlag = cli.StringFlag{ + Name: "pprofaddr", + Usage: "pprof HTTP server listening interface (deprecated, use --pprof.addr)", + Value: "127.0.0.1", + } + legacyMemprofilerateFlag = cli.IntFlag{ + Name: "memprofilerate", + Usage: "Turn on memory profiling with the given rate (deprecated, use --pprof.memprofilerate)", + Value: runtime.MemProfileRate, + } + legacyBlockprofilerateFlag = cli.IntFlag{ + Name: "blockprofilerate", + Usage: "Turn on block profiling with the given rate (deprecated, use --pprof.blockprofilerate)", + } + legacyCpuprofileFlag = cli.StringFlag{ + Name: "cpuprofile", + Usage: "Write CPU profile to the given file (deprecated, use --pprof.cpuprofile)", + } + legacyBacktraceAtFlag = cli.StringFlag{ + Name: "backtrace", + Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\") (deprecated, use --log.backtrace)", + Value: "", + } + legacyDebugFlag = cli.BoolFlag{ + Name: "debug", + Usage: "Prepends log messages with call-site location (file and line number) (deprecated, use --log.debug)", + } ) // Flags holds all command-line flags required for debugging. var Flags = []cli.Flag{ - verbosityFlag, logjsonFlag, vmoduleFlag, backtraceAtFlag, debugFlag, - pprofFlag, pprofAddrFlag, pprofPortFlag, memprofilerateFlag, - blockprofilerateFlag, cpuprofileFlag, traceFlag, + verbosityFlag, + vmoduleFlag, + logjsonFlag, + backtraceAtFlag, + debugFlag, + pprofFlag, + pprofAddrFlag, + pprofPortFlag, + memprofilerateFlag, + blockprofilerateFlag, + cpuprofileFlag, + traceFlag, } -var ( - glogger *log.GlogHandler -) +// This is the list of deprecated debugging flags. +var DeprecatedFlags = []cli.Flag{ + legacyPprofPortFlag, + legacyPprofAddrFlag, + legacyMemprofilerateFlag, + legacyBlockprofilerateFlag, + legacyCpuprofileFlag, + legacyBacktraceAtFlag, + legacyDebugFlag, +} + +var glogger *log.GlogHandler func init() { glogger = log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) @@ -124,17 +175,54 @@ func Setup(ctx *cli.Context) error { ostream = log.StreamHandler(output, log.TerminalFormat(usecolor)) } glogger.SetHandler(ostream) + // logging - log.PrintOrigins(ctx.GlobalBool(debugFlag.Name)) - glogger.Verbosity(log.Lvl(ctx.GlobalInt(verbosityFlag.Name))) - glogger.Vmodule(ctx.GlobalString(vmoduleFlag.Name)) - glogger.BacktraceAt(ctx.GlobalString(backtraceAtFlag.Name)) + verbosity := ctx.GlobalInt(verbosityFlag.Name) + glogger.Verbosity(log.Lvl(verbosity)) + vmodule := ctx.GlobalString(vmoduleFlag.Name) + glogger.Vmodule(vmodule) + + debug := ctx.GlobalBool(debugFlag.Name) + if ctx.GlobalIsSet(legacyDebugFlag.Name) { + debug = ctx.GlobalBool(legacyDebugFlag.Name) + log.Warn("The flag --debug is deprecated and will be removed in the future, please use --log.debug") + } + if ctx.GlobalIsSet(debugFlag.Name) { + debug = ctx.GlobalBool(debugFlag.Name) + } + log.PrintOrigins(debug) + + backtrace := ctx.GlobalString(backtraceAtFlag.Name) + if b := ctx.GlobalString(legacyBacktraceAtFlag.Name); b != "" { + backtrace = b + log.Warn("The flag --backtrace is deprecated and will be removed in the future, please use --log.backtrace") + } + if b := ctx.GlobalString(backtraceAtFlag.Name); b != "" { + backtrace = b + } + glogger.BacktraceAt(backtrace) + log.Root().SetHandler(glogger) // profiling, tracing - runtime.MemProfileRate = ctx.GlobalInt(memprofilerateFlag.Name) + runtime.MemProfileRate = memprofilerateFlag.Value + if ctx.GlobalIsSet(legacyMemprofilerateFlag.Name) { + runtime.MemProfileRate = ctx.GlobalInt(legacyMemprofilerateFlag.Name) + log.Warn("The flag --memprofilerate is deprecated and will be removed in the future, please use --pprof.memprofilerate") + } + if ctx.GlobalIsSet(memprofilerateFlag.Name) { + runtime.MemProfileRate = ctx.GlobalInt(memprofilerateFlag.Name) + } - Handler.SetBlockProfileRate(ctx.GlobalInt(blockprofilerateFlag.Name)) + blockProfileRate := blockprofilerateFlag.Value + if ctx.GlobalIsSet(legacyBlockprofilerateFlag.Name) { + blockProfileRate = ctx.GlobalInt(legacyBlockprofilerateFlag.Name) + log.Warn("The flag --blockprofilerate is deprecated and will be removed in the future, please use --pprof.blockprofilerate") + } + if ctx.GlobalIsSet(blockprofilerateFlag.Name) { + blockProfileRate = ctx.GlobalInt(blockprofilerateFlag.Name) + } + Handler.SetBlockProfileRate(blockProfileRate) if traceFile := ctx.GlobalString(traceFlag.Name); traceFile != "" { if err := Handler.StartGoTrace(traceFile); err != nil { From 95219ae62dd157414e4ab9b3a63d2c80cf10efab Mon Sep 17 00:00:00 2001 From: Peter Simard Date: Tue, 6 Apr 2021 10:23:35 -0400 Subject: [PATCH 255/709] cmd/utils: move cache sanity check to SetEthConfig (#22510) Move the cache sanity check to the SetEthConfig function to allow the config file to load. --- cmd/geth/main.go | 22 ---------------------- cmd/utils/flags.go | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 556ad0dbf3..12a5ae9bfc 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -19,9 +19,7 @@ package main import ( "fmt" - "math" "os" - godebug "runtime/debug" "sort" "strconv" "strings" @@ -41,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - gopsutil "github.com/shirou/gopsutil/mem" "gopkg.in/urfave/cli.v1" ) @@ -300,25 +297,6 @@ func prepare(ctx *cli.Context) { log.Info("Dropping default light client cache", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 128) ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128)) } - // Cap the cache allowance and tune the garbage collector - mem, err := gopsutil.VirtualMemory() - if err == nil { - if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 { - log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024) - mem.Total = 2 * 1024 * 1024 * 1024 - } - allowance := int(mem.Total / 1024 / 1024 / 3) - if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance { - log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) - ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance)) - } - } - // Ensure Go's GC ignores the database cache for trigger percentage - cache := ctx.GlobalInt(utils.CacheFlag.Name) - gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024))) - - log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc)) - godebug.SetGCPercent(int(gogc)) // Start metrics export if enabled utils.SetupMetrics(ctx) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a602d5a35e..d631b8e332 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -22,9 +22,11 @@ import ( "fmt" "io" "io/ioutil" + "math" "math/big" "os" "path/filepath" + godebug "runtime/debug" "strconv" "strings" "text/tabwriter" @@ -65,6 +67,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" pcsclite "github.com/gballet/go-libpcsclite" + gopsutil "github.com/shirou/gopsutil/mem" "gopkg.in/urfave/cli.v1" ) @@ -1473,6 +1476,26 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { setWhitelist(ctx, cfg) setLes(ctx, cfg) + // Cap the cache allowance and tune the garbage collector + mem, err := gopsutil.VirtualMemory() + if err == nil { + if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 { + log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024) + mem.Total = 2 * 1024 * 1024 * 1024 + } + allowance := int(mem.Total / 1024 / 1024 / 3) + if cache := ctx.GlobalInt(CacheFlag.Name); cache > allowance { + log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) + ctx.GlobalSet(CacheFlag.Name, strconv.Itoa(allowance)) + } + } + // Ensure Go's GC ignores the database cache for trigger percentage + cache := ctx.GlobalInt(CacheFlag.Name) + gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024))) + + log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc)) + godebug.SetGCPercent(int(gogc)) + if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) } From e275b1a293bdbc8170a01c1612c1cdf8e82c1ca1 Mon Sep 17 00:00:00 2001 From: Evolution404 <35091674+Evolution404@users.noreply.github.com> Date: Wed, 7 Apr 2021 02:02:52 +0800 Subject: [PATCH 256/709] consensus/ethash: replace a magic number with it's constant (#22618) --- consensus/ethash/algorithm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 80379597e2..065e60b90b 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -174,7 +174,7 @@ func generateCache(dest []uint32, epoch uint64, seed []byte) { case <-done: return case <-time.After(3 * time.Second): - logger.Info("Generating ethash verification cache", "percentage", atomic.LoadUint32(&progress)*100/uint32(rows)/4, "elapsed", common.PrettyDuration(time.Since(start))) + logger.Info("Generating ethash verification cache", "percentage", atomic.LoadUint32(&progress)*100/uint32(rows)/(cacheRounds+1), "elapsed", common.PrettyDuration(time.Since(start))) } } }() From 2d89fe0883f0ebb5f0145763d46f5335febab3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 6 Apr 2021 20:42:50 +0200 Subject: [PATCH 257/709] les: move client pool to les/vflux/server (#22495) * les: move client pool to les/vflux/server * les/vflux/server: un-expose NodeBalance, remove unused fn, fix bugs * tests/fuzzers/vflux: add ClientPool fuzzer * les/vflux/server: fixed balance tests * les: rebase fix * les/vflux/server: fixed more bugs * les/vflux/server: unexported NodeStateMachine fields and flags * les/vflux/server: unexport all internal components and functions * les/vflux/server: fixed priorityPool test * les/vflux/server: polish balance * les/vflux/server: fixed mutex locking error * les/vflux/server: priorityPool bug fixed * common/prque: make Prque wrap-around priority handling optional * les/vflux/server: rename funcs, small optimizations * les/vflux/server: fixed timeUntil * les/vflux/server: separated balance.posValue and negValue * les/vflux/server: polish setup * les/vflux/server: enforce capacity curve monotonicity * les/vflux/server: simplified requestCapacity * les/vflux/server: requestCapacity with target range, no iterations in SetCapacity * les/vflux/server: minor changes * les/vflux/server: moved default factors to balanceTracker * les/vflux/server: set inactiveFlag in priorityPool * les/vflux/server: moved related metrics to vfs package * les/vflux/client: make priorityPool temp state logic cleaner * les/vflux/server: changed log.Crit to log.Error * add vflux fuzzer to oss-fuzz Co-authored-by: rjl493456442 --- common/prque/lazyqueue.go | 6 +- common/prque/prque.go | 7 +- common/prque/sstack.go | 20 +- common/prque/sstack_test.go | 6 +- les/api.go | 103 ++--- les/clientpool.go | 453 -------------------- les/flowcontrol/manager.go | 2 +- les/metrics.go | 17 +- les/peer.go | 268 +++++++++++- les/server.go | 85 +--- les/server_handler.go | 127 +----- les/servingqueue.go | 4 +- les/test_helper.go | 12 +- les/vflux/server/balance.go | 489 ++++++++++++---------- les/vflux/server/balance_test.go | 222 +++++----- les/vflux/server/balance_tracker.go | 165 ++++---- les/vflux/server/clientpool.go | 335 +++++++++++++++ les/{ => vflux/server}/clientpool_test.go | 271 ++++++------ les/vflux/server/metrics.go | 33 ++ les/vflux/server/prioritypool.go | 416 +++++++++--------- les/vflux/server/prioritypool_test.go | 76 ++-- les/vflux/server/service.go | 6 +- les/vflux/server/status.go | 59 +++ oss-fuzz.sh | 1 + p2p/nodestate/nodestate.go | 17 + tests/fuzzers/vflux/clientpool-fuzzer.go | 289 +++++++++++++ tests/fuzzers/vflux/debug/main.go | 41 ++ 27 files changed, 1986 insertions(+), 1544 deletions(-) delete mode 100644 les/clientpool.go create mode 100644 les/vflux/server/clientpool.go rename les/{ => vflux/server}/clientpool_test.go (61%) create mode 100644 les/vflux/server/metrics.go create mode 100644 les/vflux/server/status.go create mode 100644 tests/fuzzers/vflux/clientpool-fuzzer.go create mode 100644 tests/fuzzers/vflux/debug/main.go diff --git a/common/prque/lazyqueue.go b/common/prque/lazyqueue.go index c74faab7e6..37c2f3bd42 100644 --- a/common/prque/lazyqueue.go +++ b/common/prque/lazyqueue.go @@ -55,7 +55,7 @@ type ( // NewLazyQueue creates a new lazy queue func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPriority MaxPriorityCallback, clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue { q := &LazyQueue{ - popQueue: newSstack(nil), + popQueue: newSstack(nil, false), setIndex: setIndex, priority: priority, maxPriority: maxPriority, @@ -71,8 +71,8 @@ func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPrior // Reset clears the contents of the queue func (q *LazyQueue) Reset() { - q.queue[0] = newSstack(q.setIndex0) - q.queue[1] = newSstack(q.setIndex1) + q.queue[0] = newSstack(q.setIndex0, false) + q.queue[1] = newSstack(q.setIndex1, false) } // Refresh performs queue re-evaluation if necessary diff --git a/common/prque/prque.go b/common/prque/prque.go index 3cc5a1adaf..54c78b5fc2 100755 --- a/common/prque/prque.go +++ b/common/prque/prque.go @@ -28,7 +28,12 @@ type Prque struct { // New creates a new priority queue. func New(setIndex SetIndexCallback) *Prque { - return &Prque{newSstack(setIndex)} + return &Prque{newSstack(setIndex, false)} +} + +// NewWrapAround creates a new priority queue with wrap-around priority handling. +func NewWrapAround(setIndex SetIndexCallback) *Prque { + return &Prque{newSstack(setIndex, true)} } // Pushes a value with a given priority into the queue, expanding if necessary. diff --git a/common/prque/sstack.go b/common/prque/sstack.go index 8518af54ff..b06a95413d 100755 --- a/common/prque/sstack.go +++ b/common/prque/sstack.go @@ -31,22 +31,24 @@ type SetIndexCallback func(data interface{}, index int) // the stack (heap) functionality and the Len, Less and Swap methods for the // sortability requirements of the heaps. type sstack struct { - setIndex SetIndexCallback - size int - capacity int - offset int + setIndex SetIndexCallback + size int + capacity int + offset int + wrapAround bool blocks [][]*item active []*item } // Creates a new, empty stack. -func newSstack(setIndex SetIndexCallback) *sstack { +func newSstack(setIndex SetIndexCallback, wrapAround bool) *sstack { result := new(sstack) result.setIndex = setIndex result.active = make([]*item, blockSize) result.blocks = [][]*item{result.active} result.capacity = blockSize + result.wrapAround = wrapAround return result } @@ -94,7 +96,11 @@ func (s *sstack) Len() int { // Compares the priority of two elements of the stack (higher is first). // Required by sort.Interface. func (s *sstack) Less(i, j int) bool { - return (s.blocks[i/blockSize][i%blockSize].priority - s.blocks[j/blockSize][j%blockSize].priority) > 0 + a, b := s.blocks[i/blockSize][i%blockSize].priority, s.blocks[j/blockSize][j%blockSize].priority + if s.wrapAround { + return a-b > 0 + } + return a > b } // Swaps two elements in the stack. Required by sort.Interface. @@ -110,5 +116,5 @@ func (s *sstack) Swap(i, j int) { // Resets the stack, effectively clearing its contents. func (s *sstack) Reset() { - *s = *newSstack(s.setIndex) + *s = *newSstack(s.setIndex, false) } diff --git a/common/prque/sstack_test.go b/common/prque/sstack_test.go index 2ff093579d..bc6298979c 100644 --- a/common/prque/sstack_test.go +++ b/common/prque/sstack_test.go @@ -21,7 +21,7 @@ func TestSstack(t *testing.T) { for i := 0; i < size; i++ { data[i] = &item{rand.Int(), rand.Int63()} } - stack := newSstack(nil) + stack := newSstack(nil, false) for rep := 0; rep < 2; rep++ { // Push all the data into the stack, pop out every second secs := []*item{} @@ -55,7 +55,7 @@ func TestSstackSort(t *testing.T) { data[i] = &item{rand.Int(), int64(i)} } // Push all the data into the stack - stack := newSstack(nil) + stack := newSstack(nil, false) for _, val := range data { stack.Push(val) } @@ -76,7 +76,7 @@ func TestSstackReset(t *testing.T) { for i := 0; i < size; i++ { data[i] = &item{rand.Int(), rand.Int63()} } - stack := newSstack(nil) + stack := newSstack(nil, false) for rep := 0; rep < 2; rep++ { // Push all the data into the stack, pop out every second secs := []*item{} diff --git a/les/api.go b/les/api.go index a930524516..782bb31ef2 100644 --- a/les/api.go +++ b/les/api.go @@ -31,7 +31,6 @@ var ( errNoCheckpoint = errors.New("no local checkpoint provided") errNotActivated = errors.New("checkpoint registrar is not activated") errUnknownBenchmarkType = errors.New("unknown benchmark type") - errNoPriority = errors.New("priority too low to raise capacity") ) // PrivateLightServerAPI provides an API to access the LES light server. @@ -44,8 +43,8 @@ type PrivateLightServerAPI struct { func NewPrivateLightServerAPI(server *LesServer) *PrivateLightServerAPI { return &PrivateLightServerAPI{ server: server, - defaultPosFactors: server.clientPool.defaultPosFactors, - defaultNegFactors: server.clientPool.defaultNegFactors, + defaultPosFactors: defaultPosFactors, + defaultNegFactors: defaultNegFactors, } } @@ -66,7 +65,9 @@ func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { res := make(map[string]interface{}) res["minimumCapacity"] = api.server.minCapacity res["maximumCapacity"] = api.server.maxCapacity - res["totalCapacity"], res["totalConnectedCapacity"], res["priorityConnectedCapacity"] = api.server.clientPool.capacityInfo() + _, res["totalCapacity"] = api.server.clientPool.Limits() + _, res["totalConnectedCapacity"] = api.server.clientPool.Active() + res["priorityConnectedCapacity"] = 0 //TODO connect when token sale module is added return res } @@ -80,9 +81,18 @@ func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[st } res := make(map[enode.ID]map[string]interface{}) - api.server.clientPool.forClients(ids, func(client *clientInfo) { - res[client.node.ID()] = api.clientInfo(client) - }) + if len(ids) == 0 { + ids = api.server.peers.ids() + } + for _, id := range ids { + if peer := api.server.peers.peer(id); peer != nil { + res[id] = api.clientInfo(peer, peer.balance) + } else { + api.server.clientPool.BalanceOperation(id, "", func(balance vfs.AtomicBalanceOperator) { + res[id] = api.clientInfo(nil, balance) + }) + } + } return res } @@ -94,31 +104,35 @@ func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[st // assigned to it. func (api *PrivateLightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCount int) map[enode.ID]map[string]interface{} { res := make(map[enode.ID]map[string]interface{}) - ids := api.server.clientPool.bt.GetPosBalanceIDs(start, stop, maxCount+1) + ids := api.server.clientPool.GetPosBalanceIDs(start, stop, maxCount+1) if len(ids) > maxCount { res[ids[maxCount]] = make(map[string]interface{}) ids = ids[:maxCount] } - if len(ids) != 0 { - api.server.clientPool.forClients(ids, func(client *clientInfo) { - res[client.node.ID()] = api.clientInfo(client) - }) + for _, id := range ids { + if peer := api.server.peers.peer(id); peer != nil { + res[id] = api.clientInfo(peer, peer.balance) + } else { + api.server.clientPool.BalanceOperation(id, "", func(balance vfs.AtomicBalanceOperator) { + res[id] = api.clientInfo(nil, balance) + }) + } } return res } // clientInfo creates a client info data structure -func (api *PrivateLightServerAPI) clientInfo(c *clientInfo) map[string]interface{} { +func (api *PrivateLightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadOnlyBalance) map[string]interface{} { info := make(map[string]interface{}) - pb, nb := c.balance.GetBalance() - info["isConnected"] = c.connected + pb, nb := balance.GetBalance() + info["isConnected"] = peer != nil info["pricing/balance"] = pb info["priority"] = pb != 0 // cb := api.server.clientPool.ndb.getCurrencyBalance(id) // info["pricing/currency"] = cb.amount - if c.connected { - info["connectionTime"] = float64(mclock.Now()-c.connectedAt) / float64(time.Second) - info["capacity"], _ = api.server.clientPool.ns.GetField(c.node, priorityPoolSetup.CapacityField).(uint64) + if peer != nil { + info["connectionTime"] = float64(mclock.Now()-peer.connectedAt) / float64(time.Second) + info["capacity"] = peer.getCapacity() info["pricing/negBalance"] = nb } return info @@ -126,7 +140,7 @@ func (api *PrivateLightServerAPI) clientInfo(c *clientInfo) map[string]interface // setParams either sets the given parameters for a single connected client (if specified) // or the default parameters applicable to clients connected in the future -func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientInfo, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { +func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientPeer, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { defParams := client == nil for name, value := range params { errValue := func() error { @@ -156,9 +170,8 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien setFactor(&negFactors.RequestFactor) case !defParams && name == "capacity": if capacity, ok := value.(float64); ok && uint64(capacity) >= api.server.minCapacity { - _, err = api.server.clientPool.setCapacity(client.node, client.address, uint64(capacity), 0, true) - // Don't have to call factor update explicitly. It's already done - // in setCapacity function. + _, err = api.server.clientPool.SetCapacity(client.Node(), uint64(capacity), 0, false) + // time factor recalculation is performed automatically by the balance tracker } else { err = errValue() } @@ -179,31 +192,25 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien // SetClientParams sets client parameters for all clients listed in the ids list // or all connected clients if the list is empty func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[string]interface{}) error { - var ( - ids []enode.ID - err error - ) + var err error for _, node := range nodes { - if id, err := parseNode(node); err != nil { + var id enode.ID + if id, err = parseNode(node); err != nil { return err - } else { - ids = append(ids, id) } - } - api.server.clientPool.forClients(ids, func(client *clientInfo) { - if client.connected { - posFactors, negFactors := client.balance.GetPriceFactors() - update, e := api.setParams(params, client, &posFactors, &negFactors) + if peer := api.server.peers.peer(id); peer != nil { + posFactors, negFactors := peer.balance.GetPriceFactors() + update, e := api.setParams(params, peer, &posFactors, &negFactors) if update { - client.balance.SetPriceFactors(posFactors, negFactors) + peer.balance.SetPriceFactors(posFactors, negFactors) } if e != nil { err = e } } else { - err = fmt.Errorf("client %064x is not connected", client.node.ID()) + err = fmt.Errorf("client %064x is not connected", id) } - }) + } return err } @@ -211,7 +218,7 @@ func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[str func (api *PrivateLightServerAPI) SetDefaultParams(params map[string]interface{}) error { update, err := api.setParams(params, nil, &api.defaultPosFactors, &api.defaultNegFactors) if update { - api.server.clientPool.setDefaultFactors(api.defaultPosFactors, api.defaultNegFactors) + api.server.clientPool.SetDefaultFactors(api.defaultPosFactors, api.defaultNegFactors) } return err } @@ -224,7 +231,7 @@ func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error { if bias < time.Duration(0) { return fmt.Errorf("bias illegal: %v less than 0", bias) } - api.server.clientPool.setConnectedBias(bias) + api.server.clientPool.SetConnectedBias(bias) return nil } @@ -235,8 +242,8 @@ func (api *PrivateLightServerAPI) AddBalance(node string, amount int64) (balance if id, err = parseNode(node); err != nil { return } - api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo) { - balance[0], balance[1], err = c.balance.AddBalance(amount) + api.server.clientPool.BalanceOperation(id, "", func(nb vfs.AtomicBalanceOperator) { + balance[0], balance[1], err = nb.AddBalance(amount) }) return } @@ -338,14 +345,12 @@ func (api *PrivateDebugAPI) FreezeClient(node string) error { if id, err = parseNode(node); err != nil { return err } - api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo) { - if c.connected { - c.peer.freeze() - } else { - err = fmt.Errorf("client %064x is not connected", id[:]) - } - }) - return err + if peer := api.server.peers.peer(id); peer != nil { + peer.freeze() + return nil + } else { + return fmt.Errorf("client %064x is not connected", id[:]) + } } // PrivateLightAPI provides an API to access the LES light server or light client. diff --git a/les/clientpool.go b/les/clientpool.go deleted file mode 100644 index 3965d54508..0000000000 --- a/les/clientpool.go +++ /dev/null @@ -1,453 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "fmt" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/les/utils" - "github.com/ethereum/go-ethereum/les/vflux" - vfs "github.com/ethereum/go-ethereum/les/vflux/server" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/p2p/nodestate" - "github.com/ethereum/go-ethereum/rlp" -) - -const ( - defaultNegExpTC = 3600 // default time constant (in seconds) for exponentially reducing negative balance - - // defaultConnectedBias is applied to already connected clients So that - // already connected client won't be kicked out very soon and we - // can ensure all connected clients can have enough time to request - // or sync some data. - // - // todo(rjl493456442) make it configurable. It can be the option of - // free trial time! - defaultConnectedBias = time.Minute * 3 - inactiveTimeout = time.Second * 10 -) - -// clientPool implements a client database that assigns a priority to each client -// based on a positive and negative balance. Positive balance is externally assigned -// to prioritized clients and is decreased with connection time and processed -// requests (unless the price factors are zero). If the positive balance is zero -// then negative balance is accumulated. -// -// Balance tracking and priority calculation for connected clients is done by -// balanceTracker. activeQueue ensures that clients with the lowest positive or -// highest negative balance get evicted when the total capacity allowance is full -// and new clients with a better balance want to connect. -// -// Already connected nodes receive a small bias in their favor in order to avoid -// accepting and instantly kicking out clients. In theory, we try to ensure that -// each client can have several minutes of connection time. -// -// Balances of disconnected clients are stored in nodeDB including positive balance -// and negative banalce. Boeth positive balance and negative balance will decrease -// exponentially. If the balance is low enough, then the record will be dropped. -type clientPool struct { - vfs.BalanceTrackerSetup - vfs.PriorityPoolSetup - lock sync.Mutex - clock mclock.Clock - closed bool - removePeer func(enode.ID) - synced func() bool - ns *nodestate.NodeStateMachine - pp *vfs.PriorityPool - bt *vfs.BalanceTracker - - defaultPosFactors, defaultNegFactors vfs.PriceFactors - posExpTC, negExpTC uint64 - minCap uint64 // The minimal capacity value allowed for any client - connectedBias time.Duration - capLimit uint64 -} - -// clientPoolPeer represents a client peer in the pool. -// Positive balances are assigned to node key while negative balances are assigned -// to freeClientId. Currently network IP address without port is used because -// clients have a limited access to IP addresses while new node keys can be easily -// generated so it would be useless to assign a negative value to them. -type clientPoolPeer interface { - Node() *enode.Node - freeClientId() string - updateCapacity(uint64) - freeze() - allowInactive() bool -} - -// clientInfo defines all information required by clientpool. -type clientInfo struct { - node *enode.Node - address string - peer clientPoolPeer - connected, priority bool - connectedAt mclock.AbsTime - balance *vfs.NodeBalance -} - -// newClientPool creates a new client pool -func newClientPool(ns *nodestate.NodeStateMachine, lesDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID), synced func() bool) *clientPool { - pool := &clientPool{ - ns: ns, - BalanceTrackerSetup: balanceTrackerSetup, - PriorityPoolSetup: priorityPoolSetup, - clock: clock, - minCap: minCap, - connectedBias: connectedBias, - removePeer: removePeer, - synced: synced, - } - pool.bt = vfs.NewBalanceTracker(ns, balanceTrackerSetup, lesDb, clock, &utils.Expirer{}, &utils.Expirer{}) - pool.pp = vfs.NewPriorityPool(ns, priorityPoolSetup, clock, minCap, connectedBias, 4) - - // set default expiration constants used by tests - // Note: server overwrites this if token sale is active - pool.bt.SetExpirationTCs(0, defaultNegExpTC) - - ns.SubscribeState(pool.InactiveFlag.Or(pool.PriorityFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { - if newState.Equals(pool.InactiveFlag) { - ns.AddTimeout(node, pool.InactiveFlag, inactiveTimeout) - } - if oldState.Equals(pool.InactiveFlag) && newState.Equals(pool.InactiveFlag.Or(pool.PriorityFlag)) { - ns.SetStateSub(node, pool.InactiveFlag, nodestate.Flags{}, 0) // remove timeout - } - }) - - ns.SubscribeState(pool.ActiveFlag.Or(pool.PriorityFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { - c, _ := ns.GetField(node, clientInfoField).(*clientInfo) - if c == nil { - return - } - c.priority = newState.HasAll(pool.PriorityFlag) - if newState.Equals(pool.ActiveFlag) { - cap, _ := ns.GetField(node, pool.CapacityField).(uint64) - if cap > minCap { - pool.pp.RequestCapacity(node, minCap, 0, true) - } - } - }) - - ns.SubscribeState(pool.InactiveFlag.Or(pool.ActiveFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { - if oldState.IsEmpty() { - clientConnectedMeter.Mark(1) - log.Debug("Client connected", "id", node.ID()) - } - if oldState.Equals(pool.InactiveFlag) && newState.Equals(pool.ActiveFlag) { - clientActivatedMeter.Mark(1) - log.Debug("Client activated", "id", node.ID()) - } - if oldState.Equals(pool.ActiveFlag) && newState.Equals(pool.InactiveFlag) { - clientDeactivatedMeter.Mark(1) - log.Debug("Client deactivated", "id", node.ID()) - c, _ := ns.GetField(node, clientInfoField).(*clientInfo) - if c == nil || !c.peer.allowInactive() { - pool.removePeer(node.ID()) - } - } - if newState.IsEmpty() { - clientDisconnectedMeter.Mark(1) - log.Debug("Client disconnected", "id", node.ID()) - pool.removePeer(node.ID()) - } - }) - - var totalConnected uint64 - ns.SubscribeField(pool.CapacityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { - oldCap, _ := oldValue.(uint64) - newCap, _ := newValue.(uint64) - totalConnected += newCap - oldCap - totalConnectedGauge.Update(int64(totalConnected)) - c, _ := ns.GetField(node, clientInfoField).(*clientInfo) - if c != nil { - c.peer.updateCapacity(newCap) - } - }) - return pool -} - -// stop shuts the client pool down -func (f *clientPool) stop() { - f.lock.Lock() - f.closed = true - f.lock.Unlock() - f.ns.ForEach(nodestate.Flags{}, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { - // enforces saving all balances in BalanceTracker - f.disconnectNode(node) - }) - f.bt.Stop() -} - -// connect should be called after a successful handshake. If the connection was -// rejected, there is no need to call disconnect. -func (f *clientPool) connect(peer clientPoolPeer) (uint64, error) { - f.lock.Lock() - defer f.lock.Unlock() - - // Short circuit if clientPool is already closed. - if f.closed { - return 0, fmt.Errorf("Client pool is already closed") - } - // Dedup connected peers. - node, freeID := peer.Node(), peer.freeClientId() - if f.ns.GetField(node, clientInfoField) != nil { - log.Debug("Client already connected", "address", freeID, "id", node.ID().String()) - return 0, fmt.Errorf("Client already connected address=%s id=%s", freeID, node.ID().String()) - } - now := f.clock.Now() - c := &clientInfo{ - node: node, - address: freeID, - peer: peer, - connected: true, - connectedAt: now, - } - f.ns.SetField(node, clientInfoField, c) - f.ns.SetField(node, connAddressField, freeID) - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance == nil { - f.disconnect(peer) - return 0, nil - } - c.balance.SetPriceFactors(f.defaultPosFactors, f.defaultNegFactors) - - f.ns.SetState(node, f.InactiveFlag, nodestate.Flags{}, 0) - var allowed bool - f.ns.Operation(func() { - _, allowed = f.pp.RequestCapacity(node, f.minCap, f.connectedBias, true) - }) - if allowed { - return f.minCap, nil - } - if !peer.allowInactive() { - f.disconnect(peer) - } - return 0, nil -} - -// setConnectedBias sets the connection bias, which is applied to already connected clients -// So that already connected client won't be kicked out very soon and we can ensure all -// connected clients can have enough time to request or sync some data. -func (f *clientPool) setConnectedBias(bias time.Duration) { - f.lock.Lock() - defer f.lock.Unlock() - - f.connectedBias = bias - f.pp.SetActiveBias(bias) -} - -// disconnect should be called when a connection is terminated. If the disconnection -// was initiated by the pool itself using disconnectFn then calling disconnect is -// not necessary but permitted. -func (f *clientPool) disconnect(p clientPoolPeer) { - f.disconnectNode(p.Node()) -} - -// disconnectNode removes node fields and flags related to connected status -func (f *clientPool) disconnectNode(node *enode.Node) { - f.ns.SetField(node, connAddressField, nil) - f.ns.SetField(node, clientInfoField, nil) -} - -// setDefaultFactors sets the default price factors applied to subsequently connected clients -func (f *clientPool) setDefaultFactors(posFactors, negFactors vfs.PriceFactors) { - f.lock.Lock() - defer f.lock.Unlock() - - f.defaultPosFactors = posFactors - f.defaultNegFactors = negFactors -} - -// capacityInfo returns the total capacity allowance, the total capacity of connected -// clients and the total capacity of connected and prioritized clients -func (f *clientPool) capacityInfo() (uint64, uint64, uint64) { - f.lock.Lock() - defer f.lock.Unlock() - - // total priority active cap will be supported when the token issuer module is added - _, activeCap := f.pp.Active() - return f.capLimit, activeCap, 0 -} - -// setLimits sets the maximum number and total capacity of connected clients, -// dropping some of them if necessary. -func (f *clientPool) setLimits(totalConn int, totalCap uint64) { - f.lock.Lock() - defer f.lock.Unlock() - - f.capLimit = totalCap - f.pp.SetLimits(uint64(totalConn), totalCap) -} - -// setCapacity sets the assigned capacity of a connected client -func (f *clientPool) setCapacity(node *enode.Node, freeID string, capacity uint64, bias time.Duration, setCap bool) (uint64, error) { - c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo) - if c == nil { - if setCap { - return 0, fmt.Errorf("client %064x is not connected", node.ID()) - } - c = &clientInfo{node: node} - f.ns.SetField(node, clientInfoField, c) - f.ns.SetField(node, connAddressField, freeID) - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance == nil { - log.Error("BalanceField is missing", "node", node.ID()) - return 0, fmt.Errorf("BalanceField of %064x is missing", node.ID()) - } - defer func() { - f.ns.SetField(node, connAddressField, nil) - f.ns.SetField(node, clientInfoField, nil) - }() - } - var ( - minPriority int64 - allowed bool - ) - f.ns.Operation(func() { - if !setCap || c.priority { - // check clientInfo.priority inside Operation to ensure thread safety - minPriority, allowed = f.pp.RequestCapacity(node, capacity, bias, setCap) - } - }) - if allowed { - return 0, nil - } - missing := c.balance.PosBalanceMissing(minPriority, capacity, bias) - if missing < 1 { - // ensure that we never return 0 missing and insufficient priority error - missing = 1 - } - return missing, errNoPriority -} - -// setCapacityLocked is the equivalent of setCapacity used when f.lock is already locked -func (f *clientPool) setCapacityLocked(node *enode.Node, freeID string, capacity uint64, minConnTime time.Duration, setCap bool) (uint64, error) { - f.lock.Lock() - defer f.lock.Unlock() - - return f.setCapacity(node, freeID, capacity, minConnTime, setCap) -} - -// forClients calls the supplied callback for either the listed node IDs or all connected -// nodes. It passes a valid clientInfo to the callback and ensures that the necessary -// fields and flags are set in order for BalanceTracker and PriorityPool to work even if -// the node is not connected. -func (f *clientPool) forClients(ids []enode.ID, cb func(client *clientInfo)) { - f.lock.Lock() - defer f.lock.Unlock() - - if len(ids) == 0 { - f.ns.ForEach(nodestate.Flags{}, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { - c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo) - if c != nil { - cb(c) - } - }) - } else { - for _, id := range ids { - node := f.ns.GetNode(id) - if node == nil { - node = enode.SignNull(&enr.Record{}, id) - } - c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo) - if c != nil { - cb(c) - } else { - c = &clientInfo{node: node} - f.ns.SetField(node, clientInfoField, c) - f.ns.SetField(node, connAddressField, "") - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance != nil { - cb(c) - } else { - log.Error("BalanceField is missing") - } - f.ns.SetField(node, connAddressField, nil) - f.ns.SetField(node, clientInfoField, nil) - } - } - } -} - -// serveCapQuery serves a vflux capacity query. It receives multiple token amount values -// and a bias time value. For each given token amount it calculates the maximum achievable -// capacity in case the amount is added to the balance. -func (f *clientPool) serveCapQuery(id enode.ID, freeID string, data []byte) []byte { - var req vflux.CapacityQueryReq - if rlp.DecodeBytes(data, &req) != nil { - return nil - } - if l := len(req.AddTokens); l == 0 || l > vflux.CapacityQueryMaxLen { - return nil - } - result := make(vflux.CapacityQueryReply, len(req.AddTokens)) - if !f.synced() { - capacityQueryZeroMeter.Mark(1) - reply, _ := rlp.EncodeToBytes(&result) - return reply - } - - node := f.ns.GetNode(id) - if node == nil { - node = enode.SignNull(&enr.Record{}, id) - } - c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo) - if c == nil { - c = &clientInfo{node: node} - f.ns.SetField(node, clientInfoField, c) - f.ns.SetField(node, connAddressField, freeID) - defer func() { - f.ns.SetField(node, connAddressField, nil) - f.ns.SetField(node, clientInfoField, nil) - }() - if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*vfs.NodeBalance); c.balance == nil { - log.Error("BalanceField is missing", "node", node.ID()) - return nil - } - } - // use vfs.CapacityCurve to answer request for multiple newly bought token amounts - curve := f.pp.GetCapacityCurve().Exclude(id) - bias := time.Second * time.Duration(req.Bias) - if f.connectedBias > bias { - bias = f.connectedBias - } - pb, _ := c.balance.GetBalance() - for i, addTokens := range req.AddTokens { - add := addTokens.Int64() - result[i] = curve.MaxCapacity(func(capacity uint64) int64 { - return c.balance.EstimatePriority(capacity, add, 0, bias, false) / int64(capacity) - }) - if add <= 0 && uint64(-add) >= pb && result[i] > f.minCap { - result[i] = f.minCap - } - if result[i] < f.minCap { - result[i] = 0 - } - } - // add first result to metrics (don't care about priority client multi-queries yet) - if result[0] == 0 { - capacityQueryZeroMeter.Mark(1) - } else { - capacityQueryNonZeroMeter.Mark(1) - } - reply, _ := rlp.EncodeToBytes(&result) - return reply -} diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index d6d0b1adde..c9e681c144 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -108,7 +108,7 @@ type ClientManager struct { func NewClientManager(curve PieceWiseLinear, clock mclock.Clock) *ClientManager { cm := &ClientManager{ clock: clock, - rcQueue: prque.New(func(a interface{}, i int) { a.(*ClientNode).queueIndex = i }), + rcQueue: prque.NewWrapAround(func(a interface{}, i int) { a.(*ClientNode).queueIndex = i }), capLastUpdate: clock.Now(), stop: make(chan chan struct{}), } diff --git a/les/metrics.go b/les/metrics.go index 5a8d4bbe02..d356326b76 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -73,12 +73,9 @@ var ( serverConnectionGauge = metrics.NewRegisteredGauge("les/connection/server", nil) clientConnectionGauge = metrics.NewRegisteredGauge("les/connection/client", nil) - totalCapacityGauge = metrics.NewRegisteredGauge("les/server/totalCapacity", nil) - totalRechargeGauge = metrics.NewRegisteredGauge("les/server/totalRecharge", nil) - totalConnectedGauge = metrics.NewRegisteredGauge("les/server/totalConnected", nil) - blockProcessingTimer = metrics.NewRegisteredTimer("les/server/blockProcessingTime", nil) - capacityQueryZeroMeter = metrics.NewRegisteredMeter("les/server/capQueryZero", nil) - capacityQueryNonZeroMeter = metrics.NewRegisteredMeter("les/server/capQueryNonZero", nil) + totalCapacityGauge = metrics.NewRegisteredGauge("les/server/totalCapacity", nil) + totalRechargeGauge = metrics.NewRegisteredGauge("les/server/totalRecharge", nil) + blockProcessingTimer = metrics.NewRegisteredTimer("les/server/blockProcessingTime", nil) requestServedMeter = metrics.NewRegisteredMeter("les/server/req/avgServedTime", nil) requestServedTimer = metrics.NewRegisteredTimer("les/server/req/servedTime", nil) @@ -100,12 +97,8 @@ var ( sqServedGauge = metrics.NewRegisteredGauge("les/server/servingQueue/served", nil) sqQueuedGauge = metrics.NewRegisteredGauge("les/server/servingQueue/queued", nil) - clientConnectedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/connected", nil) - clientActivatedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/activated", nil) - clientDeactivatedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/deactivated", nil) - clientDisconnectedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/disconnected", nil) - clientFreezeMeter = metrics.NewRegisteredMeter("les/server/clientEvent/freeze", nil) - clientErrorMeter = metrics.NewRegisteredMeter("les/server/clientEvent/error", nil) + clientFreezeMeter = metrics.NewRegisteredMeter("les/server/clientEvent/freeze", nil) + clientErrorMeter = metrics.NewRegisteredMeter("les/server/clientEvent/error", nil) requestRTT = metrics.NewRegisteredTimer("les/client/req/rtt", nil) requestSendDelay = metrics.NewRegisteredTimer("les/client/req/sendDelay", nil) diff --git a/les/peer.go b/les/peer.go index 78019b1d87..f6cc94dfad 100644 --- a/les/peer.go +++ b/les/peer.go @@ -17,6 +17,7 @@ package les import ( + "crypto/ecdsa" "errors" "fmt" "math/big" @@ -37,6 +38,7 @@ import ( vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -762,15 +764,22 @@ type clientPeer struct { responseLock sync.Mutex responseCount uint64 // Counter to generate an unique id for request processing. - balance *vfs.NodeBalance + balance vfs.ConnectedBalance // invalidLock is used for protecting invalidCount. invalidLock sync.RWMutex invalidCount utils.LinearExpiredValue // Counter the invalid request the client peer has made. - server bool - errCh chan error - fcClient *flowcontrol.ClientNode // Server side mirror token bucket. + capacity uint64 + // lastAnnounce is the last broadcast created by the server; may be newer than the last head + // sent to the specific client (stored in headInfo) if capacity is zero. In this case the + // latest head is sent when the client gains non-zero capacity. + lastAnnounce announceData + + connectedAt mclock.AbsTime + server bool + errCh chan error + fcClient *flowcontrol.ClientNode // Server side mirror token bucket. } func newClientPeer(version int, network uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *clientPeer { @@ -789,9 +798,9 @@ func newClientPeer(version int, network uint64, p *p2p.Peer, rw p2p.MsgReadWrite } } -// freeClientId returns a string identifier for the peer. Multiple peers with +// FreeClientId returns a string identifier for the peer. Multiple peers with // the same identifier can not be connected in free mode simultaneously. -func (p *clientPeer) freeClientId() string { +func (p *clientPeer) FreeClientId() string { if addr, ok := p.RemoteAddr().(*net.TCPAddr); ok { if addr.IP.IsLoopback() { // using peer id instead of loopback ip address allows multiple free @@ -921,25 +930,69 @@ func (p *clientPeer) sendAnnounce(request announceData) error { return p2p.Send(p.rw, AnnounceMsg, request) } -// allowInactive implements clientPoolPeer -func (p *clientPeer) allowInactive() bool { - return false +// InactiveAllowance implements vfs.clientPeer +func (p *clientPeer) InactiveAllowance() time.Duration { + return 0 // will return more than zero for les/5 clients +} + +// getCapacity returns the current capacity of the peer +func (p *clientPeer) getCapacity() uint64 { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.capacity } -// updateCapacity updates the request serving capacity assigned to a given client -// and also sends an announcement about the updated flow control parameters -func (p *clientPeer) updateCapacity(cap uint64) { +// UpdateCapacity updates the request serving capacity assigned to a given client +// and also sends an announcement about the updated flow control parameters. +// Note: UpdateCapacity implements vfs.clientPeer and should not block. The requested +// parameter is true if the callback was initiated by ClientPool.SetCapacity on the given peer. +func (p *clientPeer) UpdateCapacity(newCap uint64, requested bool) { p.lock.Lock() defer p.lock.Unlock() - if cap != p.fcParams.MinRecharge { - p.fcParams = flowcontrol.ServerParams{MinRecharge: cap, BufLimit: cap * bufLimitRatio} + if newCap != p.fcParams.MinRecharge { + p.fcParams = flowcontrol.ServerParams{MinRecharge: newCap, BufLimit: newCap * bufLimitRatio} p.fcClient.UpdateParams(p.fcParams) var kvList keyValueList - kvList = kvList.add("flowControl/MRR", cap) - kvList = kvList.add("flowControl/BL", cap*bufLimitRatio) + kvList = kvList.add("flowControl/MRR", newCap) + kvList = kvList.add("flowControl/BL", newCap*bufLimitRatio) p.queueSend(func() { p.sendAnnounce(announceData{Update: kvList}) }) } + + if p.capacity == 0 && newCap != 0 { + p.sendLastAnnounce() + } + p.capacity = newCap +} + +// announceOrStore sends the given head announcement to the client if the client is +// active (capacity != 0) and the same announcement hasn't been sent before. If the +// client is inactive the announcement is stored and sent later if the client is +// activated again. +func (p *clientPeer) announceOrStore(announce announceData) { + p.lock.Lock() + defer p.lock.Unlock() + + p.lastAnnounce = announce + if p.capacity != 0 { + p.sendLastAnnounce() + } +} + +// announce sends the given head announcement to the client if it hasn't been sent before +func (p *clientPeer) sendLastAnnounce() { + if p.lastAnnounce.Td == nil { + return + } + if p.headInfo.Td == nil || p.lastAnnounce.Td.Cmp(p.headInfo.Td) > 0 { + if !p.queueSend(func() { p.sendAnnounce(p.lastAnnounce) }) { + p.Log().Debug("Dropped announcement because queue is full", "number", p.lastAnnounce.Number, "hash", p.lastAnnounce.Hash) + } else { + p.Log().Debug("Sent announcement", "number", p.lastAnnounce.Number, "hash", p.lastAnnounce.Hash) + } + p.headInfo = blockInfo{Hash: p.lastAnnounce.Hash, Number: p.lastAnnounce.Number, Td: p.lastAnnounce.Td} + } } // freezeClient temporarily puts the client in a frozen state which means all @@ -1064,6 +1117,11 @@ func (p *clientPeer) getInvalid() uint64 { return p.invalidCount.Value(mclock.Now()) } +// Disconnect implements vfs.clientPeer +func (p *clientPeer) Disconnect() { + p.Peer.Disconnect(p2p.DiscRequested) +} + // serverPeerSubscriber is an interface to notify services about added or // removed server peers type serverPeerSubscriber interface { @@ -1221,3 +1279,181 @@ func (ps *serverPeerSet) close() { } ps.closed = true } + +// clientPeerSet represents the set of active client peers currently +// participating in the Light Ethereum sub-protocol. +type clientPeerSet struct { + peers map[enode.ID]*clientPeer + lock sync.RWMutex + closed bool + + privateKey *ecdsa.PrivateKey + lastAnnounce, signedAnnounce announceData +} + +// newClientPeerSet creates a new peer set to track the client peers. +func newClientPeerSet() *clientPeerSet { + return &clientPeerSet{peers: make(map[enode.ID]*clientPeer)} +} + +// register adds a new peer into the peer set, or returns an error if the +// peer is already known. +func (ps *clientPeerSet) register(peer *clientPeer) error { + ps.lock.Lock() + defer ps.lock.Unlock() + + if ps.closed { + return errClosed + } + if _, exist := ps.peers[peer.ID()]; exist { + return errAlreadyRegistered + } + ps.peers[peer.ID()] = peer + ps.announceOrStore(peer) + return nil +} + +// unregister removes a remote peer from the peer set, disabling any further +// actions to/from that particular entity. It also initiates disconnection +// at the networking layer. +func (ps *clientPeerSet) unregister(id enode.ID) error { + ps.lock.Lock() + defer ps.lock.Unlock() + + p, ok := ps.peers[id] + if !ok { + return errNotRegistered + } + delete(ps.peers, id) + p.Peer.Disconnect(p2p.DiscRequested) + return nil +} + +// ids returns a list of all registered peer IDs +func (ps *clientPeerSet) ids() []enode.ID { + ps.lock.RLock() + defer ps.lock.RUnlock() + + var ids []enode.ID + for id := range ps.peers { + ids = append(ids, id) + } + return ids +} + +// peer retrieves the registered peer with the given id. +func (ps *clientPeerSet) peer(id enode.ID) *clientPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return ps.peers[id] +} + +// len returns if the current number of peers in the set. +func (ps *clientPeerSet) len() int { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return len(ps.peers) +} + +// setSignerKey sets the signer key for signed announcements. Should be called before +// starting the protocol handler. +func (ps *clientPeerSet) setSignerKey(privateKey *ecdsa.PrivateKey) { + ps.privateKey = privateKey +} + +// broadcast sends the given announcements to all active peers +func (ps *clientPeerSet) broadcast(announce announceData) { + ps.lock.Lock() + defer ps.lock.Unlock() + + ps.lastAnnounce = announce + for _, peer := range ps.peers { + ps.announceOrStore(peer) + } +} + +// announceOrStore sends the requested type of announcement to the given peer or stores +// it for later if the peer is inactive (capacity == 0). +func (ps *clientPeerSet) announceOrStore(p *clientPeer) { + if ps.lastAnnounce.Td == nil { + return + } + switch p.announceType { + case announceTypeSimple: + p.announceOrStore(ps.lastAnnounce) + case announceTypeSigned: + if ps.signedAnnounce.Hash != ps.lastAnnounce.Hash { + ps.signedAnnounce = ps.lastAnnounce + ps.signedAnnounce.sign(ps.privateKey) + } + p.announceOrStore(ps.signedAnnounce) + } +} + +// close disconnects all peers. No new peers can be registered +// after close has returned. +func (ps *clientPeerSet) close() { + ps.lock.Lock() + defer ps.lock.Unlock() + + for _, p := range ps.peers { + p.Peer.Disconnect(p2p.DiscQuitting) + } + ps.closed = true +} + +// serverSet is a special set which contains all connected les servers. +// Les servers will also be discovered by discovery protocol because they +// also run the LES protocol. We can't drop them although they are useless +// for us(server) but for other protocols(e.g. ETH) upon the devp2p they +// may be useful. +type serverSet struct { + lock sync.Mutex + set map[string]*clientPeer + closed bool +} + +func newServerSet() *serverSet { + return &serverSet{set: make(map[string]*clientPeer)} +} + +func (s *serverSet) register(peer *clientPeer) error { + s.lock.Lock() + defer s.lock.Unlock() + + if s.closed { + return errClosed + } + if _, exist := s.set[peer.id]; exist { + return errAlreadyRegistered + } + s.set[peer.id] = peer + return nil +} + +func (s *serverSet) unregister(peer *clientPeer) error { + s.lock.Lock() + defer s.lock.Unlock() + + if s.closed { + return errClosed + } + if _, exist := s.set[peer.id]; !exist { + return errNotRegistered + } + delete(s.set, peer.id) + peer.Peer.Disconnect(p2p.DiscQuitting) + return nil +} + +func (s *serverSet) close() { + s.lock.Lock() + defer s.lock.Unlock() + + for _, p := range s.set { + p.Peer.Disconnect(p2p.DiscQuitting) + } + s.closed = true +} diff --git a/les/server.go b/les/server.go index be64dfe190..d44b1b57d4 100644 --- a/les/server.go +++ b/les/server.go @@ -18,7 +18,6 @@ package les import ( "crypto/ecdsa" - "reflect" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -26,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" - "github.com/ethereum/go-ethereum/les/vflux" vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" @@ -34,24 +32,16 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/p2p/nodestate" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) var ( - serverSetup = &nodestate.Setup{} - clientPeerField = serverSetup.NewField("clientPeer", reflect.TypeOf(&clientPeer{})) - clientInfoField = serverSetup.NewField("clientInfo", reflect.TypeOf(&clientInfo{})) - connAddressField = serverSetup.NewField("connAddr", reflect.TypeOf("")) - balanceTrackerSetup = vfs.NewBalanceTrackerSetup(serverSetup) - priorityPoolSetup = vfs.NewPriorityPoolSetup(serverSetup) + defaultPosFactors = vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1} + defaultNegFactors = vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1} ) -func init() { - balanceTrackerSetup.Connect(connAddressField, priorityPoolSetup.CapacityField) - priorityPoolSetup.Connect(balanceTrackerSetup.BalanceField, balanceTrackerSetup.UpdateFlag) // NodeBalance implements nodePriority -} +const defaultConnectedBias = time.Minute * 3 type ethBackend interface { ArchiveMode() bool @@ -65,10 +55,10 @@ type ethBackend interface { type LesServer struct { lesCommons - ns *nodestate.NodeStateMachine archiveMode bool // Flag whether the ethereum node runs in archive mode. handler *serverHandler - broadcaster *broadcaster + peers *clientPeerSet + serverset *serverSet vfluxServer *vfs.Server privateKey *ecdsa.PrivateKey @@ -77,7 +67,7 @@ type LesServer struct { costTracker *costTracker defParams flowcontrol.ServerParams servingQueue *servingQueue - clientPool *clientPool + clientPool *vfs.ClientPool minCapacity, maxCapacity uint64 threadsIdle int // Request serving threads count when system is idle. @@ -91,7 +81,6 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les if err != nil { return nil, err } - ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup) // Calculate the number of threads used to service the light client // requests based on the user-specified value. threads := config.LightServ * 4 / 100 @@ -111,9 +100,9 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les bloomTrieIndexer: light.NewBloomTrieIndexer(e.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency, true), closeCh: make(chan struct{}), }, - ns: ns, archiveMode: e.ArchiveMode(), - broadcaster: newBroadcaster(ns), + peers: newClientPeerSet(), + serverset: newServerSet(), vfluxServer: vfs.NewServer(time.Millisecond * 10), fcManager: flowcontrol.NewClientManager(nil, &mclock.System{}), servingQueue: newServingQueue(int64(time.Millisecond*10), float64(config.LightServ)/100), @@ -121,7 +110,6 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les threadsIdle: threads, p2pSrv: node.Server(), } - srv.vfluxServer.Register(srv) issync := e.Synced if config.LightNoSyncServe { issync = func() bool { return true } @@ -149,8 +137,10 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les srv.maxCapacity = totalRecharge } srv.fcManager.SetCapacityLimits(srv.minCapacity, srv.maxCapacity, srv.minCapacity*2) - srv.clientPool = newClientPool(ns, lesDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient, issync) - srv.clientPool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}) + srv.clientPool = vfs.NewClientPool(lesDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, issync) + srv.clientPool.Start() + srv.clientPool.SetDefaultFactors(defaultPosFactors, defaultNegFactors) + srv.vfluxServer.Register(srv.clientPool, "les", "Ethereum light client service") checkpoint := srv.latestLocalCheckpoint() if !checkpoint.Empty() { @@ -162,14 +152,6 @@ func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*Les node.RegisterProtocols(srv.Protocols()) node.RegisterAPIs(srv.APIs()) node.RegisterLifecycle(srv) - - // disconnect all peers at nsm shutdown - ns.SubscribeField(clientPeerField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { - if state.Equals(serverSetup.OfflineFlag()) && oldValue != nil { - oldValue.(*clientPeer).Peer.Disconnect(p2p.DiscRequested) - } - }) - ns.Start() return srv, nil } @@ -198,7 +180,7 @@ func (s *LesServer) APIs() []rpc.API { func (s *LesServer) Protocols() []p2p.Protocol { ps := s.makeProtocols(ServerProtocolVersions, s.handler.runPeer, func(id enode.ID) interface{} { - if p := s.getClient(id); p != nil { + if p := s.peers.peer(id); p != nil { return p.Info() } return nil @@ -215,7 +197,7 @@ func (s *LesServer) Protocols() []p2p.Protocol { // Start starts the LES server func (s *LesServer) Start() error { s.privateKey = s.p2pSrv.PrivateKey - s.broadcaster.setSignerKey(s.privateKey) + s.peers.setSignerKey(s.privateKey) s.handler.start() s.wg.Add(1) go s.capacityManagement() @@ -229,8 +211,9 @@ func (s *LesServer) Start() error { func (s *LesServer) Stop() error { close(s.closeCh) - s.clientPool.stop() - s.ns.Stop() + s.clientPool.Stop() + s.serverset.close() + s.peers.close() s.fcManager.Stop() s.costTracker.stop() s.handler.stop() @@ -261,7 +244,7 @@ func (s *LesServer) capacityManagement() { totalCapacityCh := make(chan uint64, 100) totalCapacity := s.fcManager.SubscribeTotalCapacity(totalCapacityCh) - s.clientPool.setLimits(s.config.LightPeers, totalCapacity) + s.clientPool.SetLimits(uint64(s.config.LightPeers), totalCapacity) var ( busy bool @@ -298,39 +281,9 @@ func (s *LesServer) capacityManagement() { log.Warn("Reduced free peer connections", "from", freePeers, "to", newFreePeers) } freePeers = newFreePeers - s.clientPool.setLimits(s.config.LightPeers, totalCapacity) + s.clientPool.SetLimits(uint64(s.config.LightPeers), totalCapacity) case <-s.closeCh: return } } } - -func (s *LesServer) getClient(id enode.ID) *clientPeer { - if node := s.ns.GetNode(id); node != nil { - if p, ok := s.ns.GetField(node, clientPeerField).(*clientPeer); ok { - return p - } - } - return nil -} - -func (s *LesServer) dropClient(id enode.ID) { - if p := s.getClient(id); p != nil { - p.Peer.Disconnect(p2p.DiscRequested) - } -} - -// ServiceInfo implements vfs.Service -func (s *LesServer) ServiceInfo() (string, string) { - return "les", "Ethereum light client service" -} - -// Handle implements vfs.Service -func (s *LesServer) Handle(id enode.ID, address string, name string, data []byte) []byte { - switch name { - case vflux.CapacityQueryName: - return s.clientPool.serveCapQuery(id, address, data) - default: - return nil - } -} diff --git a/les/server_handler.go b/les/server_handler.go index 7651d03cab..5e12136d90 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -17,7 +17,6 @@ package les import ( - "crypto/ecdsa" "errors" "sync" "sync/atomic" @@ -31,13 +30,10 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/nodestate" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -59,7 +55,6 @@ const ( var ( errTooManyInvalidRequest = errors.New("too many invalid requests made") - errFullClientPool = errors.New("client pool is full") ) // serverHandler is responsible for serving light client and process @@ -128,32 +123,18 @@ func (h *serverHandler) handle(p *clientPeer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } - // Reject the duplicated peer, otherwise register it to peerset. - var registered bool - if err := h.server.ns.Operation(func() { - if h.server.ns.GetField(p.Node(), clientPeerField) != nil { - registered = true - } else { - h.server.ns.SetFieldSub(p.Node(), clientPeerField, p) - } - }); err != nil { - return err - } - if registered { - return errAlreadyRegistered - } - defer func() { - h.server.ns.SetField(p.Node(), clientPeerField, nil) - if p.fcClient != nil { // is nil when connecting another server - p.fcClient.Disconnect() - } - }() if p.server { + if err := h.server.serverset.register(p); err != nil { + return err + } // connected to another server, no messages expected, just wait for disconnection _, err := p.rw.ReadMsg() + h.server.serverset.unregister(p) return err } + defer p.fcClient.Disconnect() // set by handshake if it's not another server + // Reject light clients if server is not synced. // // Put this checking here, so that "non-synced" les-server peers are still allowed @@ -162,30 +143,31 @@ func (h *serverHandler) handle(p *clientPeer) error { p.Log().Debug("Light server not synced, rejecting peer") return p2p.DiscRequested } - // Disconnect the inbound peer if it's rejected by clientPool - if cap, err := h.server.clientPool.connect(p); cap != p.fcParams.MinRecharge || err != nil { - p.Log().Debug("Light Ethereum peer rejected", "err", errFullClientPool) - return errFullClientPool + if err := h.server.peers.register(p); err != nil { + return err } - p.balance, _ = h.server.ns.GetField(p.Node(), h.server.clientPool.BalanceField).(*vfs.NodeBalance) - if p.balance == nil { + if p.balance = h.server.clientPool.Register(p); p.balance == nil { + h.server.peers.unregister(p.ID()) + p.Log().Debug("Client pool already closed") return p2p.DiscRequested } - activeCount, _ := h.server.clientPool.pp.Active() + activeCount, _ := h.server.clientPool.Active() clientConnectionGauge.Update(int64(activeCount)) + p.connectedAt = mclock.Now() var wg sync.WaitGroup // Wait group used to track all in-flight task routines. - connectedAt := mclock.Now() defer func() { wg.Wait() // Ensure all background task routines have exited. - h.server.clientPool.disconnect(p) + h.server.clientPool.Unregister(p) + h.server.peers.unregister(p.ID()) p.balance = nil - activeCount, _ := h.server.clientPool.pp.Active() + activeCount, _ := h.server.clientPool.Active() clientConnectionGauge.Update(int64(activeCount)) - connectionTimer.Update(time.Duration(mclock.Now() - connectedAt)) + connectionTimer.Update(time.Duration(mclock.Now() - p.connectedAt)) }() - // Mark the peer starts to be served. + + // Mark the peer as being served. atomic.StoreUint32(&p.serving, 1) defer atomic.StoreUint32(&p.serving, 0) @@ -448,78 +430,9 @@ func (h *serverHandler) broadcastLoop() { } lastHead, lastTd = header, td log.Debug("Announcing block to peers", "number", number, "hash", hash, "td", td, "reorg", reorg) - h.server.broadcaster.broadcast(announceData{Hash: hash, Number: number, Td: td, ReorgDepth: reorg}) + h.server.peers.broadcast(announceData{Hash: hash, Number: number, Td: td, ReorgDepth: reorg}) case <-h.closeCh: return } } } - -// broadcaster sends new header announcements to active client peers -type broadcaster struct { - ns *nodestate.NodeStateMachine - privateKey *ecdsa.PrivateKey - lastAnnounce, signedAnnounce announceData -} - -// newBroadcaster creates a new broadcaster -func newBroadcaster(ns *nodestate.NodeStateMachine) *broadcaster { - b := &broadcaster{ns: ns} - ns.SubscribeState(priorityPoolSetup.ActiveFlag, func(node *enode.Node, oldState, newState nodestate.Flags) { - if newState.Equals(priorityPoolSetup.ActiveFlag) { - // send last announcement to activated peers - b.sendTo(node) - } - }) - return b -} - -// setSignerKey sets the signer key for signed announcements. Should be called before -// starting the protocol handler. -func (b *broadcaster) setSignerKey(privateKey *ecdsa.PrivateKey) { - b.privateKey = privateKey -} - -// broadcast sends the given announcements to all active peers -func (b *broadcaster) broadcast(announce announceData) { - b.ns.Operation(func() { - // iterate in an Operation to ensure that the active set does not change while iterating - b.lastAnnounce = announce - b.ns.ForEach(priorityPoolSetup.ActiveFlag, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { - b.sendTo(node) - }) - }) -} - -// sendTo sends the most recent announcement to the given node unless the same or higher Td -// announcement has already been sent. -func (b *broadcaster) sendTo(node *enode.Node) { - if b.lastAnnounce.Td == nil { - return - } - if p, _ := b.ns.GetField(node, clientPeerField).(*clientPeer); p != nil { - if p.headInfo.Td == nil || b.lastAnnounce.Td.Cmp(p.headInfo.Td) > 0 { - announce := b.lastAnnounce - switch p.announceType { - case announceTypeSimple: - if !p.queueSend(func() { p.sendAnnounce(announce) }) { - log.Debug("Drop announcement because queue is full", "number", announce.Number, "hash", announce.Hash) - } else { - log.Debug("Sent announcement", "number", announce.Number, "hash", announce.Hash) - } - case announceTypeSigned: - if b.signedAnnounce.Hash != b.lastAnnounce.Hash { - b.signedAnnounce = b.lastAnnounce - b.signedAnnounce.sign(b.privateKey) - } - announce := b.signedAnnounce - if !p.queueSend(func() { p.sendAnnounce(announce) }) { - log.Debug("Drop announcement because queue is full", "number", announce.Number, "hash", announce.Hash) - } else { - log.Debug("Sent announcement", "number", announce.Number, "hash", announce.Hash) - } - } - p.headInfo = blockInfo{b.lastAnnounce.Hash, b.lastAnnounce.Number, b.lastAnnounce.Td} - } - } -} diff --git a/les/servingqueue.go b/les/servingqueue.go index 9db84e6159..16e064cb3f 100644 --- a/les/servingqueue.go +++ b/les/servingqueue.go @@ -123,7 +123,7 @@ func (t *servingTask) waitOrStop() bool { // newServingQueue returns a new servingQueue func newServingQueue(suspendBias int64, utilTarget float64) *servingQueue { sq := &servingQueue{ - queue: prque.New(nil), + queue: prque.NewWrapAround(nil), suspendBias: suspendBias, queueAddCh: make(chan *servingTask, 100), queueBestCh: make(chan *servingTask), @@ -279,7 +279,7 @@ func (sq *servingQueue) updateRecentTime() { func (sq *servingQueue) addTask(task *servingTask) { if sq.best == nil { sq.best = task - } else if task.priority > sq.best.priority { + } else if task.priority-sq.best.priority > 0 { sq.queue.Push(sq.best, sq.best.priority) sq.best = task } else { diff --git a/les/test_helper.go b/les/test_helper.go index e49bfc8738..ee2da2f8eb 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -45,10 +45,10 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/les/checkpointoracle" "github.com/ethereum/go-ethereum/les/flowcontrol" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/nodestate" "github.com/ethereum/go-ethereum/params" ) @@ -284,7 +284,6 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da } oracle = checkpointoracle.New(checkpointConfig, getLocal) } - ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup) server := &LesServer{ lesCommons: lesCommons{ genesis: genesis.Hash(), @@ -296,8 +295,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da oracle: oracle, closeCh: make(chan struct{}), }, - ns: ns, - broadcaster: newBroadcaster(ns), + peers: newClientPeerSet(), servingQueue: newServingQueue(int64(time.Millisecond*10), 1), defParams: flowcontrol.ServerParams{ BufLimit: testBufLimit, @@ -307,14 +305,14 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da } server.costTracker, server.minCapacity = newCostTracker(db, server.config) server.costTracker.testCostList = testCostList(0) // Disable flow control mechanism. - server.clientPool = newClientPool(ns, db, testBufRecharge, defaultConnectedBias, clock, func(id enode.ID) {}, alwaysTrueFn) - server.clientPool.setLimits(10000, 10000) // Assign enough capacity for clientpool + server.clientPool = vfs.NewClientPool(db, testBufRecharge, defaultConnectedBias, clock, alwaysTrueFn) + server.clientPool.Start() + server.clientPool.SetLimits(10000, 10000) // Assign enough capacity for clientpool server.handler = newServerHandler(server, simulation.Blockchain(), db, txpool, func() bool { return true }) if server.oracle != nil { server.oracle.Start(simulation) } server.servingQueue.setThreads(4) - ns.Start() server.handler.start() return server.handler, simulation } diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index db12a5c573..01e645a16a 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -47,21 +47,57 @@ type PriceFactors struct { TimeFactor, CapacityFactor, RequestFactor float64 } -// timePrice returns the price of connection per nanosecond at the given capacity -func (p PriceFactors) timePrice(cap uint64) float64 { - return p.TimeFactor + float64(cap)*p.CapacityFactor/1000000 +// connectionPrice returns the price of connection per nanosecond at the given capacity +// and the estimated average request cost. +func (p PriceFactors) connectionPrice(cap uint64, avgReqCost float64) float64 { + return p.TimeFactor + float64(cap)*p.CapacityFactor/1000000 + p.RequestFactor*avgReqCost } -// NodeBalance keeps track of the positive and negative balances of a connected +type ( + // nodePriority interface provides current and estimated future priorities on demand + nodePriority interface { + // priority should return the current priority of the node (higher is better) + priority(cap uint64) int64 + // estimatePriority should return a lower estimate for the minimum of the node priority + // value starting from the current moment until the given time. If the priority goes + // under the returned estimate before the specified moment then it is the caller's + // responsibility to signal with updateFlag. + estimatePriority(cap uint64, addBalance int64, future, bias time.Duration, update bool) int64 + } + + // ReadOnlyBalance provides read-only operations on the node balance + ReadOnlyBalance interface { + nodePriority + GetBalance() (uint64, uint64) + GetRawBalance() (utils.ExpiredValue, utils.ExpiredValue) + GetPriceFactors() (posFactor, negFactor PriceFactors) + } + + // ConnectedBalance provides operations permitted on connected nodes (non-read-only + // operations are not permitted inside a BalanceOperation) + ConnectedBalance interface { + ReadOnlyBalance + SetPriceFactors(posFactor, negFactor PriceFactors) + RequestServed(cost uint64) uint64 + } + + // AtomicBalanceOperator provides operations permitted in an atomic BalanceOperation + AtomicBalanceOperator interface { + ReadOnlyBalance + AddBalance(amount int64) (uint64, uint64, error) + SetBalance(pos, neg uint64) error + } +) + +// nodeBalance keeps track of the positive and negative balances of a connected // client and calculates actual and projected future priority values. // Implements nodePriority interface. -type NodeBalance struct { - bt *BalanceTracker +type nodeBalance struct { + bt *balanceTracker lock sync.RWMutex node *enode.Node connAddress string - active bool - priority bool + active, hasPriority, setFlags bool capacity uint64 balance balance posFactor, negFactor PriceFactors @@ -78,7 +114,62 @@ type NodeBalance struct { // balance represents a pair of positive and negative balances type balance struct { - pos, neg utils.ExpiredValue + pos, neg utils.ExpiredValue + posExp, negExp utils.ValueExpirer +} + +// posValue returns the value of positive balance at a given timestamp. +func (b balance) posValue(now mclock.AbsTime) uint64 { + return b.pos.Value(b.posExp.LogOffset(now)) +} + +// negValue returns the value of negative balance at a given timestamp. +func (b balance) negValue(now mclock.AbsTime) uint64 { + return b.neg.Value(b.negExp.LogOffset(now)) +} + +// addValue adds the value of a given amount to the balance. The original value and +// updated value will also be returned if the addition is successful. +// Returns the error if the given value is too large and the value overflows. +func (b *balance) addValue(now mclock.AbsTime, amount int64, pos bool, force bool) (uint64, uint64, int64, error) { + var ( + val utils.ExpiredValue + offset utils.Fixed64 + ) + if pos { + offset, val = b.posExp.LogOffset(now), b.pos + } else { + offset, val = b.negExp.LogOffset(now), b.neg + } + old := val.Value(offset) + if amount > 0 && (amount > maxBalance || old > maxBalance-uint64(amount)) { + if !force { + return old, 0, 0, errBalanceOverflow + } + val = utils.ExpiredValue{} + amount = maxBalance + } + net := val.Add(amount, offset) + if pos { + b.pos = val + } else { + b.neg = val + } + return old, val.Value(offset), net, nil +} + +// setValue sets the internal balance amount to the given values. Returns the +// error if the given value is too large. +func (b *balance) setValue(now mclock.AbsTime, pos uint64, neg uint64) error { + if pos > maxBalance || neg > maxBalance { + return errBalanceOverflow + } + var pb, nb utils.ExpiredValue + pb.Add(int64(pos), b.posExp.LogOffset(now)) + nb.Add(int64(neg), b.negExp.LogOffset(now)) + b.pos = pb + b.neg = nb + return nil } // balanceCallback represents a single callback that is activated when client priority @@ -90,18 +181,18 @@ type balanceCallback struct { } // GetBalance returns the current positive and negative balance. -func (n *NodeBalance) GetBalance() (uint64, uint64) { +func (n *nodeBalance) GetBalance() (uint64, uint64) { n.lock.Lock() defer n.lock.Unlock() now := n.bt.clock.Now() n.updateBalance(now) - return n.balance.pos.Value(n.bt.posExp.LogOffset(now)), n.balance.neg.Value(n.bt.negExp.LogOffset(now)) + return n.balance.posValue(now), n.balance.negValue(now) } // GetRawBalance returns the current positive and negative balance // but in the raw(expired value) format. -func (n *NodeBalance) GetRawBalance() (utils.ExpiredValue, utils.ExpiredValue) { +func (n *nodeBalance) GetRawBalance() (utils.ExpiredValue, utils.ExpiredValue) { n.lock.Lock() defer n.lock.Unlock() @@ -114,164 +205,147 @@ func (n *NodeBalance) GetRawBalance() (utils.ExpiredValue, utils.ExpiredValue) { // before and after the operation. Exceeding maxBalance results in an error (balance is // unchanged) while adding a negative amount higher than the current balance results in // zero balance. -func (n *NodeBalance) AddBalance(amount int64) (uint64, uint64, error) { +// Note: this function should run inside a NodeStateMachine operation +func (n *nodeBalance) AddBalance(amount int64) (uint64, uint64, error) { var ( - err error - old, new uint64 + err error + old, new uint64 + now = n.bt.clock.Now() + callbacks []func() + setPriority bool ) - n.bt.ns.Operation(func() { - var ( - callbacks []func() - setPriority bool - ) - n.bt.updateTotalBalance(n, func() bool { - now := n.bt.clock.Now() - n.updateBalance(now) - - // Ensure the given amount is valid to apply. - offset := n.bt.posExp.LogOffset(now) - old = n.balance.pos.Value(offset) - if amount > 0 && (amount > maxBalance || old > maxBalance-uint64(amount)) { - err = errBalanceOverflow - return false - } - - // Update the total positive balance counter. - n.balance.pos.Add(amount, offset) - callbacks = n.checkCallbacks(now) - setPriority = n.checkPriorityStatus() - new = n.balance.pos.Value(offset) - n.storeBalance(true, false) - return true - }) - for _, cb := range callbacks { - cb() - } - if setPriority { - n.bt.ns.SetStateSub(n.node, n.bt.PriorityFlag, nodestate.Flags{}, 0) + // Operation with holding the lock + n.bt.updateTotalBalance(n, func() bool { + n.updateBalance(now) + if old, new, _, err = n.balance.addValue(now, amount, true, false); err != nil { + return false } - n.signalPriorityUpdate() + callbacks, setPriority = n.checkCallbacks(now), n.checkPriorityStatus() + n.storeBalance(true, false) + return true }) if err != nil { return old, old, err } - + // Operation without holding the lock + for _, cb := range callbacks { + cb() + } + if n.setFlags { + if setPriority { + n.bt.ns.SetStateSub(n.node, n.bt.setup.priorityFlag, nodestate.Flags{}, 0) + } + // Note: priority flag is automatically removed by the zero priority callback if necessary + n.signalPriorityUpdate() + } return old, new, nil } // SetBalance sets the positive and negative balance to the given values -func (n *NodeBalance) SetBalance(pos, neg uint64) error { - if pos > maxBalance || neg > maxBalance { - return errBalanceOverflow - } - n.bt.ns.Operation(func() { - var ( - callbacks []func() - setPriority bool - ) - n.bt.updateTotalBalance(n, func() bool { - now := n.bt.clock.Now() - n.updateBalance(now) - - var pb, nb utils.ExpiredValue - pb.Add(int64(pos), n.bt.posExp.LogOffset(now)) - nb.Add(int64(neg), n.bt.negExp.LogOffset(now)) - n.balance.pos = pb - n.balance.neg = nb - callbacks = n.checkCallbacks(now) - setPriority = n.checkPriorityStatus() - n.storeBalance(true, true) - return true - }) - for _, cb := range callbacks { - cb() +// Note: this function should run inside a NodeStateMachine operation +func (n *nodeBalance) SetBalance(pos, neg uint64) error { + var ( + now = n.bt.clock.Now() + callbacks []func() + setPriority bool + ) + // Operation with holding the lock + n.bt.updateTotalBalance(n, func() bool { + n.updateBalance(now) + if err := n.balance.setValue(now, pos, neg); err != nil { + return false } + callbacks, setPriority = n.checkCallbacks(now), n.checkPriorityStatus() + n.storeBalance(true, true) + return true + }) + // Operation without holding the lock + for _, cb := range callbacks { + cb() + } + if n.setFlags { if setPriority { - n.bt.ns.SetStateSub(n.node, n.bt.PriorityFlag, nodestate.Flags{}, 0) + n.bt.ns.SetStateSub(n.node, n.bt.setup.priorityFlag, nodestate.Flags{}, 0) } + // Note: priority flag is automatically removed by the zero priority callback if necessary n.signalPriorityUpdate() - }) + } return nil } // RequestServed should be called after serving a request for the given peer -func (n *NodeBalance) RequestServed(cost uint64) uint64 { +func (n *nodeBalance) RequestServed(cost uint64) (newBalance uint64) { n.lock.Lock() - var callbacks []func() - defer func() { - n.lock.Unlock() - if callbacks != nil { - n.bt.ns.Operation(func() { - for _, cb := range callbacks { - cb() - } - }) - } - }() - now := n.bt.clock.Now() + var ( + check bool + fcost = float64(cost) + now = n.bt.clock.Now() + ) n.updateBalance(now) - fcost := float64(cost) - - posExp := n.bt.posExp.LogOffset(now) - var check bool if !n.balance.pos.IsZero() { - if n.posFactor.RequestFactor != 0 { - c := -int64(fcost * n.posFactor.RequestFactor) - cc := n.balance.pos.Add(c, posExp) - if c == cc { + posCost := -int64(fcost * n.posFactor.RequestFactor) + if posCost == 0 { + fcost = 0 + newBalance = n.balance.posValue(now) + } else { + var net int64 + _, newBalance, net, _ = n.balance.addValue(now, posCost, true, false) + if posCost == net { fcost = 0 } else { - fcost *= 1 - float64(cc)/float64(c) + fcost *= 1 - float64(net)/float64(posCost) } check = true - } else { - fcost = 0 } } - if fcost > 0 { - if n.negFactor.RequestFactor != 0 { - n.balance.neg.Add(int64(fcost*n.negFactor.RequestFactor), n.bt.negExp.LogOffset(now)) - check = true - } + if fcost > 0 && n.negFactor.RequestFactor != 0 { + n.balance.addValue(now, int64(fcost*n.negFactor.RequestFactor), false, false) + check = true } + n.sumReqCost += cost + + var callbacks []func() if check { callbacks = n.checkCallbacks(now) } - n.sumReqCost += cost - return n.balance.pos.Value(posExp) + n.lock.Unlock() + + if callbacks != nil { + n.bt.ns.Operation(func() { + for _, cb := range callbacks { + cb() + } + }) + } + return } -// Priority returns the actual priority based on the current balance -func (n *NodeBalance) Priority(capacity uint64) int64 { +// priority returns the actual priority based on the current balance +func (n *nodeBalance) priority(capacity uint64) int64 { n.lock.Lock() defer n.lock.Unlock() - n.updateBalance(n.bt.clock.Now()) - return n.balanceToPriority(n.balance, capacity) + now := n.bt.clock.Now() + n.updateBalance(now) + return n.balanceToPriority(now, n.balance, capacity) } // EstMinPriority gives a lower estimate for the priority at a given time in the future. // An average request cost per time is assumed that is twice the average cost per time // in the current session. -// If update is true then a priority callback is added that turns UpdateFlag on and off +// If update is true then a priority callback is added that turns updateFlag on and off // in case the priority goes below the estimated minimum. -func (n *NodeBalance) EstimatePriority(capacity uint64, addBalance int64, future, bias time.Duration, update bool) int64 { +func (n *nodeBalance) estimatePriority(capacity uint64, addBalance int64, future, bias time.Duration, update bool) int64 { n.lock.Lock() defer n.lock.Unlock() now := n.bt.clock.Now() n.updateBalance(now) - b := n.balance + + b := n.balance // copy the balance if addBalance != 0 { - offset := n.bt.posExp.LogOffset(now) - old := n.balance.pos.Value(offset) - if addBalance > 0 && (addBalance > maxBalance || old > maxBalance-uint64(addBalance)) { - b.pos = utils.ExpiredValue{} - b.pos.Add(maxBalance, offset) - } else { - b.pos.Add(addBalance, offset) - } + b.addValue(now, addBalance, true, true) } if future > 0 { var avgReqCost float64 @@ -284,52 +358,20 @@ func (n *NodeBalance) EstimatePriority(capacity uint64, addBalance int64, future if bias > 0 { b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) } - pri := n.balanceToPriority(b, capacity) + // Note: we subtract one from the estimated priority in order to ensure that biased + // estimates are always lower than actual priorities, even if the bias is very small. + // This ensures that two nodes will not ping-pong update signals forever if both of + // them have zero estimated priority drop in the projected future. + pri := n.balanceToPriority(now, b, capacity) - 1 if update { n.addCallback(balanceCallbackUpdate, pri, n.signalPriorityUpdate) } return pri } -// PosBalanceMissing calculates the missing amount of positive balance in order to -// connect at targetCapacity, stay connected for the given amount of time and then -// still have a priority of targetPriority -func (n *NodeBalance) PosBalanceMissing(targetPriority int64, targetCapacity uint64, after time.Duration) uint64 { - n.lock.Lock() - defer n.lock.Unlock() - - now := n.bt.clock.Now() - if targetPriority < 0 { - timePrice := n.negFactor.timePrice(targetCapacity) - timeCost := uint64(float64(after) * timePrice) - negBalance := n.balance.neg.Value(n.bt.negExp.LogOffset(now)) - if timeCost+negBalance < uint64(-targetPriority) { - return 0 - } - if uint64(-targetPriority) > negBalance && timePrice > 1e-100 { - if negTime := time.Duration(float64(uint64(-targetPriority)-negBalance) / timePrice); negTime < after { - after -= negTime - } else { - after = 0 - } - } - targetPriority = 0 - } - timePrice := n.posFactor.timePrice(targetCapacity) - posRequired := uint64(float64(targetPriority)*float64(targetCapacity)+float64(after)*timePrice) + 1 - if posRequired >= maxBalance { - return math.MaxUint64 // target not reachable - } - posBalance := n.balance.pos.Value(n.bt.posExp.LogOffset(now)) - if posRequired > posBalance { - return posRequired - posBalance - } - return 0 -} - // SetPriceFactors sets the price factors. TimeFactor is the price of a nanosecond of // connection while RequestFactor is the price of a request cost unit. -func (n *NodeBalance) SetPriceFactors(posFactor, negFactor PriceFactors) { +func (n *nodeBalance) SetPriceFactors(posFactor, negFactor PriceFactors) { n.lock.Lock() now := n.bt.clock.Now() n.updateBalance(now) @@ -346,7 +388,7 @@ func (n *NodeBalance) SetPriceFactors(posFactor, negFactor PriceFactors) { } // GetPriceFactors returns the price factors -func (n *NodeBalance) GetPriceFactors() (posFactor, negFactor PriceFactors) { +func (n *nodeBalance) GetPriceFactors() (posFactor, negFactor PriceFactors) { n.lock.Lock() defer n.lock.Unlock() @@ -354,7 +396,7 @@ func (n *NodeBalance) GetPriceFactors() (posFactor, negFactor PriceFactors) { } // activate starts time/capacity cost deduction. -func (n *NodeBalance) activate() { +func (n *nodeBalance) activate() { n.bt.updateTotalBalance(n, func() bool { if n.active { return false @@ -366,7 +408,7 @@ func (n *NodeBalance) activate() { } // deactivate stops time/capacity cost deduction and saves the balances in the database -func (n *NodeBalance) deactivate() { +func (n *nodeBalance) deactivate() { n.bt.updateTotalBalance(n, func() bool { if !n.active { return false @@ -383,7 +425,7 @@ func (n *NodeBalance) deactivate() { } // updateBalance updates balance based on the time factor -func (n *NodeBalance) updateBalance(now mclock.AbsTime) { +func (n *nodeBalance) updateBalance(now mclock.AbsTime) { if n.active && now > n.lastUpdate { n.balance = n.reducedBalance(n.balance, n.lastUpdate, time.Duration(now-n.lastUpdate), n.capacity, 0) n.lastUpdate = now @@ -391,7 +433,7 @@ func (n *NodeBalance) updateBalance(now mclock.AbsTime) { } // storeBalance stores the positive and/or negative balance of the node in the database -func (n *NodeBalance) storeBalance(pos, neg bool) { +func (n *nodeBalance) storeBalance(pos, neg bool) { if pos { n.bt.storeBalance(n.node.ID().Bytes(), false, n.balance.pos) } @@ -405,7 +447,7 @@ func (n *NodeBalance) storeBalance(pos, neg bool) { // immediately. // Note: should be called while n.lock is held // Note 2: the callback function runs inside a NodeStateMachine operation -func (n *NodeBalance) addCallback(id int, threshold int64, callback func()) { +func (n *nodeBalance) addCallback(id int, threshold int64, callback func()) { n.removeCallback(id) idx := 0 for idx < n.callbackCount && threshold > n.callbacks[idx].threshold { @@ -425,7 +467,7 @@ func (n *NodeBalance) addCallback(id int, threshold int64, callback func()) { // removeCallback removes the given callback and returns true if it was active // Note: should be called while n.lock is held -func (n *NodeBalance) removeCallback(id int) bool { +func (n *nodeBalance) removeCallback(id int) bool { idx := n.callbackIndex[id] if idx == -1 { return false @@ -442,11 +484,11 @@ func (n *NodeBalance) removeCallback(id int) bool { // checkCallbacks checks whether the threshold of any of the active callbacks // have been reached and returns triggered callbacks. // Note: checkCallbacks assumes that the balance has been recently updated. -func (n *NodeBalance) checkCallbacks(now mclock.AbsTime) (callbacks []func()) { +func (n *nodeBalance) checkCallbacks(now mclock.AbsTime) (callbacks []func()) { if n.callbackCount == 0 || n.capacity == 0 { return } - pri := n.balanceToPriority(n.balance, n.capacity) + pri := n.balanceToPriority(now, n.balance, n.capacity) for n.callbackCount != 0 && n.callbacks[n.callbackCount-1].threshold >= pri { n.callbackCount-- n.callbackIndex[n.callbacks[n.callbackCount].id] = -1 @@ -458,7 +500,7 @@ func (n *NodeBalance) checkCallbacks(now mclock.AbsTime) (callbacks []func()) { // scheduleCheck sets up or updates a scheduled event to ensure that it will be called // again just after the next threshold has been reached. -func (n *NodeBalance) scheduleCheck(now mclock.AbsTime) { +func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { if n.callbackCount != 0 { d, ok := n.timeUntil(n.callbacks[n.callbackCount-1].threshold) if !ok { @@ -484,7 +526,7 @@ func (n *NodeBalance) scheduleCheck(now mclock.AbsTime) { } // updateAfter schedules a balance update and callback check in the future -func (n *NodeBalance) updateAfter(dt time.Duration) { +func (n *nodeBalance) updateAfter(dt time.Duration) { if n.updateEvent == nil || n.updateEvent.Stop() { if dt == 0 { n.updateEvent = nil @@ -512,20 +554,22 @@ func (n *NodeBalance) updateAfter(dt time.Duration) { // balanceExhausted should be called when the positive balance is exhausted (priority goes to zero/negative) // Note: this function should run inside a NodeStateMachine operation -func (n *NodeBalance) balanceExhausted() { +func (n *nodeBalance) balanceExhausted() { n.lock.Lock() n.storeBalance(true, false) - n.priority = false + n.hasPriority = false n.lock.Unlock() - n.bt.ns.SetStateSub(n.node, nodestate.Flags{}, n.bt.PriorityFlag, 0) + if n.setFlags { + n.bt.ns.SetStateSub(n.node, nodestate.Flags{}, n.bt.setup.priorityFlag, 0) + } } // checkPriorityStatus checks whether the node has gained priority status and sets the priority // callback and flag if necessary. It assumes that the balance has been recently updated. // Note that the priority flag has to be set by the caller after the mutex has been released. -func (n *NodeBalance) checkPriorityStatus() bool { - if !n.priority && !n.balance.pos.IsZero() { - n.priority = true +func (n *nodeBalance) checkPriorityStatus() bool { + if !n.hasPriority && !n.balance.pos.IsZero() { + n.hasPriority = true n.addCallback(balanceCallbackZero, 0, func() { n.balanceExhausted() }) return true } @@ -534,15 +578,15 @@ func (n *NodeBalance) checkPriorityStatus() bool { // signalPriorityUpdate signals that the priority fell below the previous minimum estimate // Note: this function should run inside a NodeStateMachine operation -func (n *NodeBalance) signalPriorityUpdate() { - n.bt.ns.SetStateSub(n.node, n.bt.UpdateFlag, nodestate.Flags{}, 0) - n.bt.ns.SetStateSub(n.node, nodestate.Flags{}, n.bt.UpdateFlag, 0) +func (n *nodeBalance) signalPriorityUpdate() { + n.bt.ns.SetStateSub(n.node, n.bt.setup.updateFlag, nodestate.Flags{}, 0) + n.bt.ns.SetStateSub(n.node, nodestate.Flags{}, n.bt.setup.updateFlag, 0) } // setCapacity updates the capacity value used for priority calculation // Note: capacity should never be zero // Note 2: this function should run inside a NodeStateMachine operation -func (n *NodeBalance) setCapacity(capacity uint64) { +func (n *nodeBalance) setCapacity(capacity uint64) { n.lock.Lock() now := n.bt.clock.Now() n.updateBalance(now) @@ -557,74 +601,89 @@ func (n *NodeBalance) setCapacity(capacity uint64) { // balanceToPriority converts a balance to a priority value. Lower priority means // first to disconnect. Positive balance translates to positive priority. If positive // balance is zero then negative balance translates to a negative priority. -func (n *NodeBalance) balanceToPriority(b balance, capacity uint64) int64 { - if !b.pos.IsZero() { - return int64(b.pos.Value(n.bt.posExp.LogOffset(n.bt.clock.Now())) / capacity) +func (n *nodeBalance) balanceToPriority(now mclock.AbsTime, b balance, capacity uint64) int64 { + pos := b.posValue(now) + if pos > 0 { + return int64(pos / capacity) + } + return -int64(b.negValue(now)) +} + +// priorityToBalance converts a target priority to a requested balance value. +// If the priority is negative, then minimal negative balance is returned; +// otherwise the minimal positive balance is returned. +func (n *nodeBalance) priorityToBalance(priority int64, capacity uint64) (uint64, uint64) { + if priority > 0 { + return uint64(priority) * n.capacity, 0 } - return -int64(b.neg.Value(n.bt.negExp.LogOffset(n.bt.clock.Now()))) + return 0, uint64(-priority) } // reducedBalance estimates the reduced balance at a given time in the fututre based // on the given balance, the time factor and an estimated average request cost per time ratio -func (n *NodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { +func (n *nodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { // since the costs are applied continuously during the dt time period we calculate // the expiration offset at the middle of the period - at := start + mclock.AbsTime(dt/2) - dtf := float64(dt) + var ( + at = start + mclock.AbsTime(dt/2) + dtf = float64(dt) + ) if !b.pos.IsZero() { - factor := n.posFactor.timePrice(capacity) + n.posFactor.RequestFactor*avgReqCost + factor := n.posFactor.connectionPrice(capacity, avgReqCost) diff := -int64(dtf * factor) - dd := b.pos.Add(diff, n.bt.posExp.LogOffset(at)) - if dd == diff { + _, _, net, _ := b.addValue(at, diff, true, false) + if net == diff { dtf = 0 } else { - dtf += float64(dd) / factor + dtf += float64(net) / factor } } - if dt > 0 { - factor := n.negFactor.timePrice(capacity) + n.negFactor.RequestFactor*avgReqCost - b.neg.Add(int64(dtf*factor), n.bt.negExp.LogOffset(at)) + if dtf > 0 { + factor := n.negFactor.connectionPrice(capacity, avgReqCost) + b.addValue(at, int64(dtf*factor), false, false) } return b } // timeUntil calculates the remaining time needed to reach a given priority level // assuming that no requests are processed until then. If the given level is never -// reached then (0, false) is returned. +// reached then (0, false) is returned. If it has already been reached then (0, true) +// is returned. // Note: the function assumes that the balance has been recently updated and // calculates the time starting from the last update. -func (n *NodeBalance) timeUntil(priority int64) (time.Duration, bool) { - now := n.bt.clock.Now() - var dt float64 - if !n.balance.pos.IsZero() { - posBalance := n.balance.pos.Value(n.bt.posExp.LogOffset(now)) - timePrice := n.posFactor.timePrice(n.capacity) +func (n *nodeBalance) timeUntil(priority int64) (time.Duration, bool) { + var ( + now = n.bt.clock.Now() + pos = n.balance.posValue(now) + targetPos, targetNeg = n.priorityToBalance(priority, n.capacity) + diffTime float64 + ) + if pos > 0 { + timePrice := n.posFactor.connectionPrice(n.capacity, 0) if timePrice < 1e-100 { return 0, false } - if priority > 0 { - newBalance := uint64(priority) * n.capacity - if newBalance > posBalance { - return 0, false + if targetPos > 0 { + if targetPos > pos { + return 0, true } - dt = float64(posBalance-newBalance) / timePrice - return time.Duration(dt), true + diffTime = float64(pos-targetPos) / timePrice + return time.Duration(diffTime), true } else { - dt = float64(posBalance) / timePrice + diffTime = float64(pos) / timePrice } } else { - if priority > 0 { - return 0, false + if targetPos > 0 { + return 0, true } } - // if we have a positive balance then dt equals the time needed to get it to zero - negBalance := n.balance.neg.Value(n.bt.negExp.LogOffset(now)) - timePrice := n.negFactor.timePrice(n.capacity) - if uint64(-priority) > negBalance { + neg := n.balance.negValue(now) + if targetNeg > neg { + timePrice := n.negFactor.connectionPrice(n.capacity, 0) if timePrice < 1e-100 { return 0, false } - dt += float64(uint64(-priority)-negBalance) / timePrice + diffTime += float64(targetNeg-neg) / timePrice } - return time.Duration(dt), true + return time.Duration(diffTime), true } diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index e22074db2d..66f0d1f301 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/les/utils" "github.com/ethereum/go-ethereum/p2p/enode" @@ -31,59 +32,82 @@ import ( "github.com/ethereum/go-ethereum/p2p/nodestate" ) -var ( - testFlag = testSetup.NewFlag("testFlag") - connAddrFlag = testSetup.NewField("connAddr", reflect.TypeOf("")) - btTestSetup = NewBalanceTrackerSetup(testSetup) -) - -func init() { - btTestSetup.Connect(connAddrFlag, ppTestSetup.CapacityField) -} - type zeroExpirer struct{} func (z zeroExpirer) SetRate(now mclock.AbsTime, rate float64) {} func (z zeroExpirer) SetLogOffset(now mclock.AbsTime, logOffset utils.Fixed64) {} func (z zeroExpirer) LogOffset(now mclock.AbsTime) utils.Fixed64 { return 0 } +type balanceTestClient struct{} + +func (client balanceTestClient) FreeClientId() string { return "" } + type balanceTestSetup struct { clock *mclock.Simulated + db ethdb.KeyValueStore ns *nodestate.NodeStateMachine - bt *BalanceTracker + setup *serverSetup + bt *balanceTracker } -func newBalanceTestSetup() *balanceTestSetup { +func newBalanceTestSetup(db ethdb.KeyValueStore, posExp, negExp utils.ValueExpirer) *balanceTestSetup { + // Initialize and customize the setup for the balance testing clock := &mclock.Simulated{} - ns := nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) - db := memorydb.New() - bt := NewBalanceTracker(ns, btTestSetup, db, clock, zeroExpirer{}, zeroExpirer{}) + setup := newServerSetup() + setup.clientField = setup.setup.NewField("balancTestClient", reflect.TypeOf(balanceTestClient{})) + + ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) + if posExp == nil { + posExp = zeroExpirer{} + } + if negExp == nil { + negExp = zeroExpirer{} + } + if db == nil { + db = memorydb.New() + } + bt := newBalanceTracker(ns, setup, db, clock, posExp, negExp) ns.Start() return &balanceTestSetup{ clock: clock, + db: db, ns: ns, + setup: setup, bt: bt, } } -func (b *balanceTestSetup) newNode(capacity uint64) *NodeBalance { +func (b *balanceTestSetup) newNode(capacity uint64) *nodeBalance { node := enode.SignNull(&enr.Record{}, enode.ID{}) - b.ns.SetState(node, testFlag, nodestate.Flags{}, 0) - b.ns.SetField(node, btTestSetup.connAddressField, "") + b.ns.SetField(node, b.setup.clientField, balanceTestClient{}) if capacity != 0 { - b.ns.SetField(node, ppTestSetup.CapacityField, capacity) + b.ns.SetField(node, b.setup.capacityField, capacity) } - n, _ := b.ns.GetField(node, btTestSetup.BalanceField).(*NodeBalance) + n, _ := b.ns.GetField(node, b.setup.balanceField).(*nodeBalance) return n } +func (b *balanceTestSetup) setBalance(node *nodeBalance, pos, neg uint64) (err error) { + b.bt.BalanceOperation(node.node.ID(), node.connAddress, func(balance AtomicBalanceOperator) { + err = balance.SetBalance(pos, neg) + }) + return +} + +func (b *balanceTestSetup) addBalance(node *nodeBalance, add int64) (old, new uint64, err error) { + b.bt.BalanceOperation(node.node.ID(), node.connAddress, func(balance AtomicBalanceOperator) { + old, new, err = balance.AddBalance(add) + }) + return +} + func (b *balanceTestSetup) stop() { - b.bt.Stop() + b.bt.stop() b.ns.Stop() } func TestAddBalance(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000) @@ -100,7 +124,7 @@ func TestAddBalance(t *testing.T) { {maxBalance, [2]uint64{0, 0}, 0, true}, } for _, i := range inputs { - old, new, err := node.AddBalance(i.delta) + old, new, err := b.addBalance(node, i.delta) if i.expectErr { if err == nil { t.Fatalf("Expect get error but nil") @@ -119,7 +143,7 @@ func TestAddBalance(t *testing.T) { } func TestSetBalance(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000) @@ -130,9 +154,8 @@ func TestSetBalance(t *testing.T) { {0, 1000}, {1000, 1000}, } - for _, i := range inputs { - node.SetBalance(i.pos, i.neg) + b.setBalance(node, i.pos, i.neg) pos, neg := node.GetBalance() if pos != i.pos { t.Fatalf("Positive balance mismatch, want %v, got %v", i.pos, pos) @@ -144,13 +167,12 @@ func TestSetBalance(t *testing.T) { } func TestBalanceTimeCost(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000) - b.ns.SetField(node.node, ppTestSetup.CapacityField, uint64(1)) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) - node.SetBalance(uint64(time.Minute), 0) // 1 minute time allowance + b.setBalance(node, uint64(time.Minute), 0) // 1 minute time allowance var inputs = []struct { runTime time.Duration @@ -172,7 +194,7 @@ func TestBalanceTimeCost(t *testing.T) { } } - node.SetBalance(uint64(time.Minute), 0) // Refill 1 minute time allowance + b.setBalance(node, uint64(time.Minute), 0) // Refill 1 minute time allowance for _, i := range inputs { b.clock.Run(i.runTime) if pos, _ := node.GetBalance(); pos != i.expPos { @@ -185,13 +207,12 @@ func TestBalanceTimeCost(t *testing.T) { } func TestBalanceReqCost(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) - b.ns.SetField(node.node, ppTestSetup.CapacityField, uint64(1)) - node.SetBalance(uint64(time.Minute), 0) // 1 minute time serving time allowance + b.setBalance(node, uint64(time.Minute), 0) // 1 minute time serving time allowance var inputs = []struct { reqCost uint64 expPos uint64 @@ -214,7 +235,7 @@ func TestBalanceReqCost(t *testing.T) { } func TestBalanceToPriority(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) @@ -230,22 +251,20 @@ func TestBalanceToPriority(t *testing.T) { {0, 1000, -1000}, } for _, i := range inputs { - node.SetBalance(i.pos, i.neg) - priority := node.Priority(1000) + b.setBalance(node, i.pos, i.neg) + priority := node.priority(1000) if priority != i.priority { - t.Fatalf("Priority mismatch, want %v, got %v", i.priority, priority) + t.Fatalf("priority mismatch, want %v, got %v", i.priority, priority) } } } func TestEstimatedPriority(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000000000) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) - - b.ns.SetField(node.node, ppTestSetup.CapacityField, uint64(1)) - node.SetBalance(uint64(time.Minute), 0) + b.setBalance(node, uint64(time.Minute), 0) var inputs = []struct { runTime time.Duration // time cost futureTime time.Duration // diff of future time @@ -272,47 +291,18 @@ func TestEstimatedPriority(t *testing.T) { for _, i := range inputs { b.clock.Run(i.runTime) node.RequestServed(i.reqCost) - priority := node.EstimatePriority(1000000000, 0, i.futureTime, 0, false) - if priority != i.priority { - t.Fatalf("Estimated priority mismatch, want %v, got %v", i.priority, priority) - } - } -} - -func TestPosBalanceMissing(t *testing.T) { - b := newBalanceTestSetup() - defer b.stop() - node := b.newNode(1000) - node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) - - b.ns.SetField(node.node, ppTestSetup.CapacityField, uint64(1)) - var inputs = []struct { - pos, neg uint64 - priority int64 - cap uint64 - after time.Duration - expect uint64 - }{ - {uint64(time.Second * 2), 0, 0, 1, time.Second, 0}, - {uint64(time.Second * 2), 0, 0, 1, 2 * time.Second, 1}, - {uint64(time.Second * 2), 0, int64(time.Second), 1, 2 * time.Second, uint64(time.Second) + 1}, - {0, 0, int64(time.Second), 1, time.Second, uint64(2*time.Second) + 1}, - {0, 0, -int64(time.Second), 1, time.Second, 1}, - } - for _, i := range inputs { - node.SetBalance(i.pos, i.neg) - got := node.PosBalanceMissing(i.priority, i.cap, i.after) - if got != i.expect { - t.Fatalf("Missing budget mismatch, want %v, got %v", i.expect, got) + priority := node.estimatePriority(1000000000, 0, i.futureTime, 0, false) + if priority != i.priority-1 { + t.Fatalf("Estimated priority mismatch, want %v, got %v", i.priority-1, priority) } } } func TestPostiveBalanceCounting(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() - var nodes []*NodeBalance + var nodes []*nodeBalance for i := 0; i < 100; i += 1 { node := b.newNode(1000000) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) @@ -323,7 +313,7 @@ func TestPostiveBalanceCounting(t *testing.T) { var sum uint64 for i := 0; i < 100; i += 1 { amount := int64(rand.Intn(100) + 100) - nodes[i].AddBalance(amount) + b.addBalance(nodes[i], amount) sum += uint64(amount) } if b.bt.TotalTokenAmount() != sum { @@ -333,7 +323,7 @@ func TestPostiveBalanceCounting(t *testing.T) { // Change client status for i := 0; i < 100; i += 1 { if rand.Intn(2) == 0 { - b.ns.SetField(nodes[i].node, ppTestSetup.CapacityField, uint64(1)) + b.ns.SetField(nodes[i].node, b.setup.capacityField, uint64(1)) } } if b.bt.TotalTokenAmount() != sum { @@ -341,7 +331,7 @@ func TestPostiveBalanceCounting(t *testing.T) { } for i := 0; i < 100; i += 1 { if rand.Intn(2) == 0 { - b.ns.SetField(nodes[i].node, ppTestSetup.CapacityField, uint64(1)) + b.ns.SetField(nodes[i].node, b.setup.capacityField, uint64(1)) } } if b.bt.TotalTokenAmount() != sum { @@ -350,7 +340,7 @@ func TestPostiveBalanceCounting(t *testing.T) { } func TestCallbackChecking(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000000) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) @@ -363,7 +353,7 @@ func TestCallbackChecking(t *testing.T) { {0, time.Second}, {-int64(time.Second), 2 * time.Second}, } - node.SetBalance(uint64(time.Second), 0) + b.setBalance(node, uint64(time.Second), 0) for _, i := range inputs { diff, _ := node.timeUntil(i.priority) if diff != i.expDiff { @@ -373,14 +363,13 @@ func TestCallbackChecking(t *testing.T) { } func TestCallback(t *testing.T) { - b := newBalanceTestSetup() + b := newBalanceTestSetup(nil, nil, nil) defer b.stop() node := b.newNode(1000) node.SetPriceFactors(PriceFactors{1, 0, 1}, PriceFactors{1, 0, 1}) - b.ns.SetField(node.node, ppTestSetup.CapacityField, uint64(1)) callCh := make(chan struct{}, 1) - node.SetBalance(uint64(time.Minute), 0) + b.setBalance(node, uint64(time.Minute), 0) node.addCallback(balanceCallbackZero, 0, func() { callCh <- struct{}{} }) b.clock.Run(time.Minute) @@ -390,7 +379,7 @@ func TestCallback(t *testing.T) { t.Fatalf("Callback hasn't been called yet") } - node.SetBalance(uint64(time.Minute), 0) + b.setBalance(node, uint64(time.Minute), 0) node.addCallback(balanceCallbackZero, 0, func() { callCh <- struct{}{} }) node.removeCallback(balanceCallbackZero) @@ -403,23 +392,14 @@ func TestCallback(t *testing.T) { } func TestBalancePersistence(t *testing.T) { - clock := &mclock.Simulated{} - ns := nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) - db := memorydb.New() posExp := &utils.Expirer{} negExp := &utils.Expirer{} - posExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour*2)) // halves every two hours - negExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour)) // halves every hour - bt := NewBalanceTracker(ns, btTestSetup, db, clock, posExp, negExp) - ns.Start() - bts := &balanceTestSetup{ - clock: clock, - ns: ns, - bt: bt, - } - var nb *NodeBalance - exp := func(expPos, expNeg uint64) { - pos, neg := nb.GetBalance() + posExp.SetRate(0, math.Log(2)/float64(time.Hour*2)) // halves every two hours + negExp.SetRate(0, math.Log(2)/float64(time.Hour)) // halves every hour + setup := newBalanceTestSetup(nil, posExp, negExp) + + exp := func(balance *nodeBalance, expPos, expNeg uint64) { + pos, neg := balance.GetBalance() if pos != expPos { t.Fatalf("Positive balance incorrect, want %v, got %v", expPos, pos) } @@ -428,44 +408,32 @@ func TestBalancePersistence(t *testing.T) { } } expTotal := func(expTotal uint64) { - total := bt.TotalTokenAmount() + total := setup.bt.TotalTokenAmount() if total != expTotal { t.Fatalf("Total token amount incorrect, want %v, got %v", expTotal, total) } } expTotal(0) - nb = bts.newNode(0) + balance := setup.newNode(0) expTotal(0) - nb.SetBalance(16000000000, 16000000000) - exp(16000000000, 16000000000) + setup.setBalance(balance, 16000000000, 16000000000) + exp(balance, 16000000000, 16000000000) expTotal(16000000000) - clock.Run(time.Hour * 2) - exp(8000000000, 4000000000) + + setup.clock.Run(time.Hour * 2) + exp(balance, 8000000000, 4000000000) expTotal(8000000000) - bt.Stop() - ns.Stop() - - clock = &mclock.Simulated{} - ns = nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) - posExp = &utils.Expirer{} - negExp = &utils.Expirer{} - posExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour*2)) // halves every two hours - negExp.SetRate(clock.Now(), math.Log(2)/float64(time.Hour)) // halves every hour - bt = NewBalanceTracker(ns, btTestSetup, db, clock, posExp, negExp) - ns.Start() - bts = &balanceTestSetup{ - clock: clock, - ns: ns, - bt: bt, - } + setup.stop() + + // Test the functionalities after restart + setup = newBalanceTestSetup(setup.db, posExp, negExp) expTotal(8000000000) - nb = bts.newNode(0) - exp(8000000000, 4000000000) + balance = setup.newNode(0) + exp(balance, 8000000000, 4000000000) expTotal(8000000000) - clock.Run(time.Hour * 2) - exp(4000000000, 1000000000) + setup.clock.Run(time.Hour * 2) + exp(balance, 4000000000, 1000000000) expTotal(4000000000) - bt.Stop() - ns.Stop() + setup.stop() } diff --git a/les/vflux/server/balance_tracker.go b/les/vflux/server/balance_tracker.go index 1708019de4..746697a8c7 100644 --- a/les/vflux/server/balance_tracker.go +++ b/les/vflux/server/balance_tracker.go @@ -17,7 +17,6 @@ package server import ( - "reflect" "sync" "time" @@ -25,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/utils" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nodestate" ) @@ -34,82 +34,56 @@ const ( persistExpirationRefresh = time.Minute * 5 // refresh period of the token expiration persistence ) -// BalanceTrackerSetup contains node state flags and fields used by BalanceTracker -type BalanceTrackerSetup struct { - // controlled by PriorityPool - PriorityFlag, UpdateFlag nodestate.Flags - BalanceField nodestate.Field - // external connections - connAddressField, capacityField nodestate.Field -} - -// NewBalanceTrackerSetup creates a new BalanceTrackerSetup and initializes the fields -// and flags controlled by BalanceTracker -func NewBalanceTrackerSetup(setup *nodestate.Setup) BalanceTrackerSetup { - return BalanceTrackerSetup{ - // PriorityFlag is set if the node has a positive balance - PriorityFlag: setup.NewFlag("priorityNode"), - // UpdateFlag set and then immediately reset if the balance has been updated and - // therefore priority is suddenly changed - UpdateFlag: setup.NewFlag("balanceUpdate"), - // BalanceField contains the NodeBalance struct which implements nodePriority, - // allowing on-demand priority calculation and future priority estimation - BalanceField: setup.NewField("balance", reflect.TypeOf(&NodeBalance{})), - } -} - -// Connect sets the fields used by BalanceTracker as an input -func (bts *BalanceTrackerSetup) Connect(connAddressField, capacityField nodestate.Field) { - bts.connAddressField = connAddressField - bts.capacityField = capacityField -} - -// BalanceTracker tracks positive and negative balances for connected nodes. -// After connAddressField is set externally, a NodeBalance is created and previous +// balanceTracker tracks positive and negative balances for connected nodes. +// After clientField is set externally, a nodeBalance is created and previous // balance values are loaded from the database. Both balances are exponentially expired // values. Costs are deducted from the positive balance if present, otherwise added to // the negative balance. If the capacity is non-zero then a time cost is applied // continuously while individual request costs are applied immediately. // The two balances are translated into a single priority value that also depends // on the actual capacity. -type BalanceTracker struct { - BalanceTrackerSetup - clock mclock.Clock - lock sync.Mutex - ns *nodestate.NodeStateMachine - ndb *nodeDB - posExp, negExp utils.ValueExpirer - posExpTC, negExpTC uint64 +type balanceTracker struct { + setup *serverSetup + clock mclock.Clock + lock sync.Mutex + ns *nodestate.NodeStateMachine + ndb *nodeDB + posExp, negExp utils.ValueExpirer + + posExpTC, negExpTC uint64 + defaultPosFactors, defaultNegFactors PriceFactors active, inactive utils.ExpiredValue balanceTimer *utils.UpdateTimer quit chan struct{} } -// NewBalanceTracker creates a new BalanceTracker -func NewBalanceTracker(ns *nodestate.NodeStateMachine, setup BalanceTrackerSetup, db ethdb.KeyValueStore, clock mclock.Clock, posExp, negExp utils.ValueExpirer) *BalanceTracker { +// newBalanceTracker creates a new balanceTracker +func newBalanceTracker(ns *nodestate.NodeStateMachine, setup *serverSetup, db ethdb.KeyValueStore, clock mclock.Clock, posExp, negExp utils.ValueExpirer) *balanceTracker { ndb := newNodeDB(db, clock) - bt := &BalanceTracker{ - ns: ns, - BalanceTrackerSetup: setup, - ndb: ndb, - clock: clock, - posExp: posExp, - negExp: negExp, - balanceTimer: utils.NewUpdateTimer(clock, time.Second*10), - quit: make(chan struct{}), + bt := &balanceTracker{ + ns: ns, + setup: setup, + ndb: ndb, + clock: clock, + posExp: posExp, + negExp: negExp, + balanceTimer: utils.NewUpdateTimer(clock, time.Second*10), + quit: make(chan struct{}), } posOffset, negOffset := bt.ndb.getExpiration() posExp.SetLogOffset(clock.Now(), posOffset) negExp.SetLogOffset(clock.Now(), negOffset) + // Load all persisted balance entries of priority nodes, + // calculate the total number of issued service tokens. bt.ndb.forEachBalance(false, func(id enode.ID, balance utils.ExpiredValue) bool { bt.inactive.AddExp(balance) return true }) - ns.SubscribeField(bt.capacityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { - n, _ := ns.GetField(node, bt.BalanceField).(*NodeBalance) + ns.SubscribeField(bt.setup.capacityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + n, _ := ns.GetField(node, bt.setup.balanceField).(*nodeBalance) if n == nil { return } @@ -126,15 +100,22 @@ func NewBalanceTracker(ns *nodestate.NodeStateMachine, setup BalanceTrackerSetup n.deactivate() } }) - ns.SubscribeField(bt.connAddressField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + ns.SubscribeField(bt.setup.clientField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + type peer interface { + FreeClientId() string + } if newValue != nil { - ns.SetFieldSub(node, bt.BalanceField, bt.newNodeBalance(node, newValue.(string))) + n := bt.newNodeBalance(node, newValue.(peer).FreeClientId(), true) + bt.lock.Lock() + n.SetPriceFactors(bt.defaultPosFactors, bt.defaultNegFactors) + bt.lock.Unlock() + ns.SetFieldSub(node, bt.setup.balanceField, n) } else { - ns.SetStateSub(node, nodestate.Flags{}, bt.PriorityFlag, 0) - if b, _ := ns.GetField(node, bt.BalanceField).(*NodeBalance); b != nil { + ns.SetStateSub(node, nodestate.Flags{}, bt.setup.priorityFlag, 0) + if b, _ := ns.GetField(node, bt.setup.balanceField).(*nodeBalance); b != nil { b.deactivate() } - ns.SetFieldSub(node, bt.BalanceField, nil) + ns.SetFieldSub(node, bt.setup.balanceField, nil) } }) @@ -157,31 +138,31 @@ func NewBalanceTracker(ns *nodestate.NodeStateMachine, setup BalanceTrackerSetup return bt } -// Stop saves expiration offset and unsaved node balances and shuts BalanceTracker down -func (bt *BalanceTracker) Stop() { +// Stop saves expiration offset and unsaved node balances and shuts balanceTracker down +func (bt *balanceTracker) stop() { now := bt.clock.Now() bt.ndb.setExpiration(bt.posExp.LogOffset(now), bt.negExp.LogOffset(now)) close(bt.quit) bt.ns.ForEach(nodestate.Flags{}, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { - if n, ok := bt.ns.GetField(node, bt.BalanceField).(*NodeBalance); ok { + if n, ok := bt.ns.GetField(node, bt.setup.balanceField).(*nodeBalance); ok { n.lock.Lock() n.storeBalance(true, true) n.lock.Unlock() - bt.ns.SetField(node, bt.BalanceField, nil) + bt.ns.SetField(node, bt.setup.balanceField, nil) } }) bt.ndb.close() } // TotalTokenAmount returns the current total amount of service tokens in existence -func (bt *BalanceTracker) TotalTokenAmount() uint64 { +func (bt *balanceTracker) TotalTokenAmount() uint64 { bt.lock.Lock() defer bt.lock.Unlock() bt.balanceTimer.Update(func(_ time.Duration) bool { bt.active = utils.ExpiredValue{} bt.ns.ForEach(nodestate.Flags{}, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { - if n, ok := bt.ns.GetField(node, bt.BalanceField).(*NodeBalance); ok && n.active { + if n, ok := bt.ns.GetField(node, bt.setup.balanceField).(*nodeBalance); ok && n.active { pos, _ := n.GetRawBalance() bt.active.AddExp(pos) } @@ -194,13 +175,21 @@ func (bt *BalanceTracker) TotalTokenAmount() uint64 { } // GetPosBalanceIDs lists node IDs with an associated positive balance -func (bt *BalanceTracker) GetPosBalanceIDs(start, stop enode.ID, maxCount int) (result []enode.ID) { +func (bt *balanceTracker) GetPosBalanceIDs(start, stop enode.ID, maxCount int) (result []enode.ID) { return bt.ndb.getPosBalanceIDs(start, stop, maxCount) } +// SetDefaultFactors sets the default price factors applied to subsequently connected clients +func (bt *balanceTracker) SetDefaultFactors(posFactors, negFactors PriceFactors) { + bt.lock.Lock() + bt.defaultPosFactors = posFactors + bt.defaultNegFactors = negFactors + bt.lock.Unlock() +} + // SetExpirationTCs sets positive and negative token expiration time constants. // Specified in seconds, 0 means infinite (no expiration). -func (bt *BalanceTracker) SetExpirationTCs(pos, neg uint64) { +func (bt *balanceTracker) SetExpirationTCs(pos, neg uint64) { bt.lock.Lock() defer bt.lock.Unlock() @@ -220,39 +209,55 @@ func (bt *BalanceTracker) SetExpirationTCs(pos, neg uint64) { // GetExpirationTCs returns the current positive and negative token expiration // time constants -func (bt *BalanceTracker) GetExpirationTCs() (pos, neg uint64) { +func (bt *balanceTracker) GetExpirationTCs() (pos, neg uint64) { bt.lock.Lock() defer bt.lock.Unlock() return bt.posExpTC, bt.negExpTC } -// newNodeBalance loads balances from the database and creates a NodeBalance instance -// for the given node. It also sets the PriorityFlag and adds balanceCallbackZero if +// BalanceOperation allows atomic operations on the balance of a node regardless of whether +// it is currently connected or not +func (bt *balanceTracker) BalanceOperation(id enode.ID, connAddress string, cb func(AtomicBalanceOperator)) { + bt.ns.Operation(func() { + var nb *nodeBalance + if node := bt.ns.GetNode(id); node != nil { + nb, _ = bt.ns.GetField(node, bt.setup.balanceField).(*nodeBalance) + } else { + node = enode.SignNull(&enr.Record{}, id) + nb = bt.newNodeBalance(node, connAddress, false) + } + cb(nb) + }) +} + +// newNodeBalance loads balances from the database and creates a nodeBalance instance +// for the given node. It also sets the priorityFlag and adds balanceCallbackZero if // the node has a positive balance. // Note: this function should run inside a NodeStateMachine operation -func (bt *BalanceTracker) newNodeBalance(node *enode.Node, negBalanceKey string) *NodeBalance { +func (bt *balanceTracker) newNodeBalance(node *enode.Node, connAddress string, setFlags bool) *nodeBalance { pb := bt.ndb.getOrNewBalance(node.ID().Bytes(), false) - nb := bt.ndb.getOrNewBalance([]byte(negBalanceKey), true) - n := &NodeBalance{ + nb := bt.ndb.getOrNewBalance([]byte(connAddress), true) + n := &nodeBalance{ bt: bt, node: node, - connAddress: negBalanceKey, - balance: balance{pos: pb, neg: nb}, + setFlags: setFlags, + connAddress: connAddress, + balance: balance{pos: pb, neg: nb, posExp: bt.posExp, negExp: bt.negExp}, initTime: bt.clock.Now(), lastUpdate: bt.clock.Now(), } for i := range n.callbackIndex { n.callbackIndex[i] = -1 } - if n.checkPriorityStatus() { - n.bt.ns.SetStateSub(n.node, n.bt.PriorityFlag, nodestate.Flags{}, 0) + if setFlags && n.checkPriorityStatus() { + n.bt.ns.SetStateSub(n.node, n.bt.setup.priorityFlag, nodestate.Flags{}, 0) } return n } // storeBalance stores either a positive or a negative balance in the database -func (bt *BalanceTracker) storeBalance(id []byte, neg bool, value utils.ExpiredValue) { +func (bt *balanceTracker) storeBalance(id []byte, neg bool, value utils.ExpiredValue) { if bt.canDropBalance(bt.clock.Now(), neg, value) { bt.ndb.delBalance(id, neg) // balance is small enough, drop it directly. } else { @@ -262,7 +267,7 @@ func (bt *BalanceTracker) storeBalance(id []byte, neg bool, value utils.ExpiredV // canDropBalance tells whether a positive or negative balance is below the threshold // and therefore can be dropped from the database -func (bt *BalanceTracker) canDropBalance(now mclock.AbsTime, neg bool, b utils.ExpiredValue) bool { +func (bt *balanceTracker) canDropBalance(now mclock.AbsTime, neg bool, b utils.ExpiredValue) bool { if neg { return b.Value(bt.negExp.LogOffset(now)) <= negThreshold } @@ -270,7 +275,7 @@ func (bt *BalanceTracker) canDropBalance(now mclock.AbsTime, neg bool, b utils.E } // updateTotalBalance adjusts the total balance after executing given callback. -func (bt *BalanceTracker) updateTotalBalance(n *NodeBalance, callback func() bool) { +func (bt *balanceTracker) updateTotalBalance(n *nodeBalance, callback func() bool) { bt.lock.Lock() defer bt.lock.Unlock() diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go new file mode 100644 index 0000000000..2e5fdd0ee7 --- /dev/null +++ b/les/vflux/server/clientpool.go @@ -0,0 +1,335 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package server + +import ( + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/les/utils" + "github.com/ethereum/go-ethereum/les/vflux" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/nodestate" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + ErrNotConnected = errors.New("client not connected") + ErrNoPriority = errors.New("priority too low to raise capacity") + ErrCantFindMaximum = errors.New("Unable to find maximum allowed capacity") +) + +// ClientPool implements a client database that assigns a priority to each client +// based on a positive and negative balance. Positive balance is externally assigned +// to prioritized clients and is decreased with connection time and processed +// requests (unless the price factors are zero). If the positive balance is zero +// then negative balance is accumulated. +// +// Balance tracking and priority calculation for connected clients is done by +// balanceTracker. PriorityQueue ensures that clients with the lowest positive or +// highest negative balance get evicted when the total capacity allowance is full +// and new clients with a better balance want to connect. +// +// Already connected nodes receive a small bias in their favor in order to avoid +// accepting and instantly kicking out clients. In theory, we try to ensure that +// each client can have several minutes of connection time. +// +// Balances of disconnected clients are stored in nodeDB including positive balance +// and negative banalce. Boeth positive balance and negative balance will decrease +// exponentially. If the balance is low enough, then the record will be dropped. +type ClientPool struct { + *priorityPool + *balanceTracker + + setup *serverSetup + clock mclock.Clock + closed bool + ns *nodestate.NodeStateMachine + synced func() bool + + lock sync.RWMutex + connectedBias time.Duration + + minCap uint64 // the minimal capacity value allowed for any client + capReqNode *enode.Node // node that is requesting capacity change; only used inside NSM operation +} + +// clientPeer represents a peer in the client pool. None of the callbacks should block. +type clientPeer interface { + Node() *enode.Node + FreeClientId() string // unique id for non-priority clients (typically a prefix of the network address) + InactiveAllowance() time.Duration // disconnection timeout for inactive non-priority peers + UpdateCapacity(newCap uint64, requested bool) // signals a capacity update (requested is true if it is a result of a SetCapacity call on the given peer + Disconnect() // initiates disconnection (Unregister should always be called) +} + +// NewClientPool creates a new client pool +func NewClientPool(balanceDb ethdb.KeyValueStore, minCap uint64, connectedBias time.Duration, clock mclock.Clock, synced func() bool) *ClientPool { + setup := newServerSetup() + ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) + cp := &ClientPool{ + priorityPool: newPriorityPool(ns, setup, clock, minCap, connectedBias, 4, 100), + balanceTracker: newBalanceTracker(ns, setup, balanceDb, clock, &utils.Expirer{}, &utils.Expirer{}), + setup: setup, + ns: ns, + clock: clock, + minCap: minCap, + connectedBias: connectedBias, + synced: synced, + } + + ns.SubscribeState(nodestate.MergeFlags(setup.activeFlag, setup.inactiveFlag, setup.priorityFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { + if newState.Equals(setup.inactiveFlag) { + // set timeout for non-priority inactive client + var timeout time.Duration + if c, ok := ns.GetField(node, setup.clientField).(clientPeer); ok { + timeout = c.InactiveAllowance() + } + if timeout > 0 { + ns.AddTimeout(node, setup.inactiveFlag, timeout) + } else { + // Note: if capacity is immediately available then priorityPool will set the active + // flag simultaneously with removing the inactive flag and therefore this will not + // initiate disconnection + ns.SetStateSub(node, nodestate.Flags{}, setup.inactiveFlag, 0) + } + } + if oldState.Equals(setup.inactiveFlag) && newState.Equals(setup.inactiveFlag.Or(setup.priorityFlag)) { + ns.SetStateSub(node, setup.inactiveFlag, nodestate.Flags{}, 0) // priority gained; remove timeout + } + if newState.Equals(setup.activeFlag) { + // active with no priority; limit capacity to minCap + cap, _ := ns.GetField(node, setup.capacityField).(uint64) + if cap > minCap { + cp.requestCapacity(node, minCap, minCap, 0) + } + } + if newState.Equals(nodestate.Flags{}) { + if c, ok := ns.GetField(node, setup.clientField).(clientPeer); ok { + c.Disconnect() + } + } + }) + + ns.SubscribeField(setup.capacityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + if c, ok := ns.GetField(node, setup.clientField).(clientPeer); ok { + newCap, _ := newValue.(uint64) + c.UpdateCapacity(newCap, node == cp.capReqNode) + } + }) + + // add metrics + cp.ns.SubscribeState(nodestate.MergeFlags(cp.setup.activeFlag, cp.setup.inactiveFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { + if oldState.IsEmpty() && !newState.IsEmpty() { + clientConnectedMeter.Mark(1) + } + if !oldState.IsEmpty() && newState.IsEmpty() { + clientDisconnectedMeter.Mark(1) + } + if oldState.HasNone(cp.setup.activeFlag) && oldState.HasAll(cp.setup.activeFlag) { + clientActivatedMeter.Mark(1) + } + if oldState.HasAll(cp.setup.activeFlag) && oldState.HasNone(cp.setup.activeFlag) { + clientDeactivatedMeter.Mark(1) + } + _, connected := cp.Active() + totalConnectedGauge.Update(int64(connected)) + }) + return cp +} + +// Start starts the client pool. Should be called before Register/Unregister. +func (cp *ClientPool) Start() { + cp.ns.Start() +} + +// Stop shuts the client pool down. The clientPeer interface callbacks will not be called +// after Stop. Register calls will return nil. +func (cp *ClientPool) Stop() { + cp.balanceTracker.stop() + cp.ns.Stop() +} + +// Register registers the peer into the client pool. If the peer has insufficient +// priority and remains inactive for longer than the allowed timeout then it will be +// disconnected by calling the Disconnect function of the clientPeer interface. +func (cp *ClientPool) Register(peer clientPeer) ConnectedBalance { + cp.ns.SetField(peer.Node(), cp.setup.clientField, peerWrapper{peer}) + balance, _ := cp.ns.GetField(peer.Node(), cp.setup.balanceField).(*nodeBalance) + return balance +} + +// Unregister removes the peer from the client pool +func (cp *ClientPool) Unregister(peer clientPeer) { + cp.ns.SetField(peer.Node(), cp.setup.clientField, nil) +} + +// setConnectedBias sets the connection bias, which is applied to already connected clients +// So that already connected client won't be kicked out very soon and we can ensure all +// connected clients can have enough time to request or sync some data. +func (cp *ClientPool) SetConnectedBias(bias time.Duration) { + cp.lock.Lock() + cp.connectedBias = bias + cp.setActiveBias(bias) + cp.lock.Unlock() +} + +// SetCapacity sets the assigned capacity of a connected client +func (cp *ClientPool) SetCapacity(node *enode.Node, reqCap uint64, bias time.Duration, requested bool) (capacity uint64, err error) { + cp.lock.RLock() + if cp.connectedBias > bias { + bias = cp.connectedBias + } + cp.lock.RUnlock() + + cp.ns.Operation(func() { + balance, _ := cp.ns.GetField(node, cp.setup.balanceField).(*nodeBalance) + if balance == nil { + err = ErrNotConnected + return + } + capacity, _ = cp.ns.GetField(node, cp.setup.capacityField).(uint64) + if capacity == 0 { + // if the client is inactive then it has insufficient priority for the minimal capacity + // (will be activated automatically with minCap when possible) + return + } + if reqCap < cp.minCap { + // can't request less than minCap; switching between 0 (inactive state) and minCap is + // performed by the server automatically as soon as necessary/possible + reqCap = cp.minCap + } + if reqCap > cp.minCap && cp.ns.GetState(node).HasNone(cp.setup.priorityFlag) { + err = ErrNoPriority + return + } + if reqCap == capacity { + return + } + if requested { + // mark the requested node so that the UpdateCapacity callback can signal + // whether the update is the direct result of a SetCapacity call on the given node + cp.capReqNode = node + defer func() { + cp.capReqNode = nil + }() + } + + var minTarget, maxTarget uint64 + if reqCap > capacity { + // Estimate maximum available capacity at the current priority level and request + // the estimated amount. + // Note: requestCapacity could find the highest available capacity between the + // current and the requested capacity but it could cost a lot of iterations with + // fine step adjustment if the requested capacity is very high. By doing a quick + // estimation of the maximum available capacity based on the capacity curve we + // can limit the number of required iterations. + curve := cp.getCapacityCurve().exclude(node.ID()) + maxTarget = curve.maxCapacity(func(capacity uint64) int64 { + return balance.estimatePriority(capacity, 0, 0, bias, false) + }) + if maxTarget <= capacity { + return + } + if maxTarget > reqCap { + maxTarget = reqCap + } + // Specify a narrow target range that allows a limited number of fine step + // iterations + minTarget = maxTarget - maxTarget/20 + if minTarget < capacity { + minTarget = capacity + } + } else { + minTarget, maxTarget = reqCap, reqCap + } + if newCap := cp.requestCapacity(node, minTarget, maxTarget, bias); newCap >= minTarget && newCap <= maxTarget { + capacity = newCap + return + } + // we should be able to find the maximum allowed capacity in a few iterations + log.Error("Unable to find maximum allowed capacity") + err = ErrCantFindMaximum + }) + return +} + +// serveCapQuery serves a vflux capacity query. It receives multiple token amount values +// and a bias time value. For each given token amount it calculates the maximum achievable +// capacity in case the amount is added to the balance. +func (cp *ClientPool) serveCapQuery(id enode.ID, freeID string, data []byte) []byte { + var req vflux.CapacityQueryReq + if rlp.DecodeBytes(data, &req) != nil { + return nil + } + if l := len(req.AddTokens); l == 0 || l > vflux.CapacityQueryMaxLen { + return nil + } + result := make(vflux.CapacityQueryReply, len(req.AddTokens)) + if !cp.synced() { + capacityQueryZeroMeter.Mark(1) + reply, _ := rlp.EncodeToBytes(&result) + return reply + } + + bias := time.Second * time.Duration(req.Bias) + cp.lock.RLock() + if cp.connectedBias > bias { + bias = cp.connectedBias + } + cp.lock.RUnlock() + + // use capacityCurve to answer request for multiple newly bought token amounts + curve := cp.getCapacityCurve().exclude(id) + cp.BalanceOperation(id, freeID, func(balance AtomicBalanceOperator) { + pb, _ := balance.GetBalance() + for i, addTokens := range req.AddTokens { + add := addTokens.Int64() + result[i] = curve.maxCapacity(func(capacity uint64) int64 { + return balance.estimatePriority(capacity, add, 0, bias, false) / int64(capacity) + }) + if add <= 0 && uint64(-add) >= pb && result[i] > cp.minCap { + result[i] = cp.minCap + } + if result[i] < cp.minCap { + result[i] = 0 + } + } + }) + // add first result to metrics (don't care about priority client multi-queries yet) + if result[0] == 0 { + capacityQueryZeroMeter.Mark(1) + } else { + capacityQueryNonZeroMeter.Mark(1) + } + reply, _ := rlp.EncodeToBytes(&result) + return reply +} + +// Handle implements Service +func (cp *ClientPool) Handle(id enode.ID, address string, name string, data []byte) []byte { + switch name { + case vflux.CapacityQueryName: + return cp.serveCapQuery(id, address, data) + default: + return nil + } +} diff --git a/les/clientpool_test.go b/les/vflux/server/clientpool_test.go similarity index 61% rename from les/clientpool_test.go rename to les/vflux/server/clientpool_test.go index 2aee444545..9503121697 100644 --- a/les/clientpool_test.go +++ b/les/vflux/server/clientpool_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package les +package server import ( "fmt" @@ -24,12 +24,13 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/rawdb" - vfs "github.com/ethereum/go-ethereum/les/vflux/server" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nodestate" ) +const defaultConnectedBias = time.Minute * 3 + func TestClientPoolL10C100Free(t *testing.T) { testClientPool(t, 10, 100, 0, true) } @@ -64,11 +65,6 @@ type poolTestPeer struct { inactiveAllowed bool } -func testStateMachine() *nodestate.NodeStateMachine { - return nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup) - -} - func newPoolTestPeer(i int, disconnCh chan int) *poolTestPeer { return &poolTestPeer{ index: i, @@ -81,36 +77,39 @@ func (i *poolTestPeer) Node() *enode.Node { return i.node } -func (i *poolTestPeer) freeClientId() string { +func (i *poolTestPeer) FreeClientId() string { return fmt.Sprintf("addr #%d", i.index) } -func (i *poolTestPeer) updateCapacity(cap uint64) { - i.cap = cap +func (i *poolTestPeer) InactiveAllowance() time.Duration { + if i.inactiveAllowed { + return time.Second * 10 + } + return 0 } -func (i *poolTestPeer) freeze() {} - -func (i *poolTestPeer) allowInactive() bool { - return i.inactiveAllowed +func (i *poolTestPeer) UpdateCapacity(capacity uint64, requested bool) { + i.cap = capacity } -func getBalance(pool *clientPool, p *poolTestPeer) (pos, neg uint64) { - temp := pool.ns.GetField(p.node, clientInfoField) == nil - if temp { - pool.ns.SetField(p.node, connAddressField, p.freeClientId()) - } - n, _ := pool.ns.GetField(p.node, pool.BalanceField).(*vfs.NodeBalance) - pos, neg = n.GetBalance() - if temp { - pool.ns.SetField(p.node, connAddressField, nil) +func (i *poolTestPeer) Disconnect() { + if i.disconnCh == nil { + return } + id := i.node.ID() + i.disconnCh <- int(id[0]) + int(id[1])<<8 +} + +func getBalance(pool *ClientPool, p *poolTestPeer) (pos, neg uint64) { + pool.BalanceOperation(p.node.ID(), p.FreeClientId(), func(nb AtomicBalanceOperator) { + pos, neg = nb.GetBalance() + }) return } -func addBalance(pool *clientPool, id enode.ID, amount int64) { - pool.forClients([]enode.ID{id}, func(c *clientInfo) { - c.balance.AddBalance(amount) +func addBalance(pool *ClientPool, id enode.ID, amount int64) { + pool.BalanceOperation(id, "", func(nb AtomicBalanceOperator) { + nb.AddBalance(amount) }) } @@ -122,6 +121,19 @@ func checkDiff(a, b uint64) bool { return a > b+maxDiff || b > a+maxDiff } +func connect(pool *ClientPool, peer *poolTestPeer) uint64 { + pool.Register(peer) + return peer.cap +} + +func disconnect(pool *ClientPool, peer *poolTestPeer) { + pool.Unregister(peer) +} + +func alwaysTrueFn() bool { + return true +} + func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, randomDisconnect bool) { rand.Seed(time.Now().UnixNano()) var ( @@ -130,19 +142,17 @@ func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, rando connected = make([]bool, clientCount) connTicks = make([]int, clientCount) disconnCh = make(chan int, clientCount) - disconnFn = func(id enode.ID) { - disconnCh <- int(id[0]) + int(id[1])<<8 - } - pool = newClientPool(testStateMachine(), db, 1, 0, &clock, disconnFn, alwaysTrueFn) + pool = NewClientPool(db, 1, 0, &clock, alwaysTrueFn) ) - pool.ns.Start() + pool.Start() + pool.SetExpirationTCs(0, 1000) - pool.setLimits(activeLimit, uint64(activeLimit)) - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool.SetLimits(uint64(activeLimit), uint64(activeLimit)) + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) // pool should accept new peers up to its connected limit for i := 0; i < activeLimit; i++ { - if cap, _ := pool.connect(newPoolTestPeer(i, disconnCh)); cap != 0 { + if cap := connect(pool, newPoolTestPeer(i, disconnCh)); cap != 0 { connected[i] = true } else { t.Fatalf("Test peer #%d rejected", i) @@ -163,23 +173,23 @@ func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, rando i := rand.Intn(clientCount) if connected[i] { if randomDisconnect { - pool.disconnect(newPoolTestPeer(i, disconnCh)) + disconnect(pool, newPoolTestPeer(i, disconnCh)) connected[i] = false connTicks[i] += tickCounter } } else { - if cap, _ := pool.connect(newPoolTestPeer(i, disconnCh)); cap != 0 { + if cap := connect(pool, newPoolTestPeer(i, disconnCh)); cap != 0 { connected[i] = true connTicks[i] -= tickCounter } else { - pool.disconnect(newPoolTestPeer(i, disconnCh)) + disconnect(pool, newPoolTestPeer(i, disconnCh)) } } pollDisconnects: for { select { case i := <-disconnCh: - pool.disconnect(newPoolTestPeer(i, disconnCh)) + disconnect(pool, newPoolTestPeer(i, disconnCh)) if connected[i] { connTicks[i] += tickCounter connected[i] = false @@ -211,18 +221,18 @@ func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, rando t.Errorf("Total connected time of test node #%d (%d) outside expected range (%d to %d)", i, connTicks[i], min, max) } } - pool.stop() + pool.Stop() } -func testPriorityConnect(t *testing.T, pool *clientPool, p *poolTestPeer, cap uint64, expSuccess bool) { - if cap, _ := pool.connect(p); cap == 0 { +func testPriorityConnect(t *testing.T, pool *ClientPool, p *poolTestPeer, cap uint64, expSuccess bool) { + if cap := connect(pool, p); cap == 0 { if expSuccess { t.Fatalf("Failed to connect paid client") } else { return } } - if _, err := pool.setCapacity(p.node, "", cap, defaultConnectedBias, true); err != nil { + if newCap, _ := pool.SetCapacity(p.node, cap, defaultConnectedBias, true); newCap != cap { if expSuccess { t.Fatalf("Failed to raise capacity of paid client") } else { @@ -239,11 +249,11 @@ func TestConnectPaidClient(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) // Add balance for an external client and mark it as paid client addBalance(pool, newPoolTestPeer(0, nil).node.ID(), int64(time.Minute)) @@ -255,16 +265,16 @@ func TestConnectPaidClientToSmallPool(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) // Add balance for an external client and mark it as paid client addBalance(pool, newPoolTestPeer(0, nil).node.ID(), int64(time.Minute)) - // Connect a fat paid client to pool, should reject it. + // connect a fat paid client to pool, should reject it. testPriorityConnect(t, pool, newPoolTestPeer(0, nil), 100, false) } @@ -273,24 +283,23 @@ func TestConnectPaidClientToFullPool(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - removeFn := func(enode.ID) {} // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { addBalance(pool, newPoolTestPeer(i, nil).node.ID(), int64(time.Second*20)) - pool.connect(newPoolTestPeer(i, nil)) + connect(pool, newPoolTestPeer(i, nil)) } addBalance(pool, newPoolTestPeer(11, nil).node.ID(), int64(time.Second*2)) // Add low balance to new paid client - if cap, _ := pool.connect(newPoolTestPeer(11, nil)); cap != 0 { + if cap := connect(pool, newPoolTestPeer(11, nil)); cap != 0 { t.Fatalf("Low balance paid client should be rejected") } clock.Run(time.Second) addBalance(pool, newPoolTestPeer(12, nil).node.ID(), int64(time.Minute*5)) // Add high balance to new paid client - if cap, _ := pool.connect(newPoolTestPeer(12, nil)); cap == 0 { + if cap := connect(pool, newPoolTestPeer(12, nil)); cap == 0 { t.Fatalf("High balance paid client should be accepted") } } @@ -301,23 +310,20 @@ func TestPaidClientKickedOut(t *testing.T) { db = rawdb.NewMemoryDatabase() kickedCh = make(chan int, 100) ) - removeFn := func(id enode.ID) { - kickedCh <- int(id[0]) - } - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) - pool.ns.Start() - pool.bt.SetExpirationTCs(0, 0) - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + pool.SetExpirationTCs(0, 0) + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { addBalance(pool, newPoolTestPeer(i, kickedCh).node.ID(), 10000000000) // 10 second allowance - pool.connect(newPoolTestPeer(i, kickedCh)) + connect(pool, newPoolTestPeer(i, kickedCh)) clock.Run(time.Millisecond) } clock.Run(defaultConnectedBias + time.Second*11) - if cap, _ := pool.connect(newPoolTestPeer(11, kickedCh)); cap == 0 { + if cap := connect(pool, newPoolTestPeer(11, kickedCh)); cap == 0 { t.Fatalf("Free client should be accepted") } select { @@ -335,12 +341,12 @@ func TestConnectFreeClient(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) - if cap, _ := pool.connect(newPoolTestPeer(0, nil)); cap == 0 { + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + if cap := connect(pool, newPoolTestPeer(0, nil)); cap == 0 { t.Fatalf("Failed to connect free client") } testPriorityConnect(t, pool, newPoolTestPeer(0, nil), 2, false) @@ -351,26 +357,25 @@ func TestConnectFreeClientToFullPool(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - removeFn := func(enode.ID) {} // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { - pool.connect(newPoolTestPeer(i, nil)) + connect(pool, newPoolTestPeer(i, nil)) } - if cap, _ := pool.connect(newPoolTestPeer(11, nil)); cap != 0 { + if cap := connect(pool, newPoolTestPeer(11, nil)); cap != 0 { t.Fatalf("New free client should be rejected") } clock.Run(time.Minute) - if cap, _ := pool.connect(newPoolTestPeer(12, nil)); cap != 0 { + if cap := connect(pool, newPoolTestPeer(12, nil)); cap != 0 { t.Fatalf("New free client should be rejected") } clock.Run(time.Millisecond) clock.Run(4 * time.Minute) - if cap, _ := pool.connect(newPoolTestPeer(13, nil)); cap == 0 { + if cap := connect(pool, newPoolTestPeer(13, nil)); cap == 0 { t.Fatalf("Old client connects more than 5min should be kicked") } } @@ -381,18 +386,17 @@ func TestFreeClientKickedOut(t *testing.T) { db = rawdb.NewMemoryDatabase() kicked = make(chan int, 100) ) - removeFn := func(id enode.ID) { kicked <- int(id[0]) } - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { - pool.connect(newPoolTestPeer(i, kicked)) + connect(pool, newPoolTestPeer(i, kicked)) clock.Run(time.Millisecond) } - if cap, _ := pool.connect(newPoolTestPeer(10, kicked)); cap != 0 { + if cap := connect(pool, newPoolTestPeer(10, kicked)); cap != 0 { t.Fatalf("New free client should be rejected") } select { @@ -400,10 +404,10 @@ func TestFreeClientKickedOut(t *testing.T) { case <-time.NewTimer(time.Second).C: t.Fatalf("timeout") } - pool.disconnect(newPoolTestPeer(10, kicked)) + disconnect(pool, newPoolTestPeer(10, kicked)) clock.Run(5 * time.Minute) for i := 0; i < 10; i++ { - pool.connect(newPoolTestPeer(i+10, kicked)) + connect(pool, newPoolTestPeer(i+10, kicked)) } for i := 0; i < 10; i++ { select { @@ -423,18 +427,17 @@ func TestPositiveBalanceCalculation(t *testing.T) { db = rawdb.NewMemoryDatabase() kicked = make(chan int, 10) ) - removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) addBalance(pool, newPoolTestPeer(0, kicked).node.ID(), int64(time.Minute*3)) testPriorityConnect(t, pool, newPoolTestPeer(0, kicked), 10, true) clock.Run(time.Minute) - pool.disconnect(newPoolTestPeer(0, kicked)) + disconnect(pool, newPoolTestPeer(0, kicked)) pb, _ := getBalance(pool, newPoolTestPeer(0, kicked)) if checkDiff(pb, uint64(time.Minute*2)) { t.Fatalf("Positive balance mismatch, want %v, got %v", uint64(time.Minute*2), pb) @@ -447,12 +450,11 @@ func TestDowngradePriorityClient(t *testing.T) { db = rawdb.NewMemoryDatabase() kicked = make(chan int, 10) ) - removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}) p := newPoolTestPeer(0, kicked) addBalance(pool, p.node.ID(), int64(time.Minute)) @@ -483,30 +485,31 @@ func TestNegativeBalanceCalculation(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(10, uint64(10)) // Total capacity limit is 10 - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}, vfs.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetExpirationTCs(0, 3600) + pool.SetLimits(10, uint64(10)) // Total capacity limit is 10 + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}, PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}) for i := 0; i < 10; i++ { - pool.connect(newPoolTestPeer(i, nil)) + connect(pool, newPoolTestPeer(i, nil)) } clock.Run(time.Second) for i := 0; i < 10; i++ { - pool.disconnect(newPoolTestPeer(i, nil)) + disconnect(pool, newPoolTestPeer(i, nil)) _, nb := getBalance(pool, newPoolTestPeer(i, nil)) if nb != 0 { t.Fatalf("Short connection shouldn't be recorded") } } for i := 0; i < 10; i++ { - pool.connect(newPoolTestPeer(i, nil)) + connect(pool, newPoolTestPeer(i, nil)) } clock.Run(time.Minute) for i := 0; i < 10; i++ { - pool.disconnect(newPoolTestPeer(i, nil)) + disconnect(pool, newPoolTestPeer(i, nil)) _, nb := getBalance(pool, newPoolTestPeer(i, nil)) exp := uint64(time.Minute) / 1000 exp -= exp / 120 // correct for negative balance expiration @@ -521,10 +524,10 @@ func TestInactiveClient(t *testing.T) { clock mclock.Simulated db = rawdb.NewMemoryDatabase() ) - pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {}, alwaysTrueFn) - pool.ns.Start() - defer pool.stop() - pool.setLimits(2, uint64(2)) + pool := NewClientPool(db, 1, defaultConnectedBias, &clock, alwaysTrueFn) + pool.Start() + defer pool.Stop() + pool.SetLimits(2, uint64(2)) p1 := newPoolTestPeer(1, nil) p1.inactiveAllowed = true @@ -535,15 +538,15 @@ func TestInactiveClient(t *testing.T) { addBalance(pool, p1.node.ID(), 1000*int64(time.Second)) addBalance(pool, p3.node.ID(), 2000*int64(time.Second)) // p1: 1000 p2: 0 p3: 2000 - p1.cap, _ = pool.connect(p1) + p1.cap = connect(pool, p1) if p1.cap != 1 { t.Fatalf("Failed to connect peer #1") } - p2.cap, _ = pool.connect(p2) + p2.cap = connect(pool, p2) if p2.cap != 1 { t.Fatalf("Failed to connect peer #2") } - p3.cap, _ = pool.connect(p3) + p3.cap = connect(pool, p3) if p3.cap != 1 { t.Fatalf("Failed to connect peer #3") } @@ -566,11 +569,11 @@ func TestInactiveClient(t *testing.T) { if p2.cap != 0 { t.Fatalf("Failed to deactivate peer #2") } - pool.setDefaultFactors(vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}, vfs.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}) + pool.SetDefaultFactors(PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}, PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 0}) p4 := newPoolTestPeer(4, nil) addBalance(pool, p4.node.ID(), 1500*int64(time.Second)) // p1: 1000 p2: 500 p3: 2000 p4: 1500 - p4.cap, _ = pool.connect(p4) + p4.cap = connect(pool, p4) if p4.cap != 1 { t.Fatalf("Failed to activate peer #4") } @@ -579,8 +582,8 @@ func TestInactiveClient(t *testing.T) { } clock.Run(time.Second * 600) // manually trigger a check to avoid a long real-time wait - pool.ns.SetState(p1.node, pool.UpdateFlag, nodestate.Flags{}, 0) - pool.ns.SetState(p1.node, nodestate.Flags{}, pool.UpdateFlag, 0) + pool.ns.SetState(p1.node, pool.setup.updateFlag, nodestate.Flags{}, 0) + pool.ns.SetState(p1.node, nodestate.Flags{}, pool.setup.updateFlag, 0) // p1: 1000 p2: 500 p3: 2000 p4: 900 if p1.cap != 1 { t.Fatalf("Failed to activate peer #1") @@ -588,8 +591,8 @@ func TestInactiveClient(t *testing.T) { if p4.cap != 0 { t.Fatalf("Failed to deactivate peer #4") } - pool.disconnect(p2) - pool.disconnect(p4) + disconnect(pool, p2) + disconnect(pool, p4) addBalance(pool, p1.node.ID(), -1000*int64(time.Second)) if p1.cap != 1 { t.Fatalf("Should not deactivate peer #1") diff --git a/les/vflux/server/metrics.go b/les/vflux/server/metrics.go new file mode 100644 index 0000000000..307b8347af --- /dev/null +++ b/les/vflux/server/metrics.go @@ -0,0 +1,33 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package server + +import ( + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + totalConnectedGauge = metrics.NewRegisteredGauge("vflux/server/totalConnected", nil) + + clientConnectedMeter = metrics.NewRegisteredMeter("vflux/server/clientEvent/connected", nil) + clientActivatedMeter = metrics.NewRegisteredMeter("vflux/server/clientEvent/activated", nil) + clientDeactivatedMeter = metrics.NewRegisteredMeter("vflux/server/clientEvent/deactivated", nil) + clientDisconnectedMeter = metrics.NewRegisteredMeter("vflux/server/clientEvent/disconnected", nil) + + capacityQueryZeroMeter = metrics.NewRegisteredMeter("vflux/server/capQueryZero", nil) + capacityQueryNonZeroMeter = metrics.NewRegisteredMeter("vflux/server/capQueryNonZero", nil) +) diff --git a/les/vflux/server/prioritypool.go b/les/vflux/server/prioritypool.go index e940ac7c65..573a3570a4 100644 --- a/les/vflux/server/prioritypool.go +++ b/les/vflux/server/prioritypool.go @@ -18,7 +18,6 @@ package server import ( "math" - "reflect" "sync" "time" @@ -33,36 +32,7 @@ const ( lazyQueueRefresh = time.Second * 10 // refresh period of the active queue ) -// PriorityPoolSetup contains node state flags and fields used by PriorityPool -// Note: ActiveFlag and InactiveFlag can be controlled both externally and by the pool, -// see PriorityPool description for details. -type PriorityPoolSetup struct { - // controlled by PriorityPool - ActiveFlag, InactiveFlag nodestate.Flags - CapacityField, ppNodeInfoField nodestate.Field - // external connections - updateFlag nodestate.Flags - priorityField nodestate.Field -} - -// NewPriorityPoolSetup creates a new PriorityPoolSetup and initializes the fields -// and flags controlled by PriorityPool -func NewPriorityPoolSetup(setup *nodestate.Setup) PriorityPoolSetup { - return PriorityPoolSetup{ - ActiveFlag: setup.NewFlag("active"), - InactiveFlag: setup.NewFlag("inactive"), - CapacityField: setup.NewField("capacity", reflect.TypeOf(uint64(0))), - ppNodeInfoField: setup.NewField("ppNodeInfo", reflect.TypeOf(&ppNodeInfo{})), - } -} - -// Connect sets the fields and flags used by PriorityPool as an input -func (pps *PriorityPoolSetup) Connect(priorityField nodestate.Field, updateFlag nodestate.Flags) { - pps.priorityField = priorityField // should implement nodePriority - pps.updateFlag = updateFlag // triggers an immediate priority update -} - -// PriorityPool handles a set of nodes where each node has a capacity (a scalar value) +// priorityPool handles a set of nodes where each node has a capacity (a scalar value) // and a priority (which can change over time and can also depend on the capacity). // A node is active if it has at least the necessary minimal amount of capacity while // inactive nodes have 0 capacity (values between 0 and the minimum are not allowed). @@ -79,70 +49,70 @@ func (pps *PriorityPoolSetup) Connect(priorityField nodestate.Field, updateFlag // This time bias can be interpreted as minimum expected active time at the given // capacity (if the threshold priority stays the same). // -// Nodes in the pool always have either InactiveFlag or ActiveFlag set. A new node is -// added to the pool by externally setting InactiveFlag. PriorityPool can switch a node -// between InactiveFlag and ActiveFlag at any time. Nodes can be removed from the pool -// by externally resetting both flags. ActiveFlag should not be set externally. +// Nodes in the pool always have either inactiveFlag or activeFlag set. A new node is +// added to the pool by externally setting inactiveFlag. priorityPool can switch a node +// between inactiveFlag and activeFlag at any time. Nodes can be removed from the pool +// by externally resetting both flags. activeFlag should not be set externally. // // The highest priority nodes in "inactive" state are moved to "active" state as soon as // the minimum capacity can be granted for them. The capacity of lower priority active // nodes is reduced or they are demoted to "inactive" state if their priority is // insufficient even at minimal capacity. -type PriorityPool struct { - PriorityPoolSetup - ns *nodestate.NodeStateMachine - clock mclock.Clock - lock sync.Mutex - activeQueue *prque.LazyQueue - inactiveQueue *prque.Prque - changed []*ppNodeInfo - activeCount, activeCap uint64 - maxCount, maxCap uint64 - minCap uint64 - activeBias time.Duration - capacityStepDiv uint64 - - cachedCurve *CapacityCurve +type priorityPool struct { + setup *serverSetup + ns *nodestate.NodeStateMachine + clock mclock.Clock + lock sync.Mutex + inactiveQueue *prque.Prque + maxCount, maxCap uint64 + minCap uint64 + activeBias time.Duration + capacityStepDiv, fineStepDiv uint64 + + cachedCurve *capacityCurve ccUpdatedAt mclock.AbsTime ccUpdateForced bool -} -// nodePriority interface provides current and estimated future priorities on demand -type nodePriority interface { - // Priority should return the current priority of the node (higher is better) - Priority(cap uint64) int64 - // EstMinPriority should return a lower estimate for the minimum of the node priority - // value starting from the current moment until the given time. If the priority goes - // under the returned estimate before the specified moment then it is the caller's - // responsibility to signal with updateFlag. - EstimatePriority(cap uint64, addBalance int64, future, bias time.Duration, update bool) int64 + tempState []*ppNodeInfo // nodes currently in temporary state + // the following fields represent the temporary state if tempState is not empty + activeCount, activeCap uint64 + activeQueue *prque.LazyQueue } -// ppNodeInfo is the internal node descriptor of PriorityPool +// ppNodeInfo is the internal node descriptor of priorityPool type ppNodeInfo struct { nodePriority nodePriority node *enode.Node connected bool - capacity, origCap uint64 - bias time.Duration - forced, changed bool + capacity uint64 // only changed when temporary state is committed activeIndex, inactiveIndex int -} -// NewPriorityPool creates a new PriorityPool -func NewPriorityPool(ns *nodestate.NodeStateMachine, setup PriorityPoolSetup, clock mclock.Clock, minCap uint64, activeBias time.Duration, capacityStepDiv uint64) *PriorityPool { - pp := &PriorityPool{ - ns: ns, - PriorityPoolSetup: setup, - clock: clock, - inactiveQueue: prque.New(inactiveSetIndex), - minCap: minCap, - activeBias: activeBias, - capacityStepDiv: capacityStepDiv, + tempState bool // should only be true while the priorityPool lock is held + tempCapacity uint64 // equals capacity when tempState is false + // the following fields only affect the temporary state and they are set to their + // default value when entering the temp state + minTarget, stepDiv uint64 + bias time.Duration +} + +// newPriorityPool creates a new priorityPool +func newPriorityPool(ns *nodestate.NodeStateMachine, setup *serverSetup, clock mclock.Clock, minCap uint64, activeBias time.Duration, capacityStepDiv, fineStepDiv uint64) *priorityPool { + pp := &priorityPool{ + setup: setup, + ns: ns, + clock: clock, + inactiveQueue: prque.New(inactiveSetIndex), + minCap: minCap, + activeBias: activeBias, + capacityStepDiv: capacityStepDiv, + fineStepDiv: fineStepDiv, + } + if pp.activeBias < time.Duration(1) { + pp.activeBias = time.Duration(1) } pp.activeQueue = prque.NewLazyQueue(activeSetIndex, activePriority, pp.activeMaxPriority, clock, lazyQueueRefresh) - ns.SubscribeField(pp.priorityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + ns.SubscribeField(pp.setup.balanceField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { if newValue != nil { c := &ppNodeInfo{ node: node, @@ -150,18 +120,19 @@ func NewPriorityPool(ns *nodestate.NodeStateMachine, setup PriorityPoolSetup, cl activeIndex: -1, inactiveIndex: -1, } - ns.SetFieldSub(node, pp.ppNodeInfoField, c) + ns.SetFieldSub(node, pp.setup.queueField, c) + ns.SetStateSub(node, setup.inactiveFlag, nodestate.Flags{}, 0) } else { - ns.SetStateSub(node, nodestate.Flags{}, pp.ActiveFlag.Or(pp.InactiveFlag), 0) - if n, _ := pp.ns.GetField(node, pp.ppNodeInfoField).(*ppNodeInfo); n != nil { + ns.SetStateSub(node, nodestate.Flags{}, pp.setup.activeFlag.Or(pp.setup.inactiveFlag), 0) + if n, _ := pp.ns.GetField(node, pp.setup.queueField).(*ppNodeInfo); n != nil { pp.disconnectedNode(n) } - ns.SetFieldSub(node, pp.CapacityField, nil) - ns.SetFieldSub(node, pp.ppNodeInfoField, nil) + ns.SetFieldSub(node, pp.setup.capacityField, nil) + ns.SetFieldSub(node, pp.setup.queueField, nil) } }) - ns.SubscribeState(pp.ActiveFlag.Or(pp.InactiveFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { - if c, _ := pp.ns.GetField(node, pp.ppNodeInfoField).(*ppNodeInfo); c != nil { + ns.SubscribeState(pp.setup.activeFlag.Or(pp.setup.inactiveFlag), func(node *enode.Node, oldState, newState nodestate.Flags) { + if c, _ := pp.ns.GetField(node, pp.setup.queueField).(*ppNodeInfo); c != nil { if oldState.IsEmpty() { pp.connectedNode(c) } @@ -170,7 +141,7 @@ func NewPriorityPool(ns *nodestate.NodeStateMachine, setup PriorityPoolSetup, cl } } }) - ns.SubscribeState(pp.updateFlag, func(node *enode.Node, oldState, newState nodestate.Flags) { + ns.SubscribeState(pp.setup.updateFlag, func(node *enode.Node, oldState, newState nodestate.Flags) { if !newState.IsEmpty() { pp.updatePriority(node) } @@ -178,18 +149,12 @@ func NewPriorityPool(ns *nodestate.NodeStateMachine, setup PriorityPoolSetup, cl return pp } -// RequestCapacity checks whether changing the capacity of a node to the given target -// is possible (bias is applied in favor of other active nodes if the target is higher -// than the current capacity). -// If setCap is true then it also performs the change if possible. The function returns -// the minimum priority needed to do the change and whether it is currently allowed. -// If setCap and allowed are both true then the caller can assume that the change was -// successful. -// Note: priorityField should always be set before calling RequestCapacity. If setCap -// is false then both InactiveFlag and ActiveFlag can be unset and they are not changed -// by this function call either. -// Note 2: this function should run inside a NodeStateMachine operation -func (pp *PriorityPool) RequestCapacity(node *enode.Node, targetCap uint64, bias time.Duration, setCap bool) (minPriority int64, allowed bool) { +// requestCapacity tries to set the capacity of a connected node to the highest possible +// value inside the given target range. If maxTarget is not reachable then the capacity is +// iteratively reduced in fine steps based on the fineStepDiv parameter until minTarget is reached. +// The function returns the new capacity if successful and the original capacity otherwise. +// Note: this function should run inside a NodeStateMachine operation +func (pp *priorityPool) requestCapacity(node *enode.Node, minTarget, maxTarget uint64, bias time.Duration) uint64 { pp.lock.Lock() pp.activeQueue.Refresh() var updates []capUpdate @@ -198,39 +163,37 @@ func (pp *PriorityPool) RequestCapacity(node *enode.Node, targetCap uint64, bias pp.updateFlags(updates) }() - if targetCap < pp.minCap { - targetCap = pp.minCap + if minTarget < pp.minCap { + minTarget = pp.minCap + } + if maxTarget < minTarget { + maxTarget = minTarget } if bias < pp.activeBias { bias = pp.activeBias } - c, _ := pp.ns.GetField(node, pp.ppNodeInfoField).(*ppNodeInfo) + c, _ := pp.ns.GetField(node, pp.setup.queueField).(*ppNodeInfo) if c == nil { - log.Error("RequestCapacity called for unknown node", "id", node.ID()) - return math.MaxInt64, false + log.Error("requestCapacity called for unknown node", "id", node.ID()) + return 0 } - var priority int64 - if targetCap > c.capacity { - priority = c.nodePriority.EstimatePriority(targetCap, 0, 0, bias, false) - } else { - priority = c.nodePriority.Priority(targetCap) + pp.setTempState(c) + if maxTarget > c.capacity { + c.bias = bias + c.stepDiv = pp.fineStepDiv } - pp.markForChange(c) - pp.setCapacity(c, targetCap) - c.forced = true + pp.setTempCapacity(c, maxTarget) + c.minTarget = minTarget pp.activeQueue.Remove(c.activeIndex) pp.inactiveQueue.Remove(c.inactiveIndex) pp.activeQueue.Push(c) - _, minPriority = pp.enforceLimits() - // if capacity update is possible now then minPriority == math.MinInt64 - // if it is not possible at all then minPriority == math.MaxInt64 - allowed = priority > minPriority - updates = pp.finalizeChanges(setCap && allowed) - return + pp.enforceLimits() + updates = pp.finalizeChanges(c.tempCapacity >= minTarget && c.tempCapacity <= maxTarget && c.tempCapacity != c.capacity) + return c.capacity } // SetLimits sets the maximum number and total capacity of simultaneously active nodes -func (pp *PriorityPool) SetLimits(maxCount, maxCap uint64) { +func (pp *priorityPool) SetLimits(maxCount, maxCap uint64) { pp.lock.Lock() pp.activeQueue.Refresh() var updates []capUpdate @@ -247,27 +210,38 @@ func (pp *PriorityPool) SetLimits(maxCount, maxCap uint64) { updates = pp.finalizeChanges(true) } if inc { - updates = pp.tryActivate() + updates = append(updates, pp.tryActivate(false)...) } } -// SetActiveBias sets the bias applied when trying to activate inactive nodes -func (pp *PriorityPool) SetActiveBias(bias time.Duration) { +// setActiveBias sets the bias applied when trying to activate inactive nodes +func (pp *priorityPool) setActiveBias(bias time.Duration) { pp.lock.Lock() - defer pp.lock.Unlock() - pp.activeBias = bias - pp.tryActivate() + if pp.activeBias < time.Duration(1) { + pp.activeBias = time.Duration(1) + } + updates := pp.tryActivate(false) + pp.lock.Unlock() + pp.ns.Operation(func() { pp.updateFlags(updates) }) } // Active returns the number and total capacity of currently active nodes -func (pp *PriorityPool) Active() (uint64, uint64) { +func (pp *priorityPool) Active() (uint64, uint64) { pp.lock.Lock() defer pp.lock.Unlock() return pp.activeCount, pp.activeCap } +// Limits returns the maximum allowed number and total capacity of active nodes +func (pp *priorityPool) Limits() (uint64, uint64) { + pp.lock.Lock() + defer pp.lock.Unlock() + + return pp.maxCount, pp.maxCap +} + // inactiveSetIndex callback updates ppNodeInfo item index in inactiveQueue func inactiveSetIndex(a interface{}, index int) { a.(*ppNodeInfo).inactiveIndex = index @@ -290,37 +264,31 @@ func invertPriority(p int64) int64 { // activePriority callback returns actual priority of ppNodeInfo item in activeQueue func activePriority(a interface{}) int64 { c := a.(*ppNodeInfo) - if c.forced { - return math.MinInt64 - } if c.bias == 0 { - return invertPriority(c.nodePriority.Priority(c.capacity)) + return invertPriority(c.nodePriority.priority(c.tempCapacity)) } else { - return invertPriority(c.nodePriority.EstimatePriority(c.capacity, 0, 0, c.bias, true)) + return invertPriority(c.nodePriority.estimatePriority(c.tempCapacity, 0, 0, c.bias, true)) } } // activeMaxPriority callback returns estimated maximum priority of ppNodeInfo item in activeQueue -func (pp *PriorityPool) activeMaxPriority(a interface{}, until mclock.AbsTime) int64 { +func (pp *priorityPool) activeMaxPriority(a interface{}, until mclock.AbsTime) int64 { c := a.(*ppNodeInfo) - if c.forced { - return math.MinInt64 - } future := time.Duration(until - pp.clock.Now()) if future < 0 { future = 0 } - return invertPriority(c.nodePriority.EstimatePriority(c.capacity, 0, future, c.bias, false)) + return invertPriority(c.nodePriority.estimatePriority(c.tempCapacity, 0, future, c.bias, false)) } // inactivePriority callback returns actual priority of ppNodeInfo item in inactiveQueue -func (pp *PriorityPool) inactivePriority(p *ppNodeInfo) int64 { - return p.nodePriority.Priority(pp.minCap) +func (pp *priorityPool) inactivePriority(p *ppNodeInfo) int64 { + return p.nodePriority.priority(pp.minCap) } -// connectedNode is called when a new node has been added to the pool (InactiveFlag set) +// connectedNode is called when a new node has been added to the pool (inactiveFlag set) // Note: this function should run inside a NodeStateMachine operation -func (pp *PriorityPool) connectedNode(c *ppNodeInfo) { +func (pp *priorityPool) connectedNode(c *ppNodeInfo) { pp.lock.Lock() pp.activeQueue.Refresh() var updates []capUpdate @@ -334,13 +302,13 @@ func (pp *PriorityPool) connectedNode(c *ppNodeInfo) { } c.connected = true pp.inactiveQueue.Push(c, pp.inactivePriority(c)) - updates = pp.tryActivate() + updates = pp.tryActivate(false) } -// disconnectedNode is called when a node has been removed from the pool (both InactiveFlag -// and ActiveFlag reset) +// disconnectedNode is called when a node has been removed from the pool (both inactiveFlag +// and activeFlag reset) // Note: this function should run inside a NodeStateMachine operation -func (pp *PriorityPool) disconnectedNode(c *ppNodeInfo) { +func (pp *priorityPool) disconnectedNode(c *ppNodeInfo) { pp.lock.Lock() pp.activeQueue.Refresh() var updates []capUpdate @@ -356,42 +324,51 @@ func (pp *PriorityPool) disconnectedNode(c *ppNodeInfo) { pp.activeQueue.Remove(c.activeIndex) pp.inactiveQueue.Remove(c.inactiveIndex) if c.capacity != 0 { - pp.setCapacity(c, 0) - updates = pp.tryActivate() + pp.setTempState(c) + pp.setTempCapacity(c, 0) + updates = pp.tryActivate(true) } } -// markForChange internally puts a node in a temporary state that can either be reverted +// setTempState internally puts a node in a temporary state that can either be reverted // or confirmed later. This temporary state allows changing the capacity of a node and -// moving it between the active and inactive queue. ActiveFlag/InactiveFlag and -// CapacityField are not changed while the changes are still temporary. -func (pp *PriorityPool) markForChange(c *ppNodeInfo) { - if c.changed { +// moving it between the active and inactive queue. activeFlag/inactiveFlag and +// capacityField are not changed while the changes are still temporary. +func (pp *priorityPool) setTempState(c *ppNodeInfo) { + if c.tempState { return } - c.changed = true - c.origCap = c.capacity - pp.changed = append(pp.changed, c) + c.tempState = true + if c.tempCapacity != c.capacity { // should never happen + log.Error("tempCapacity != capacity when entering tempState") + } + c.minTarget = pp.minCap + c.stepDiv = pp.capacityStepDiv + pp.tempState = append(pp.tempState, c) } -// setCapacity changes the capacity of a node and adjusts activeCap and activeCount -// accordingly. Note that this change is performed in the temporary state so it should -// be called after markForChange and before finalizeChanges. -func (pp *PriorityPool) setCapacity(n *ppNodeInfo, cap uint64) { - pp.activeCap += cap - n.capacity - if n.capacity == 0 { +// setTempCapacity changes the capacity of a node in the temporary state and adjusts +// activeCap and activeCount accordingly. Since this change is performed in the temporary +// state it should be called after setTempState and before finalizeChanges. +func (pp *priorityPool) setTempCapacity(n *ppNodeInfo, cap uint64) { + if !n.tempState { // should never happen + log.Error("Node is not in temporary state") + return + } + pp.activeCap += cap - n.tempCapacity + if n.tempCapacity == 0 { pp.activeCount++ } if cap == 0 { pp.activeCount-- } - n.capacity = cap + n.tempCapacity = cap } // enforceLimits enforces active node count and total capacity limits. It returns the // lowest active node priority. Note that this function is performed on the temporary // internal state. -func (pp *PriorityPool) enforceLimits() (*ppNodeInfo, int64) { +func (pp *priorityPool) enforceLimits() (*ppNodeInfo, int64) { if pp.activeCap <= pp.maxCap && pp.activeCount <= pp.maxCount { return nil, math.MinInt64 } @@ -401,16 +378,19 @@ func (pp *PriorityPool) enforceLimits() (*ppNodeInfo, int64) { ) pp.activeQueue.MultiPop(func(data interface{}, priority int64) bool { c = data.(*ppNodeInfo) - pp.markForChange(c) + pp.setTempState(c) maxActivePriority = priority - if c.capacity == pp.minCap || pp.activeCount > pp.maxCount { - pp.setCapacity(c, 0) + if c.tempCapacity == c.minTarget || pp.activeCount > pp.maxCount { + pp.setTempCapacity(c, 0) } else { - sub := c.capacity / pp.capacityStepDiv - if c.capacity-sub < pp.minCap { - sub = c.capacity - pp.minCap + sub := c.tempCapacity / c.stepDiv + if sub == 0 { + sub = 1 } - pp.setCapacity(c, c.capacity-sub) + if c.tempCapacity-sub < c.minTarget { + sub = c.tempCapacity - c.minTarget + } + pp.setTempCapacity(c, c.tempCapacity-sub) pp.activeQueue.Push(c) } return pp.activeCap > pp.maxCap || pp.activeCount > pp.maxCount @@ -421,71 +401,74 @@ func (pp *PriorityPool) enforceLimits() (*ppNodeInfo, int64) { // finalizeChanges either commits or reverts temporary changes. The necessary capacity // field and according flag updates are not performed here but returned in a list because // they should be performed while the mutex is not held. -func (pp *PriorityPool) finalizeChanges(commit bool) (updates []capUpdate) { - for _, c := range pp.changed { - // always remove and push back in order to update biased/forced priority +func (pp *priorityPool) finalizeChanges(commit bool) (updates []capUpdate) { + for _, c := range pp.tempState { + // always remove and push back in order to update biased priority pp.activeQueue.Remove(c.activeIndex) pp.inactiveQueue.Remove(c.inactiveIndex) - c.bias = 0 - c.forced = false - c.changed = false - if !commit { - pp.setCapacity(c, c.origCap) + oldCapacity := c.capacity + if commit { + c.capacity = c.tempCapacity + } else { + pp.setTempCapacity(c, c.capacity) // revert activeCount/activeCap } + c.tempState = false + c.bias = 0 + c.stepDiv = pp.capacityStepDiv + c.minTarget = pp.minCap if c.connected { if c.capacity != 0 { pp.activeQueue.Push(c) } else { pp.inactiveQueue.Push(c, pp.inactivePriority(c)) } - if c.capacity != c.origCap && commit { - updates = append(updates, capUpdate{c.node, c.origCap, c.capacity}) + if c.capacity != oldCapacity { + updates = append(updates, capUpdate{c.node, oldCapacity, c.capacity}) } } - c.origCap = 0 } - pp.changed = nil + pp.tempState = nil if commit { pp.ccUpdateForced = true } return } -// capUpdate describes a CapacityField and ActiveFlag/InactiveFlag update +// capUpdate describes a capacityField and activeFlag/inactiveFlag update type capUpdate struct { node *enode.Node oldCap, newCap uint64 } -// updateFlags performs CapacityField and ActiveFlag/InactiveFlag updates while the +// updateFlags performs capacityField and activeFlag/inactiveFlag updates while the // pool mutex is not held // Note: this function should run inside a NodeStateMachine operation -func (pp *PriorityPool) updateFlags(updates []capUpdate) { +func (pp *priorityPool) updateFlags(updates []capUpdate) { for _, f := range updates { if f.oldCap == 0 { - pp.ns.SetStateSub(f.node, pp.ActiveFlag, pp.InactiveFlag, 0) + pp.ns.SetStateSub(f.node, pp.setup.activeFlag, pp.setup.inactiveFlag, 0) } if f.newCap == 0 { - pp.ns.SetStateSub(f.node, pp.InactiveFlag, pp.ActiveFlag, 0) - pp.ns.SetFieldSub(f.node, pp.CapacityField, nil) + pp.ns.SetStateSub(f.node, pp.setup.inactiveFlag, pp.setup.activeFlag, 0) + pp.ns.SetFieldSub(f.node, pp.setup.capacityField, nil) } else { - pp.ns.SetFieldSub(f.node, pp.CapacityField, f.newCap) + pp.ns.SetFieldSub(f.node, pp.setup.capacityField, f.newCap) } } } // tryActivate tries to activate inactive nodes if possible -func (pp *PriorityPool) tryActivate() []capUpdate { - var commit bool +func (pp *priorityPool) tryActivate(commit bool) []capUpdate { for pp.inactiveQueue.Size() > 0 { c := pp.inactiveQueue.PopItem().(*ppNodeInfo) - pp.markForChange(c) - pp.setCapacity(c, pp.minCap) + pp.setTempState(c) + pp.setTempCapacity(c, pp.minCap) c.bias = pp.activeBias pp.activeQueue.Push(c) pp.enforceLimits() - if c.capacity > 0 { + if c.tempCapacity > 0 { commit = true + c.bias = 0 } else { break } @@ -497,7 +480,7 @@ func (pp *PriorityPool) tryActivate() []capUpdate { // updatePriority gets the current priority value of the given node from the nodePriority // interface and performs the necessary changes. It is triggered by updateFlag. // Note: this function should run inside a NodeStateMachine operation -func (pp *PriorityPool) updatePriority(node *enode.Node) { +func (pp *priorityPool) updatePriority(node *enode.Node) { pp.lock.Lock() pp.activeQueue.Refresh() var updates []capUpdate @@ -506,7 +489,7 @@ func (pp *PriorityPool) updatePriority(node *enode.Node) { pp.updateFlags(updates) }() - c, _ := pp.ns.GetField(node, pp.ppNodeInfoField).(*ppNodeInfo) + c, _ := pp.ns.GetField(node, pp.setup.queueField).(*ppNodeInfo) if c == nil || !c.connected { return } @@ -517,15 +500,15 @@ func (pp *PriorityPool) updatePriority(node *enode.Node) { } else { pp.inactiveQueue.Push(c, pp.inactivePriority(c)) } - updates = pp.tryActivate() + updates = pp.tryActivate(false) } -// CapacityCurve is a snapshot of the priority pool contents in a format that can efficiently +// capacityCurve is a snapshot of the priority pool contents in a format that can efficiently // estimate how much capacity could be granted to a given node at a given priority level. -type CapacityCurve struct { +type capacityCurve struct { points []curvePoint // curve points sorted in descending order of priority index map[enode.ID][]int // curve point indexes belonging to each node - exclude []int // curve point indexes of excluded node + excludeList []int // curve point indexes of excluded node excludeFirst bool // true if activeCount == maxCount } @@ -534,8 +517,8 @@ type curvePoint struct { nextPri int64 // next priority level where more capacity will be available } -// GetCapacityCurve returns a new or recently cached CapacityCurve based on the contents of the pool -func (pp *PriorityPool) GetCapacityCurve() *CapacityCurve { +// getCapacityCurve returns a new or recently cached capacityCurve based on the contents of the pool +func (pp *priorityPool) getCapacityCurve() *capacityCurve { pp.lock.Lock() defer pp.lock.Unlock() @@ -547,7 +530,7 @@ func (pp *PriorityPool) GetCapacityCurve() *CapacityCurve { pp.ccUpdateForced = false pp.ccUpdatedAt = now - curve := &CapacityCurve{ + curve := &capacityCurve{ index: make(map[enode.ID][]int), } pp.cachedCurve = curve @@ -556,6 +539,7 @@ func (pp *PriorityPool) GetCapacityCurve() *CapacityCurve { excludeFirst := pp.maxCount == pp.activeCount // reduce node capacities or remove nodes until nothing is left in the queue; // record the available capacity and the necessary priority after each step + lastPri := int64(math.MinInt64) for pp.activeCap > 0 { cp := curvePoint{} if pp.activeCap > pp.maxCap { @@ -570,9 +554,15 @@ func (pp *PriorityPool) GetCapacityCurve() *CapacityCurve { // enforceLimits removes the lowest priority node if it has minimal capacity, // otherwise reduces its capacity next, cp.nextPri = pp.enforceLimits() + if cp.nextPri < lastPri { + // enforce monotonicity which may be broken by continuously changing priorities + cp.nextPri = lastPri + } else { + lastPri = cp.nextPri + } pp.activeCap -= tempCap if next == nil { - log.Error("GetCapacityCurve: cannot remove next element from the priority queue") + log.Error("getCapacityCurve: cannot remove next element from the priority queue") break } id := next.node.ID() @@ -595,34 +585,34 @@ func (pp *PriorityPool) GetCapacityCurve() *CapacityCurve { nextPri: math.MaxInt64, }) if curve.excludeFirst { - curve.exclude = curve.index[excludeID] + curve.excludeList = curve.index[excludeID] } return curve } -// Exclude returns a CapacityCurve with the given node excluded from the original curve -func (cc *CapacityCurve) Exclude(id enode.ID) *CapacityCurve { - if exclude, ok := cc.index[id]; ok { +// exclude returns a capacityCurve with the given node excluded from the original curve +func (cc *capacityCurve) exclude(id enode.ID) *capacityCurve { + if excludeList, ok := cc.index[id]; ok { // return a new version of the curve (only one excluded node can be selected) // Note: if the first node was excluded by default (excludeFirst == true) then // we can forget about that and exclude the node with the given id instead. - return &CapacityCurve{ - points: cc.points, - index: cc.index, - exclude: exclude, + return &capacityCurve{ + points: cc.points, + index: cc.index, + excludeList: excludeList, } } return cc } -func (cc *CapacityCurve) getPoint(i int) curvePoint { +func (cc *capacityCurve) getPoint(i int) curvePoint { cp := cc.points[i] if i == 0 && cc.excludeFirst { cp.freeCap = 0 return cp } - for ii := len(cc.exclude) - 1; ii >= 0; ii-- { - ei := cc.exclude[ii] + for ii := len(cc.excludeList) - 1; ii >= 0; ii-- { + ei := cc.excludeList[ii] if ei < i { break } @@ -632,11 +622,11 @@ func (cc *CapacityCurve) getPoint(i int) curvePoint { return cp } -// MaxCapacity calculates the maximum capacity available for a node with a given +// maxCapacity calculates the maximum capacity available for a node with a given // (monotonically decreasing) priority vs. capacity function. Note that if the requesting // node is already in the pool then it should be excluded from the curve in order to get // the correct result. -func (cc *CapacityCurve) MaxCapacity(priority func(cap uint64) int64) uint64 { +func (cc *capacityCurve) maxCapacity(priority func(cap uint64) int64) uint64 { min, max := 0, len(cc.points)-1 // the curve always has at least one point for min < max { mid := (min + max) / 2 diff --git a/les/vflux/server/prioritypool_test.go b/les/vflux/server/prioritypool_test.go index d83ddc1767..5152312116 100644 --- a/les/vflux/server/prioritypool_test.go +++ b/les/vflux/server/prioritypool_test.go @@ -28,18 +28,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/nodestate" ) -var ( - testSetup = &nodestate.Setup{} - ppTestClientFlag = testSetup.NewFlag("ppTestClientFlag") - ppTestClientField = testSetup.NewField("ppTestClient", reflect.TypeOf(&ppTestClient{})) - ppUpdateFlag = testSetup.NewFlag("ppUpdateFlag") - ppTestSetup = NewPriorityPoolSetup(testSetup) -) - -func init() { - ppTestSetup.Connect(ppTestClientField, ppUpdateFlag) -} - const ( testCapacityStepDiv = 100 testCapacityToleranceDiv = 10 @@ -51,25 +39,27 @@ type ppTestClient struct { balance, cap uint64 } -func (c *ppTestClient) Priority(cap uint64) int64 { +func (c *ppTestClient) priority(cap uint64) int64 { return int64(c.balance / cap) } -func (c *ppTestClient) EstimatePriority(cap uint64, addBalance int64, future, bias time.Duration, update bool) int64 { +func (c *ppTestClient) estimatePriority(cap uint64, addBalance int64, future, bias time.Duration, update bool) int64 { return int64(c.balance / cap) } func TestPriorityPool(t *testing.T) { clock := &mclock.Simulated{} - ns := nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) + setup := newServerSetup() + setup.balanceField = setup.setup.NewField("ppTestClient", reflect.TypeOf(&ppTestClient{})) + ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) - ns.SubscribeField(ppTestSetup.CapacityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { - if n := ns.GetField(node, ppTestSetup.priorityField); n != nil { + ns.SubscribeField(setup.capacityField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + if n := ns.GetField(node, setup.balanceField); n != nil { c := n.(*ppTestClient) c.cap = newValue.(uint64) } }) - pp := NewPriorityPool(ns, ppTestSetup, clock, testMinCap, 0, testCapacityStepDiv) + pp := newPriorityPool(ns, setup, clock, testMinCap, 0, testCapacityStepDiv, testCapacityStepDiv) ns.Start() pp.SetLimits(100, 1000000) clients := make([]*ppTestClient, 100) @@ -77,7 +67,8 @@ func TestPriorityPool(t *testing.T) { for { var ok bool ns.Operation(func() { - _, ok = pp.RequestCapacity(c.node, c.cap+c.cap/testCapacityStepDiv, 0, true) + newCap := c.cap + c.cap/testCapacityStepDiv + ok = pp.requestCapacity(c.node, newCap, newCap, 0) == newCap }) if !ok { return @@ -101,9 +92,8 @@ func TestPriorityPool(t *testing.T) { } sumBalance += c.balance clients[i] = c - ns.SetState(c.node, ppTestClientFlag, nodestate.Flags{}, 0) - ns.SetField(c.node, ppTestSetup.priorityField, c) - ns.SetState(c.node, ppTestSetup.InactiveFlag, nodestate.Flags{}, 0) + ns.SetField(c.node, setup.balanceField, c) + ns.SetState(c.node, setup.inactiveFlag, nodestate.Flags{}, 0) raise(c) check(c) } @@ -113,8 +103,8 @@ func TestPriorityPool(t *testing.T) { oldBalance := c.balance c.balance = uint64(rand.Int63n(100000000000) + 100000000000) sumBalance += c.balance - oldBalance - pp.ns.SetState(c.node, ppUpdateFlag, nodestate.Flags{}, 0) - pp.ns.SetState(c.node, nodestate.Flags{}, ppUpdateFlag, 0) + pp.ns.SetState(c.node, setup.updateFlag, nodestate.Flags{}, 0) + pp.ns.SetState(c.node, nodestate.Flags{}, setup.updateFlag, 0) if c.balance > oldBalance { raise(c) } else { @@ -129,32 +119,28 @@ func TestPriorityPool(t *testing.T) { if count%10 == 0 { // test available capacity calculation with capacity curve c = clients[rand.Intn(len(clients))] - curve := pp.GetCapacityCurve().Exclude(c.node.ID()) + curve := pp.getCapacityCurve().exclude(c.node.ID()) add := uint64(rand.Int63n(10000000000000)) c.balance += add sumBalance += add - expCap := curve.MaxCapacity(func(cap uint64) int64 { + expCap := curve.maxCapacity(func(cap uint64) int64 { return int64(c.balance / cap) }) - //fmt.Println(expCap, c.balance, sumBalance) - /*for i, cp := range curve.points { - fmt.Println("cp", i, cp, "ex", curve.getPoint(i)) - }*/ var ok bool - expFail := expCap + 1 + expFail := expCap + 10 if expFail < testMinCap { expFail = testMinCap } ns.Operation(func() { - _, ok = pp.RequestCapacity(c.node, expFail, 0, true) + ok = pp.requestCapacity(c.node, expFail, expFail, 0) == expFail }) if ok { t.Errorf("Request for more than expected available capacity succeeded") } if expCap >= testMinCap { ns.Operation(func() { - _, ok = pp.RequestCapacity(c.node, expCap, 0, true) + ok = pp.requestCapacity(c.node, expCap, expCap, 0) == expCap }) if !ok { t.Errorf("Request for expected available capacity failed") @@ -162,8 +148,8 @@ func TestPriorityPool(t *testing.T) { } c.balance -= add sumBalance -= add - pp.ns.SetState(c.node, ppUpdateFlag, nodestate.Flags{}, 0) - pp.ns.SetState(c.node, nodestate.Flags{}, ppUpdateFlag, 0) + pp.ns.SetState(c.node, setup.updateFlag, nodestate.Flags{}, 0) + pp.ns.SetState(c.node, nodestate.Flags{}, setup.updateFlag, 0) for _, c := range clients { raise(c) } @@ -175,8 +161,11 @@ func TestPriorityPool(t *testing.T) { func TestCapacityCurve(t *testing.T) { clock := &mclock.Simulated{} - ns := nodestate.NewNodeStateMachine(nil, nil, clock, testSetup) - pp := NewPriorityPool(ns, ppTestSetup, clock, 400000, 0, 2) + setup := newServerSetup() + setup.balanceField = setup.setup.NewField("ppTestClient", reflect.TypeOf(&ppTestClient{})) + ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) + + pp := newPriorityPool(ns, setup, clock, 400000, 0, 2, 2) ns.Start() pp.SetLimits(10, 10000000) clients := make([]*ppTestClient, 10) @@ -188,17 +177,16 @@ func TestCapacityCurve(t *testing.T) { cap: 1000000, } clients[i] = c - ns.SetState(c.node, ppTestClientFlag, nodestate.Flags{}, 0) - ns.SetField(c.node, ppTestSetup.priorityField, c) - ns.SetState(c.node, ppTestSetup.InactiveFlag, nodestate.Flags{}, 0) + ns.SetField(c.node, setup.balanceField, c) + ns.SetState(c.node, setup.inactiveFlag, nodestate.Flags{}, 0) ns.Operation(func() { - pp.RequestCapacity(c.node, c.cap, 0, true) + pp.requestCapacity(c.node, c.cap, c.cap, 0) }) } - curve := pp.GetCapacityCurve() + curve := pp.getCapacityCurve() check := func(balance, expCap uint64) { - cap := curve.MaxCapacity(func(cap uint64) int64 { + cap := curve.maxCapacity(func(cap uint64) int64 { return int64(balance / cap) }) var fail bool @@ -226,7 +214,7 @@ func TestCapacityCurve(t *testing.T) { check(1000000000000, 2500000) pp.SetLimits(11, 10000000) - curve = pp.GetCapacityCurve() + curve = pp.getCapacityCurve() check(0, 0) check(10000000000, 100000) diff --git a/les/vflux/server/service.go b/les/vflux/server/service.go index ab759ae441..80a0f47543 100644 --- a/les/vflux/server/service.go +++ b/les/vflux/server/service.go @@ -40,7 +40,6 @@ type ( // Service is a service registered at the Server and identified by a string id Service interface { - ServiceInfo() (id, desc string) // only called during registration Handle(id enode.ID, address string, name string, data []byte) []byte // never called concurrently } @@ -60,9 +59,8 @@ func NewServer(delayPerRequest time.Duration) *Server { } // Register registers a Service -func (s *Server) Register(b Service) { - srv := &serviceEntry{backend: b} - srv.id, srv.desc = b.ServiceInfo() +func (s *Server) Register(b Service, id, desc string) { + srv := &serviceEntry{backend: b, id: id, desc: desc} if strings.Contains(srv.id, ":") { // srv.id + ":" will be used as a service database prefix log.Error("Service ID contains ':'", "id", srv.id) diff --git a/les/vflux/server/status.go b/les/vflux/server/status.go new file mode 100644 index 0000000000..469190777b --- /dev/null +++ b/les/vflux/server/status.go @@ -0,0 +1,59 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package server + +import ( + "reflect" + + "github.com/ethereum/go-ethereum/p2p/nodestate" +) + +type peerWrapper struct{ clientPeer } // the NodeStateMachine type system needs this wrapper + +// serverSetup is a wrapper of the node state machine setup, which contains +// all the created flags and fields used in the vflux server side. +type serverSetup struct { + setup *nodestate.Setup + clientField nodestate.Field // Field contains the client peer handler + + // Flags and fields controlled by balance tracker. BalanceTracker + // is responsible for setting/deleting these flags or fields. + priorityFlag nodestate.Flags // Flag is set if the node has a positive balance + updateFlag nodestate.Flags // Flag is set whenever the node balance is changed(priority changed) + balanceField nodestate.Field // Field contains the client balance for priority calculation + + // Flags and fields controlled by priority queue. Priority queue + // is responsible for setting/deleting these flags or fields. + activeFlag nodestate.Flags // Flag is set if the node is active + inactiveFlag nodestate.Flags // Flag is set if the node is inactive + capacityField nodestate.Field // Field contains the capacity of the node + queueField nodestate.Field // Field contains the infomration in the priority queue +} + +// newServerSetup initializes the setup for state machine and returns the flags/fields group. +func newServerSetup() *serverSetup { + setup := &serverSetup{setup: &nodestate.Setup{}} + setup.clientField = setup.setup.NewField("client", reflect.TypeOf(peerWrapper{})) + setup.priorityFlag = setup.setup.NewFlag("priority") + setup.updateFlag = setup.setup.NewFlag("update") + setup.balanceField = setup.setup.NewField("balance", reflect.TypeOf(&nodeBalance{})) + setup.activeFlag = setup.setup.NewFlag("active") + setup.inactiveFlag = setup.setup.NewFlag("inactive") + setup.capacityField = setup.setup.NewField("capacity", reflect.TypeOf(uint64(0))) + setup.queueField = setup.setup.NewField("queue", reflect.TypeOf(&ppNodeInfo{})) + return setup +} diff --git a/oss-fuzz.sh b/oss-fuzz.sh index ac93a5a467..f8152f0fad 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -102,6 +102,7 @@ compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty compile_fuzzer tests/fuzzers/abi Fuzz fuzzAbi compile_fuzzer tests/fuzzers/les Fuzz fuzzLes +compile_fuzzer tests/fuzzers/vflux FuzzClientPool fuzzClientPool compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul diff --git a/p2p/nodestate/nodestate.go b/p2p/nodestate/nodestate.go index d3166f1d87..9323d53cbd 100644 --- a/p2p/nodestate/nodestate.go +++ b/p2p/nodestate/nodestate.go @@ -858,6 +858,23 @@ func (ns *NodeStateMachine) GetField(n *enode.Node, field Field) interface{} { return nil } +// GetState retrieves the current state of the given node. Note that when used in a +// subscription callback the result can be out of sync with the state change represented +// by the callback parameters so extra safety checks might be necessary. +func (ns *NodeStateMachine) GetState(n *enode.Node) Flags { + ns.lock.Lock() + defer ns.lock.Unlock() + + ns.checkStarted() + if ns.closed { + return Flags{} + } + if _, node := ns.updateEnode(n); node != nil { + return Flags{mask: node.state, setup: ns.setup} + } + return Flags{} +} + // SetField sets the given field of the given node and blocks until the operation is finished func (ns *NodeStateMachine) SetField(n *enode.Node, field Field, value interface{}) error { ns.lock.Lock() diff --git a/tests/fuzzers/vflux/clientpool-fuzzer.go b/tests/fuzzers/vflux/clientpool-fuzzer.go new file mode 100644 index 0000000000..41b8627348 --- /dev/null +++ b/tests/fuzzers/vflux/clientpool-fuzzer.go @@ -0,0 +1,289 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package vflux + +import ( + "bytes" + "encoding/binary" + "io" + "math" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/les/vflux" + vfs "github.com/ethereum/go-ethereum/les/vflux/server" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rlp" +) + +type fuzzer struct { + peers [256]*clientPeer + disconnectList []*clientPeer + input io.Reader + exhausted bool + activeCount, activeCap uint64 + maxCount, maxCap uint64 +} + +type clientPeer struct { + fuzzer *fuzzer + node *enode.Node + freeID string + timeout time.Duration + + balance vfs.ConnectedBalance + capacity uint64 +} + +func (p *clientPeer) Node() *enode.Node { + return p.node +} + +func (p *clientPeer) FreeClientId() string { + return p.freeID +} + +func (p *clientPeer) InactiveAllowance() time.Duration { + return p.timeout +} + +func (p *clientPeer) UpdateCapacity(newCap uint64, requested bool) { + p.fuzzer.activeCap -= p.capacity + if p.capacity != 0 { + p.fuzzer.activeCount-- + } + p.capacity = newCap + p.fuzzer.activeCap += p.capacity + if p.capacity != 0 { + p.fuzzer.activeCount++ + } +} + +func (p *clientPeer) Disconnect() { + p.fuzzer.disconnectList = append(p.fuzzer.disconnectList, p) + p.fuzzer.activeCap -= p.capacity + if p.capacity != 0 { + p.fuzzer.activeCount-- + } + p.capacity = 0 + p.balance = nil +} + +func newFuzzer(input []byte) *fuzzer { + f := &fuzzer{ + input: bytes.NewReader(input), + } + for i := range f.peers { + f.peers[i] = &clientPeer{ + fuzzer: f, + node: enode.SignNull(new(enr.Record), enode.ID{byte(i)}), + freeID: string([]byte{byte(i)}), + timeout: f.randomDelay(), + } + } + return f +} + +func (f *fuzzer) read(size int) []byte { + out := make([]byte, size) + if _, err := f.input.Read(out); err != nil { + f.exhausted = true + } + return out +} + +func (f *fuzzer) randomByte() byte { + d := f.read(1) + return d[0] +} + +func (f *fuzzer) randomBool() bool { + d := f.read(1) + return d[0]&1 == 1 +} + +func (f *fuzzer) randomInt(max int) int { + if max == 0 { + return 0 + } + if max <= 256 { + return int(f.randomByte()) % max + } + var a uint16 + if err := binary.Read(f.input, binary.LittleEndian, &a); err != nil { + f.exhausted = true + } + return int(a % uint16(max)) +} + +func (f *fuzzer) randomTokenAmount(signed bool) int64 { + x := uint64(f.randomInt(65000)) + x = x * x * x * x + + if signed && (x&1) == 1 { + if x <= math.MaxInt64 { + return -int64(x) + } + return math.MinInt64 + } + if x <= math.MaxInt64 { + return int64(x) + } + return math.MaxInt64 +} + +func (f *fuzzer) randomDelay() time.Duration { + delay := f.randomByte() + if delay < 128 { + return time.Duration(delay) * time.Second + } + return 0 +} + +func (f *fuzzer) randomFactors() vfs.PriceFactors { + return vfs.PriceFactors{ + TimeFactor: float64(f.randomByte()) / 25500, + CapacityFactor: float64(f.randomByte()) / 255, + RequestFactor: float64(f.randomByte()) / 255, + } +} + +func (f *fuzzer) connectedBalanceOp(balance vfs.ConnectedBalance) { + switch f.randomInt(3) { + case 0: + balance.RequestServed(uint64(f.randomTokenAmount(false))) + case 1: + balance.SetPriceFactors(f.randomFactors(), f.randomFactors()) + case 2: + balance.GetBalance() + balance.GetRawBalance() + balance.GetPriceFactors() + } +} + +func (f *fuzzer) atomicBalanceOp(balance vfs.AtomicBalanceOperator) { + switch f.randomInt(3) { + case 0: + balance.AddBalance(f.randomTokenAmount(true)) + case 1: + balance.SetBalance(uint64(f.randomTokenAmount(false)), uint64(f.randomTokenAmount(false))) + case 2: + balance.GetBalance() + balance.GetRawBalance() + balance.GetPriceFactors() + } +} + +func FuzzClientPool(input []byte) int { + if len(input) > 10000 { + return -1 + } + f := newFuzzer(input) + if f.exhausted { + return 0 + } + clock := &mclock.Simulated{} + db := memorydb.New() + pool := vfs.NewClientPool(db, 10, f.randomDelay(), clock, func() bool { return true }) + pool.Start() + defer pool.Stop() + + count := 0 + for !f.exhausted && count < 1000 { + count++ + switch f.randomInt(11) { + case 0: + i := int(f.randomByte()) + f.peers[i].balance = pool.Register(f.peers[i]) + case 1: + i := int(f.randomByte()) + f.peers[i].Disconnect() + case 2: + f.maxCount = uint64(f.randomByte()) + f.maxCap = uint64(f.randomByte()) + f.maxCap *= f.maxCap + pool.SetLimits(f.maxCount, f.maxCap) + case 3: + pool.SetConnectedBias(f.randomDelay()) + case 4: + pool.SetDefaultFactors(f.randomFactors(), f.randomFactors()) + case 5: + pool.SetExpirationTCs(uint64(f.randomInt(50000)), uint64(f.randomInt(50000))) + case 6: + if _, err := pool.SetCapacity(f.peers[f.randomByte()].node, uint64(f.randomByte()), f.randomDelay(), f.randomBool()); err == vfs.ErrCantFindMaximum { + panic(nil) + } + case 7: + if balance := f.peers[f.randomByte()].balance; balance != nil { + f.connectedBalanceOp(balance) + } + case 8: + pool.BalanceOperation(f.peers[f.randomByte()].node.ID(), f.peers[f.randomByte()].freeID, func(balance vfs.AtomicBalanceOperator) { + count := f.randomInt(4) + for i := 0; i < count; i++ { + f.atomicBalanceOp(balance) + } + }) + case 9: + pool.TotalTokenAmount() + pool.GetExpirationTCs() + pool.Active() + pool.Limits() + pool.GetPosBalanceIDs(f.peers[f.randomByte()].node.ID(), f.peers[f.randomByte()].node.ID(), f.randomInt(100)) + case 10: + req := vflux.CapacityQueryReq{ + Bias: uint64(f.randomByte()), + AddTokens: make([]vflux.IntOrInf, f.randomInt(vflux.CapacityQueryMaxLen+1)), + } + for i := range req.AddTokens { + v := vflux.IntOrInf{Type: uint8(f.randomInt(4))} + if v.Type < 2 { + v.Value = *big.NewInt(f.randomTokenAmount(false)) + } + req.AddTokens[i] = v + } + reqEnc, err := rlp.EncodeToBytes(&req) + if err != nil { + panic(err) + } + p := int(f.randomByte()) + if p < len(reqEnc) { + reqEnc[p] = f.randomByte() + } + pool.Handle(f.peers[f.randomByte()].node.ID(), f.peers[f.randomByte()].freeID, vflux.CapacityQueryName, reqEnc) + } + + for _, peer := range f.disconnectList { + pool.Unregister(peer) + } + f.disconnectList = nil + if d := f.randomDelay(); d > 0 { + clock.Run(d) + } + //fmt.Println(f.activeCount, f.maxCount, f.activeCap, f.maxCap) + if activeCount, activeCap := pool.Active(); activeCount != f.activeCount || activeCap != f.activeCap { + panic(nil) + } + if f.activeCount > f.maxCount || f.activeCap > f.maxCap { + panic(nil) + } + } + return 0 +} diff --git a/tests/fuzzers/vflux/debug/main.go b/tests/fuzzers/vflux/debug/main.go new file mode 100644 index 0000000000..de0b5d4124 --- /dev/null +++ b/tests/fuzzers/vflux/debug/main.go @@ -0,0 +1,41 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/tests/fuzzers/vflux" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: debug \n") + fmt.Fprintf(os.Stderr, "Example\n") + fmt.Fprintf(os.Stderr, " $ debug ../crashers/4bbef6857c733a87ecf6fd8b9e7238f65eb9862a\n") + os.Exit(1) + } + crasher := os.Args[1] + data, err := ioutil.ReadFile(crasher) + if err != nil { + fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) + os.Exit(1) + } + vflux.FuzzClientPool(data) +} From a600dab7e58a72968f111f8430c0e92a0511bc9c Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 7 Apr 2021 15:30:26 +0800 Subject: [PATCH 258/709] eth, les: fix tracers (#22473) * eth, les: fix tracer * eth: isolate live trie database in tracer * eth: fix nil * eth: fix * eth, les: add checkLive param * eth/tracer: fix --- eth/api.go | 3 +- eth/api_backend.go | 10 +- eth/state_accessor.go | 207 ++++++++++++++-------------------------- eth/tracers/api.go | 96 +++++++++++-------- eth/tracers/api_test.go | 36 ++----- les/api_backend.go | 8 +- les/state_accessor.go | 36 +++---- 7 files changed, 155 insertions(+), 241 deletions(-) diff --git a/eth/api.go b/eth/api.go index fb51e78c1a..e02c0ca4d2 100644 --- a/eth/api.go +++ b/eth/api.go @@ -407,11 +407,10 @@ func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, c if block == nil { return StorageRangeResult{}, fmt.Errorf("block %#x not found", blockHash) } - _, _, statedb, release, err := api.eth.stateAtTransaction(block, txIndex, 0) + _, _, statedb, err := api.eth.stateAtTransaction(block, txIndex, 0) if err != nil { return StorageRangeResult{}, err } - defer release() st := statedb.StorageTrie(contractAddress) if st == nil { return StorageRangeResult{}, fmt.Errorf("account %x doesn't exist", contractAddress) diff --git a/eth/api_backend.go b/eth/api_backend.go index 2569972e52..cc780775cc 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -332,14 +332,10 @@ func (b *EthAPIBackend) StartMining(threads int) error { return b.eth.StartMining(threads) } -func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { - return b.eth.stateAtBlock(block, reexec) +func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (*state.StateDB, error) { + return b.eth.stateAtBlock(block, reexec, base, checkLive) } -func (b *EthAPIBackend) StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { - return b.eth.statesInRange(fromBlock, toBlock, reexec) -} - -func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { +func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { return b.eth.stateAtTransaction(block, txIndex, reexec) } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index cbbd9a8202..d7564a0844 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -31,39 +31,58 @@ import ( ) // stateAtBlock retrieves the state database associated with a certain block. -// If no state is locally available for the given block, a number of blocks are -// attempted to be reexecuted to generate the desired state. -func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64) (statedb *state.StateDB, release func(), err error) { - // If we have the state fully available, use that - statedb, err = eth.blockchain.StateAt(block.Root()) - if err == nil { - return statedb, func() {}, nil +// If no state is locally available for the given block, a number of blocks +// are attempted to be reexecuted to generate the desired state. The optional +// base layer statedb can be passed then it's regarded as the statedb of the +// parent block. +func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (statedb *state.StateDB, err error) { + var ( + current *types.Block + database state.Database + report = true + origin = block.NumberU64() + ) + // Check the live database first if we have the state fully available, use that. + if checkLive { + statedb, err = eth.blockchain.StateAt(block.Root()) + if err == nil { + return statedb, nil + } } - // Otherwise try to reexec blocks until we find a state or reach our limit - origin := block.NumberU64() - database := state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16, Preimages: true}) + if base != nil { + // The optional base statedb is given, mark the start point as parent block + statedb, database, report = base, base.Database(), false + current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + } else { + // Otherwise try to reexec blocks until we find a state or reach our limit + current = block - for i := uint64(0); i < reexec; i++ { - if block.NumberU64() == 0 { - return nil, nil, errors.New("genesis state is missing") - } - parent := eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return nil, nil, fmt.Errorf("missing block %v %d", block.ParentHash(), block.NumberU64()-1) - } - block = parent + // Create an ephemeral trie.Database for isolating the live one. Otherwise + // the internal junks created by tracing will be persisted into the disk. + database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) - statedb, err = state.New(block.Root(), database, nil) - if err == nil { - break + for i := uint64(0); i < reexec; i++ { + if current.NumberU64() == 0 { + return nil, errors.New("genesis state is missing") + } + parent := eth.blockchain.GetBlock(current.ParentHash(), current.NumberU64()-1) + if parent == nil { + return nil, fmt.Errorf("missing block %v %d", current.ParentHash(), current.NumberU64()-1) + } + current = parent + + statedb, err = state.New(current.Root(), database, nil) + if err == nil { + break + } } - } - if err != nil { - switch err.(type) { - case *trie.MissingNodeError: - return nil, nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) - default: - return nil, nil, err + if err != nil { + switch err.(type) { + case *trie.MissingNodeError: + return nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) + default: + return nil, err + } } } // State was available at historical point, regenerate @@ -72,33 +91,29 @@ func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64) (statedb *s logged time.Time parent common.Hash ) - defer func() { - if err != nil && parent != (common.Hash{}) { - database.TrieDB().Dereference(parent) - } - }() - for block.NumberU64() < origin { + for current.NumberU64() < origin { // Print progress logs if long enough time elapsed - if time.Since(logged) > 8*time.Second { - log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "remaining", origin-block.NumberU64()-1, "elapsed", time.Since(start)) + if time.Since(logged) > 8*time.Second && report { + log.Info("Regenerating historical state", "block", current.NumberU64()+1, "target", origin, "remaining", origin-current.NumberU64()-1, "elapsed", time.Since(start)) logged = time.Now() } // Retrieve the next block to regenerate and process it - if block = eth.blockchain.GetBlockByNumber(block.NumberU64() + 1); block == nil { - return nil, nil, fmt.Errorf("block #%d not found", block.NumberU64()+1) + next := current.NumberU64() + 1 + if current = eth.blockchain.GetBlockByNumber(next); current == nil { + return nil, fmt.Errorf("block #%d not found", next) } - _, _, _, err := eth.blockchain.Processor().Process(block, statedb, vm.Config{}) + _, _, _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) if err != nil { - return nil, nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) + return nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(block.Number())) + root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(current.Number())) if err != nil { - return nil, nil, err + return nil, err } statedb, err = state.New(root, database, nil) if err != nil { - return nil, nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) + return nil, fmt.Errorf("state reset after block %d failed: %v", current.NumberU64(), err) } database.TrieDB().Reference(root, common.Hash{}) if parent != (common.Hash{}) { @@ -106,104 +121,32 @@ func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64) (statedb *s } parent = root } - nodes, imgs := database.TrieDB().Size() - log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) - return statedb, func() { database.TrieDB().Dereference(parent) }, nil -} - -// statesInRange retrieves a batch of state databases associated with the specific -// block ranges. If no state is locally available for the given range, a number of -// blocks are attempted to be reexecuted to generate the ancestor state. -func (eth *Ethereum) statesInRange(fromBlock, toBlock *types.Block, reexec uint64) (states []*state.StateDB, release func(), err error) { - statedb, err := eth.blockchain.StateAt(fromBlock.Root()) - if err != nil { - statedb, _, err = eth.stateAtBlock(fromBlock, reexec) - } - if err != nil { - return nil, nil, err - } - states = append(states, statedb.Copy()) - - var ( - logged time.Time - parent common.Hash - start = time.Now() - refs = []common.Hash{fromBlock.Root()} - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16, Preimages: true}) - ) - // Release all resources(including the states referenced by `stateAtBlock`) - // if error is returned. - defer func() { - if err != nil { - for _, ref := range refs { - database.TrieDB().Dereference(ref) - } - } - }() - for i := fromBlock.NumberU64() + 1; i <= toBlock.NumberU64(); i++ { - // Print progress logs if long enough time elapsed - if time.Since(logged) > 8*time.Second { - logged = time.Now() - log.Info("Regenerating historical state", "block", i, "target", fromBlock.NumberU64(), "remaining", toBlock.NumberU64()-i, "elapsed", time.Since(start)) - } - // Retrieve the next block to regenerate and process it - block := eth.blockchain.GetBlockByNumber(i) - if block == nil { - return nil, nil, fmt.Errorf("block #%d not found", i) - } - _, _, _, err := eth.blockchain.Processor().Process(block, statedb, vm.Config{}) - if err != nil { - return nil, nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) - } - // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(block.Number())) - if err != nil { - return nil, nil, err - } - statedb, err := eth.blockchain.StateAt(root) - if err != nil { - return nil, nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) - } - states = append(states, statedb.Copy()) - - // Reference the trie twice, once for us, once for the tracer - database.TrieDB().Reference(root, common.Hash{}) - database.TrieDB().Reference(root, common.Hash{}) - refs = append(refs, root) - - // Dereference all past tries we ourselves are done working with - if parent != (common.Hash{}) { - database.TrieDB().Dereference(parent) - } - parent = root - } - // release is handler to release all states referenced, including - // the one referenced in `stateAtBlock`. - release = func() { - for _, ref := range refs { - database.TrieDB().Dereference(ref) - } + if report { + nodes, imgs := database.TrieDB().Size() + log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } - return states, release, nil + return statedb, nil } // stateAtTransaction returns the execution environment of a certain transaction. -func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { +func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { // Short circuit if it's genesis block. if block.NumberU64() == 0 { - return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") + return nil, vm.BlockContext{}, nil, errors.New("no transaction in genesis") } // Create the parent state database parent := eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) + return nil, vm.BlockContext{}, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } - statedb, release, err := eth.stateAtBlock(parent, reexec) + // Lookup the statedb of parent block from the live database, + // otherwise regenerate it on the flight. + statedb, err := eth.stateAtBlock(parent, reexec, nil, true) if err != nil { - return nil, vm.BlockContext{}, nil, nil, err + return nil, vm.BlockContext{}, nil, err } if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, release, nil + return nil, vm.BlockContext{}, statedb, nil } // Recompute transactions up to the target index. signer := types.MakeSigner(eth.blockchain.Config(), block.Number()) @@ -213,19 +156,17 @@ func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) if idx == txIndex { - return msg, context, statedb, release, nil + return msg, context, statedb, nil } // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, txContext, statedb, eth.blockchain.Config(), vm.Config{}) statedb.Prepare(tx.Hash(), block.Hash(), idx) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - release() - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) } - release() - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) + return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 75ab403471..5a28d6889e 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -67,9 +67,8 @@ type Backend interface { ChainConfig() *params.ChainConfig Engine() consensus.Engine ChainDb() ethdb.Database - StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) - StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) - StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) + StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (*state.StateDB, error) + StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) } // API is the collection of tracing APIs exposed over the private debugging endpoint. @@ -187,6 +186,7 @@ type txTraceResult struct { type blockTraceTask struct { statedb *state.StateDB // Intermediate state prepped for tracing block *types.Block // Block to trace the transactions from + rootref common.Hash // Trie root reference held for this task results []*txTraceResult // Trace results procudes by the task } @@ -233,33 +233,22 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config } sub := notifier.CreateSubscription() - // Shift the border to a block ahead in order to get the states - // before these blocks. - endBlock, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(end.NumberU64()-1), end.ParentHash()) - if err != nil { - return nil, err - } // Prepare all the states for tracing. Note this procedure can take very // long time. Timeout mechanism is necessary. reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec } - states, release, err := api.backend.StatesInRange(ctx, start, endBlock, reexec) - if err != nil { - return nil, err - } - defer release() // Release all the resources in the last step. - blocks := int(end.NumberU64() - start.NumberU64()) threads := runtime.NumCPU() if threads > blocks { threads = blocks } var ( - pend = new(sync.WaitGroup) - tasks = make(chan *blockTraceTask, threads) - results = make(chan *blockTraceTask, threads) + pend = new(sync.WaitGroup) + tasks = make(chan *blockTraceTask, threads) + results = make(chan *blockTraceTask, threads) + localctx = context.Background() ) for th := 0; th < threads; th++ { pend.Add(1) @@ -269,7 +258,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config // Fetch and execute the next block trace tasks for task := range tasks { signer := types.MakeSigner(api.backend.ChainConfig(), task.block.Number()) - blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) + blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(localctx), nil) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { msg, _ := tx.AsMessage(signer) @@ -278,7 +267,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config hash: tx.Hash(), block: task.block.Hash(), } - res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) + res, err := api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) if err != nil { task.results[i] = &txTraceResult{Error: err.Error()} log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) @@ -302,10 +291,12 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config go func() { var ( - logged time.Time - number uint64 - traced uint64 - failed error + logged time.Time + number uint64 + traced uint64 + failed error + parent common.Hash + statedb *state.StateDB ) // Ensure everything is properly cleaned up on any exit path defer func() { @@ -323,7 +314,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config close(results) }() // Feed all the blocks both into the tracer, as well as fast process concurrently - for number = start.NumberU64() + 1; number <= end.NumberU64(); number++ { + for number = start.NumberU64(); number < end.NumberU64(); number++ { // Stop tracing if interruption was requested select { case <-notifier.Closed(): @@ -335,16 +326,39 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config logged = time.Now() log.Info("Tracing chain segment", "start", start.NumberU64(), "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin)) } - // Retrieve the next block to trace - block, err := api.blockByNumber(ctx, rpc.BlockNumber(number)) + // Retrieve the parent state to trace on top + block, err := api.blockByNumber(localctx, rpc.BlockNumber(number)) + if err != nil { + failed = err + break + } + // Prepare the statedb for tracing. Don't use the live database for + // tracing to avoid persisting state junks into the database. + statedb, err = api.backend.StateAtBlock(localctx, block, reexec, statedb, false) + if err != nil { + failed = err + break + } + if statedb.Database().TrieDB() != nil { + // Hold the reference for tracer, will be released at the final stage + statedb.Database().TrieDB().Reference(block.Root(), common.Hash{}) + + // Release the parent state because it's already held by the tracer + if parent != (common.Hash{}) { + statedb.Database().TrieDB().Dereference(parent) + } + } + parent = block.Root() + + next, err := api.blockByNumber(localctx, rpc.BlockNumber(number+1)) if err != nil { failed = err break } // Send the block over to the concurrent tracers (if not in the fast-forward phase) - txs := block.Transactions() + txs := next.Transactions() select { - case tasks <- &blockTraceTask{statedb: states[int(number-start.NumberU64()-1)], block: block, results: make([]*txTraceResult, len(txs))}: + case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: next, rootref: block.Root(), results: make([]*txTraceResult, len(txs))}: case <-notifier.Closed(): return } @@ -367,6 +381,10 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config } done[uint64(result.Block)] = result + // Dereference any parent tries held in memory by this task + if res.statedb.Database().TrieDB() != nil { + res.statedb.Database().TrieDB().Dereference(res.rootref) + } // Stream completed traces to the user, aborting on the first error for result, ok := done[next]; ok; result, ok = done[next] { if len(result.Traces) > 0 || next == end.NumberU64() { @@ -470,12 +488,10 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec) + statedb, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true) if err != nil { return nil, err } - defer release() - // Execute all the transaction contained within the block concurrently var ( signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) @@ -561,12 +577,10 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec) + statedb, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true) if err != nil { return nil, err } - defer release() - // Retrieve the tracing configurations, or use default values var ( logConfig vm.LogConfig @@ -690,12 +704,10 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * if err != nil { return nil, err } - msg, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index), reexec) + msg, vmctx, statedb, err := api.backend.StateAtTransaction(ctx, block, int(index), reexec) if err != nil { return nil, err } - defer release() - txctx := &txTraceContext{ index: int(index), hash: hash, @@ -727,12 +739,10 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHa if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, release, err := api.backend.StateAtBlock(ctx, block, reexec) + statedb, err := api.backend.StateAtBlock(ctx, block, reexec, nil, true) if err != nil { return nil, err } - defer release() - // Execute the trace msg := args.ToMessage(api.backend.RPCGasCap()) vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) @@ -767,7 +777,9 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *txTrac deadlineCtx, cancel := context.WithTimeout(ctx, timeout) go func() { <-deadlineCtx.Done() - tracer.(*Tracer).Stop(errors.New("execution timeout")) + if deadlineCtx.Err() == context.DeadlineExceeded { + tracer.(*Tracer).Stop(errors.New("execution timeout")) + } }() defer cancel() diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 688b983bab..7ca90a6608 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -137,25 +137,25 @@ func (b *testBackend) ChainDb() ethdb.Database { return b.chaindb } -func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { +func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (*state.StateDB, error) { statedb, err := b.chain.StateAt(block.Root()) if err != nil { - return nil, nil, errStateNotFound + return nil, errStateNotFound } - return statedb, func() {}, nil + return statedb, nil } -func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { +func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { parent := b.chain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, vm.BlockContext{}, nil, nil, errBlockNotFound + return nil, vm.BlockContext{}, nil, errBlockNotFound } statedb, err := b.chain.StateAt(parent.Root()) if err != nil { - return nil, vm.BlockContext{}, nil, nil, errStateNotFound + return nil, vm.BlockContext{}, nil, errStateNotFound } if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, func() {}, nil + return nil, vm.BlockContext{}, statedb, nil } // Recompute transactions up to the target index. signer := types.MakeSigner(b.chainConfig, block.Number()) @@ -164,31 +164,15 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), b.chain, nil) if idx == txIndex { - return msg, context, statedb, func() {}, nil + return msg, context, statedb, nil } vmenv := vm.NewEVM(context, txContext, statedb, b.chainConfig, vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) } - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) -} - -func (b *testBackend) StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { - var result []*state.StateDB - for number := fromBlock.NumberU64(); number <= toBlock.NumberU64(); number += 1 { - block := b.chain.GetBlockByNumber(number) - if block == nil { - return nil, nil, errBlockNotFound - } - statedb, err := b.chain.StateAt(block.Root()) - if err != nil { - return nil, nil, errStateNotFound - } - result = append(result, statedb) - } - return result, func() {}, nil + return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } func TestTraceCall(t *testing.T) { diff --git a/les/api_backend.go b/les/api_backend.go index f5d2354b60..fc7821fb06 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -298,14 +298,10 @@ func (b *LesApiBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *LesApiBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { +func (b *LesApiBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (*state.StateDB, error) { return b.eth.stateAtBlock(ctx, block, reexec) } -func (b *LesApiBackend) StatesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { - return b.eth.statesInRange(ctx, fromBlock, toBlock, reexec) -} - -func (b *LesApiBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { +func (b *LesApiBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { return b.eth.stateAtTransaction(ctx, block, txIndex, reexec) } diff --git a/les/state_accessor.go b/les/state_accessor.go index 2f49bb921e..af5df36508 100644 --- a/les/state_accessor.go +++ b/les/state_accessor.go @@ -29,41 +29,27 @@ import ( ) // stateAtBlock retrieves the state database associated with a certain block. -func (leth *LightEthereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, func(), error) { - return light.NewState(ctx, block.Header(), leth.odr), func() {}, nil -} - -// statesInRange retrieves a batch of state databases associated with the specific -// block ranges. -func (leth *LightEthereum) statesInRange(ctx context.Context, fromBlock *types.Block, toBlock *types.Block, reexec uint64) ([]*state.StateDB, func(), error) { - var states []*state.StateDB - for number := fromBlock.NumberU64(); number <= toBlock.NumberU64(); number++ { - header, err := leth.blockchain.GetHeaderByNumberOdr(ctx, number) - if err != nil { - return nil, nil, err - } - states = append(states, light.NewState(ctx, header, leth.odr)) - } - return states, nil, nil +func (leth *LightEthereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, error) { + return light.NewState(ctx, block.Header(), leth.odr), nil } // stateAtTransaction returns the execution environment of a certain transaction. -func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, func(), error) { +func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { // Short circuit if it's genesis block. if block.NumberU64() == 0 { - return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") + return nil, vm.BlockContext{}, nil, errors.New("no transaction in genesis") } // Create the parent state database parent, err := leth.blockchain.GetBlock(ctx, block.ParentHash(), block.NumberU64()-1) if err != nil { - return nil, vm.BlockContext{}, nil, nil, err + return nil, vm.BlockContext{}, nil, err } - statedb, _, err := leth.stateAtBlock(ctx, parent, reexec) + statedb, err := leth.stateAtBlock(ctx, parent, reexec) if err != nil { - return nil, vm.BlockContext{}, nil, nil, err + return nil, vm.BlockContext{}, nil, err } if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, func() {}, nil + return nil, vm.BlockContext{}, statedb, nil } // Recompute transactions up to the target index. signer := types.MakeSigner(leth.blockchain.Config(), block.Number()) @@ -74,16 +60,16 @@ func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types. context := core.NewEVMBlockContext(block.Header(), leth.blockchain, nil) statedb.Prepare(tx.Hash(), block.Hash(), idx) if idx == txIndex { - return msg, context, statedb, func() {}, nil + return msg, context, statedb, nil } // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, txContext, statedb, leth.blockchain.Config(), vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) } - return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) + return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } From 9d10856e84e884936c2390360e60e9ebca6d7a34 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 7 Apr 2021 16:54:31 +0200 Subject: [PATCH 259/709] core, eth, internal/ethapi: create access list RPC API (#22550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * core/vm: implement AccessListTracer * eth: implement debug.createAccessList * core/vm: fixed nil panics in accessListTracer * eth: better error messages for createAccessList * eth: some fixes on CreateAccessList * eth: allow for provided accesslists * eth: pass accesslist by value * eth: remove created acocunt from accesslist * core/vm: simplify access list tracer * core/vm: unexport accessListTracer * eth: return best guess if al iteration times out * eth: return best guess if al iteration times out * core: docstring, unexport methods * eth: typo * internal/ethapi: move createAccessList to eth package * internal/ethapi: remove reexec from createAccessList * internal/ethapi: break if al is equal to last run, not if gas is equal * internal/web3ext: fixed arguments * core/types: fixed equality check for accesslist * core/types: no hardcoded vals * core, internal: simplify access list generation, make it precise * core/vm: fix typo Co-authored-by: Martin Holst Swende Co-authored-by: Péter Szilágyi --- core/state_transition.go | 5 +- core/vm/access_list_tracer.go | 177 ++++++++++++++++++++++++++++++++++ core/vm/contracts.go | 14 +++ core/vm/evm.go | 15 --- core/vm/runtime/runtime.go | 16 ++- eth/api_backend.go | 8 +- internal/ethapi/api.go | 104 +++++++++++++++++++- internal/ethapi/backend.go | 2 +- internal/web3ext/web3ext.go | 6 ++ les/api_backend.go | 7 +- 10 files changed, 318 insertions(+), 36 deletions(-) create mode 100644 core/vm/access_list_tracer.go diff --git a/core/state_transition.go b/core/state_transition.go index d511e40bd6..cdffc100a1 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -259,10 +259,9 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { } // Set up the initial access list. - if st.evm.ChainConfig().IsBerlin(st.evm.Context.BlockNumber) { - st.state.PrepareAccessList(msg.From(), msg.To(), st.evm.ActivePrecompiles(), msg.AccessList()) + if rules := st.evm.ChainConfig().Rules(st.evm.Context.BlockNumber); rules.IsBerlin { + st.state.PrepareAccessList(msg.From(), msg.To(), vm.ActivePrecompiles(rules), msg.AccessList()) } - var ( ret []byte vmerr error // vm errors do not effect consensus and are therefore not assigned to err diff --git a/core/vm/access_list_tracer.go b/core/vm/access_list_tracer.go new file mode 100644 index 0000000000..b5bc961c84 --- /dev/null +++ b/core/vm/access_list_tracer.go @@ -0,0 +1,177 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package vm + +import ( + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// accessList is an accumulator for the set of accounts and storage slots an EVM +// contract execution touches. +type accessList map[common.Address]accessListSlots + +// accessListSlots is an accumulator for the set of storage slots within a single +// contract that an EVM contract execution touches. +type accessListSlots map[common.Hash]struct{} + +// newAccessList creates a new accessList. +func newAccessList() accessList { + return make(map[common.Address]accessListSlots) +} + +// addAddress adds an address to the accesslist. +func (al accessList) addAddress(address common.Address) { + // Set address if not previously present + if _, present := al[address]; !present { + al[address] = make(map[common.Hash]struct{}) + } +} + +// addSlot adds a storage slot to the accesslist. +func (al accessList) addSlot(address common.Address, slot common.Hash) { + // Set address if not previously present + al.addAddress(address) + + // Set the slot on the surely existent storage set + al[address][slot] = struct{}{} +} + +// equal checks if the content of the current access list is the same as the +// content of the other one. +func (al accessList) equal(other accessList) bool { + // Cross reference the accounts first + if len(al) != len(other) { + return false + } + for addr := range al { + if _, ok := other[addr]; !ok { + return false + } + } + for addr := range other { + if _, ok := al[addr]; !ok { + return false + } + } + // Accounts match, cross reference the storage slots too + for addr, slots := range al { + otherslots := other[addr] + + if len(slots) != len(otherslots) { + return false + } + for hash := range slots { + if _, ok := otherslots[hash]; !ok { + return false + } + } + for hash := range otherslots { + if _, ok := slots[hash]; !ok { + return false + } + } + } + return true +} + +// accesslist converts the accesslist to a types.AccessList. +func (al accessList) accessList() types.AccessList { + acl := make(types.AccessList, 0, len(al)) + for addr, slots := range al { + tuple := types.AccessTuple{Address: addr} + for slot := range slots { + tuple.StorageKeys = append(tuple.StorageKeys, slot) + } + acl = append(acl, tuple) + } + return acl +} + +// AccessListTracer is a tracer that accumulates touched accounts and storage +// slots into an internal set. +type AccessListTracer struct { + excl map[common.Address]struct{} // Set of account to exclude from the list + list accessList // Set of accounts and storage slots touched +} + +// NewAccessListTracer creates a new tracer that can generate AccessLists. +// An optional AccessList can be specified to occupy slots and addresses in +// the resulting accesslist. +func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompiles []common.Address) *AccessListTracer { + excl := map[common.Address]struct{}{ + from: {}, to: {}, + } + for _, addr := range precompiles { + excl[addr] = struct{}{} + } + list := newAccessList() + for _, al := range acl { + if _, ok := excl[al.Address]; !ok { + list.addAddress(al.Address) + } + for _, slot := range al.StorageKeys { + list.addSlot(al.Address, slot) + } + } + return &AccessListTracer{ + excl: excl, + list: list, + } +} + +func (a *AccessListTracer) CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +} + +// CaptureState captures all opcodes that touch storage or addresses and adds them to the accesslist. +func (a *AccessListTracer) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) { + stack := scope.Stack + if (op == SLOAD || op == SSTORE) && stack.len() >= 1 { + slot := common.Hash(stack.data[stack.len()-1].Bytes32()) + a.list.addSlot(scope.Contract.Address(), slot) + } + if (op == EXTCODECOPY || op == EXTCODEHASH || op == EXTCODESIZE || op == BALANCE || op == SELFDESTRUCT) && stack.len() >= 1 { + addr := common.Address(stack.data[stack.len()-1].Bytes20()) + if _, ok := a.excl[addr]; !ok { + a.list.addAddress(addr) + } + } + if (op == DELEGATECALL || op == CALL || op == STATICCALL || op == CALLCODE) && stack.len() >= 5 { + addr := common.Address(stack.data[stack.len()-2].Bytes20()) + if _, ok := a.excl[addr]; !ok { + a.list.addAddress(addr) + } + } +} + +func (*AccessListTracer) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error) { +} + +func (*AccessListTracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {} + +// AccessList returns the current accesslist maintained by the tracer. +func (a *AccessListTracer) AccessList() types.AccessList { + return a.list.accessList() +} + +// Equal returns if the content of two access list traces are equal. +func (a *AccessListTracer) Equal(other *AccessListTracer) bool { + return a.list.equal(other.list) +} diff --git a/core/vm/contracts.go b/core/vm/contracts.go index a3ceece0e9..9210f5486c 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -128,6 +128,20 @@ func init() { } } +// ActivePrecompiles returns the precompiles enabled with the current configuration. +func ActivePrecompiles(rules params.Rules) []common.Address { + switch { + case rules.IsBerlin: + return PrecompiledAddressesBerlin + case rules.IsIstanbul: + return PrecompiledAddressesIstanbul + case rules.IsByzantium: + return PrecompiledAddressesByzantium + default: + return PrecompiledAddressesHomestead + } +} + // RunPrecompiledContract runs and evaluates the output of a precompiled contract. // It returns // - the returned bytes, diff --git a/core/vm/evm.go b/core/vm/evm.go index 6fac50f721..3f16f33b2d 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -42,21 +42,6 @@ type ( GetHashFunc func(uint64) common.Hash ) -// ActivePrecompiles returns the addresses of the precompiles enabled with the current -// configuration -func (evm *EVM) ActivePrecompiles() []common.Address { - switch { - case evm.chainRules.IsBerlin: - return PrecompiledAddressesBerlin - case evm.chainRules.IsIstanbul: - return PrecompiledAddressesIstanbul - case evm.chainRules.IsByzantium: - return PrecompiledAddressesByzantium - default: - return PrecompiledAddressesHomestead - } -} - func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) { var precompiles map[common.Address]PrecompiledContract switch { diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 9cb69e1c76..72601441d5 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -114,8 +114,8 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { vmenv = NewEnv(cfg) sender = vm.AccountRef(cfg.Origin) ) - if cfg.ChainConfig.IsBerlin(vmenv.Context.BlockNumber) { - cfg.State.PrepareAccessList(cfg.Origin, &address, vmenv.ActivePrecompiles(), nil) + if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber); rules.IsBerlin { + cfg.State.PrepareAccessList(cfg.Origin, &address, vm.ActivePrecompiles(rules), nil) } cfg.State.CreateAccount(address) // set the receiver's (the executing contract) code for execution. @@ -146,10 +146,9 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { vmenv = NewEnv(cfg) sender = vm.AccountRef(cfg.Origin) ) - if cfg.ChainConfig.IsBerlin(vmenv.Context.BlockNumber) { - cfg.State.PrepareAccessList(cfg.Origin, nil, vmenv.ActivePrecompiles(), nil) + if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber); rules.IsBerlin { + cfg.State.PrepareAccessList(cfg.Origin, nil, vm.ActivePrecompiles(rules), nil) } - // Call the code with the given configuration. code, address, leftOverGas, err := vmenv.Create( sender, @@ -172,10 +171,10 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er sender := cfg.State.GetOrNewStateObject(cfg.Origin) statedb := cfg.State - if cfg.ChainConfig.IsBerlin(vmenv.Context.BlockNumber) { - statedb.PrepareAccessList(cfg.Origin, &address, vmenv.ActivePrecompiles(), nil) - } + if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber); rules.IsBerlin { + statedb.PrepareAccessList(cfg.Origin, &address, vm.ActivePrecompiles(rules), nil) + } // Call the code with the given configuration. ret, leftOverGas, err := vmenv.Call( sender, @@ -184,6 +183,5 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er cfg.GasLimit, cfg.Value, ) - return ret, leftOverGas, err } diff --git a/eth/api_backend.go b/eth/api_backend.go index cc780775cc..7ac1f82a86 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -192,12 +192,14 @@ func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(hash) } -func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { vmError := func() error { return nil } - + if vmConfig == nil { + vmConfig = b.eth.blockchain.GetVMConfig() + } txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, b.eth.BlockChain(), nil) - return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *b.eth.blockchain.GetVMConfig()), vmError, nil + return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *vmConfig), vmError, nil } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 861d427851..fe5c3388b5 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -865,7 +865,7 @@ func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.Blo // Get a new instance of the EVM. msg := args.ToMessage(globalGasCap) - evm, vmError, err := b.GetEVM(ctx, msg, state, header) + evm, vmError, err := b.GetEVM(ctx, msg, state, header, nil) if err != nil { return nil, err } @@ -1303,6 +1303,106 @@ func newRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransa return nil } +// accessListResult returns an optional accesslist +// Its the result of the `debug_createAccessList` RPC call. +// It contains an error if the transaction itself failed. +type accessListResult struct { + Accesslist *types.AccessList `json:"accessList"` + Error string `json:"error,omitempty"` + GasUsed hexutil.Uint64 `json:"gasUsed"` +} + +// CreateAccessList creates a EIP-2930 type AccessList for the given transaction. +// Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. +func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args SendTxArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { + bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) + if blockNrOrHash != nil { + bNrOrHash = *blockNrOrHash + } + acl, gasUsed, vmerr, err := AccessList(ctx, s.b, bNrOrHash, args) + if err != nil { + return nil, err + } + result := &accessListResult{Accesslist: &acl, GasUsed: hexutil.Uint64(gasUsed)} + if vmerr != nil { + result.Error = vmerr.Error() + } + return result, nil +} + +// AccessList creates an access list for the given transaction. +// If the accesslist creation fails an error is returned. +// If the transaction itself fails, an vmErr is returned. +func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args SendTxArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { + // Retrieve the execution context + db, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if db == nil || err != nil { + return nil, 0, nil, err + } + // If the gas amount is not set, extract this as it will depend on access + // lists and we'll need to reestimate every time + nogas := args.Gas == nil + + // Ensure any missing fields are filled, extract the recipient and input data + if err := args.setDefaults(ctx, b); err != nil { + return nil, 0, nil, err + } + var to common.Address + if args.To != nil { + to = *args.To + } else { + to = crypto.CreateAddress(args.From, uint64(*args.Nonce)) + } + var input []byte + if args.Input != nil { + input = *args.Input + } else if args.Data != nil { + input = *args.Data + } + // Retrieve the precompiles since they don't need to be added to the access list + precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number)) + + // Create an initial tracer + prevTracer := vm.NewAccessListTracer(nil, args.From, to, precompiles) + if args.AccessList != nil { + prevTracer = vm.NewAccessListTracer(*args.AccessList, args.From, to, precompiles) + } + for { + // Retrieve the current access list to expand + accessList := prevTracer.AccessList() + log.Trace("Creating access list", "input", accessList) + + // If no gas amount was specified, each unique access list needs it's own + // gas calculation. This is quite expensive, but we need to be accurate + // and it's convered by the sender only anyway. + if nogas { + args.Gas = nil + if err := args.setDefaults(ctx, b); err != nil { + return nil, 0, nil, err // shouldn't happen, just in case + } + } + // Copy the original db so we don't modify it + statedb := db.Copy() + msg := types.NewMessage(args.From, args.To, uint64(*args.Nonce), args.Value.ToInt(), uint64(*args.Gas), args.GasPrice.ToInt(), input, accessList, false) + + // Apply the transaction with the access list tracer + tracer := vm.NewAccessListTracer(accessList, args.From, to, precompiles) + config := vm.Config{Tracer: tracer, Debug: true} + vmenv, _, err := b.GetEVM(ctx, msg, statedb, header, &config) + if err != nil { + return nil, 0, nil, err + } + res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) + if err != nil { + return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.toTransaction().Hash(), err) + } + if tracer.Equal(prevTracer) { + return accessList, res.UsedGas, res.Err, nil + } + prevTracer = tracer + } +} + // PublicTransactionPoolAPI exposes methods for the RPC interface type PublicTransactionPoolAPI struct { b Backend @@ -1539,7 +1639,6 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { return errors.New(`contract creation without any data provided`) } } - // Estimate the gas usage if necessary. if args.Gas == nil { // For backwards-compatibility reason, we try both input and data @@ -1580,7 +1679,6 @@ func (args *SendTxArgs) toTransaction() *types.Transaction { } else if args.Data != nil { input = *args.Data } - var data types.TxData if args.AccessList == nil { data = &types.LegacyTx{ diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index ebb088fef5..07e76583f3 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -63,7 +63,7 @@ type Backend interface { StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int - GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index e1f20ad72a..1934412c90 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -575,6 +575,12 @@ web3._extend({ params: 3, inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter] }), + new web3._extend.Method({ + name: 'createAccessList', + call: 'eth_createAccessList', + params: 2, + inputFormatter: [null, web3._extend.formatters.inputBlockNumberFormatter], + }), ], properties: [ new web3._extend.Property({ diff --git a/les/api_backend.go b/les/api_backend.go index fc7821fb06..60c64a8bdf 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -171,10 +171,13 @@ func (b *LesApiBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { +func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + if vmConfig == nil { + vmConfig = new(vm.Config) + } txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, b.eth.blockchain, nil) - return vm.NewEVM(context, txContext, state, b.eth.chainConfig, vm.Config{}), state.Error, nil + return vm.NewEVM(context, txContext, state, b.eth.chainConfig, *vmConfig), state.Error, nil } func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { From 2a7c4b62a9cce1b3d9cd3ad68d0c8b2acd25630c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 7 Apr 2021 18:14:24 +0300 Subject: [PATCH 260/709] eth: fix tracing state retrieval if requesting the non-dirty genesis --- eth/state_accessor.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eth/state_accessor.go b/eth/state_accessor.go index d7564a0844..84cfaf4d73 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -61,6 +61,16 @@ func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64, base *state // the internal junks created by tracing will be persisted into the disk. database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // If we didn't check the dirty database, do check the clean one, otherwise + // we would rewind past a persisted block (specific corner case is chain + // tracing from the genesis). + if !checkLive { + statedb, err = state.New(current.Root(), database, nil) + if err == nil { + return statedb, nil + } + } + // Database does not have the state for the given block, try to regenerate for i := uint64(0); i < reexec; i++ { if current.NumberU64() == 0 { return nil, errors.New("genesis state is missing") From e3ff37c47a5aedd363e32c16c83be42e364253e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 8 Apr 2021 13:23:34 +0300 Subject: [PATCH 261/709] params: update CHTs for v1.10.2 --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 8f8b36b57d..143e2e2a36 100644 --- a/params/config.go +++ b/params/config.go @@ -74,10 +74,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 364, - SectionHead: common.HexToHash("0x3fd20ff221f5e962bb66f57a61973bfc2ba959879a6509384a80a45d208b5afc"), - CHTRoot: common.HexToHash("0xe35b3b807f4e9427fb4e2929961c78a9dc10f503a538319031cc7d00946a0591"), - BloomRoot: common.HexToHash("0x340553b378b2db214b898be15c80ac5be7caffc2e6448fd6f7aff23290d89296"), + SectionIndex: 371, + SectionHead: common.HexToHash("0x50fd3cec5376ede90ef9129772022690cd1467f22c18abb7faa11e793c51e9c9"), + CHTRoot: common.HexToHash("0xb57b4b22a77b5930847b1ca9f62daa11eae6578948cb7b18997f2c0fe5757025"), + BloomRoot: common.HexToHash("0xa338f8a868a194fa90327d0f5877f656a9f3640c618d2a01a01f2e76ef9ef954"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -157,10 +157,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 248, - SectionHead: common.HexToHash("0x26874cf023695778cc3175d1bec19894204d8d0b756b587e81e35f300dc5b33c"), - CHTRoot: common.HexToHash("0xc129d1ed6673c5d3e1068e9d97244e72952b7ca08acbd7b3bfa58bc3085c442c"), - BloomRoot: common.HexToHash("0x1dafe79dcd7d348782aa834a4a4397890d9ad90643736791132ed5c16879a037"), + SectionIndex: 254, + SectionHead: common.HexToHash("0x0cba01dd71baa22ac8fa0b105bc908e94f9ecfbc79b4eb97427fe07b5851dd10"), + CHTRoot: common.HexToHash("0x5673d8fc49c9c7d8729068640e4b392d46952a5a38798973bac1cf1d0d27ad7d"), + BloomRoot: common.HexToHash("0x70e01232b66df9a7778ae3291c9217afb9a2d9f799f32d7b912bd37e7bce83a8"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -198,10 +198,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 132, - SectionHead: common.HexToHash("0x29fa240c97b47ecbfef3fea8b3cff035d93154d1d48b25e3333cf2f7067c5324"), - CHTRoot: common.HexToHash("0x85e5c59e5b202284291405dadc40dc36ab6417bd189fb18be24f6dcab6b80511"), - BloomRoot: common.HexToHash("0x0b7afdd200477f46e982e2cabc822ac454424986fa50d899685dfaeede1f882d"), + SectionIndex: 138, + SectionHead: common.HexToHash("0xb7ea0566abd7d0def5b3c9afa3431debb7bb30b65af35f106ca93a59e6c859a7"), + CHTRoot: common.HexToHash("0x378c7ea9081242beb982e2e39567ba12f2ed3e59e5aba3f9db1d595646d7c9f4"), + BloomRoot: common.HexToHash("0x523c169286cfca52e8a6579d8c35dc8bf093412d8a7478163bfa81ae91c2492d"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. From 97d11b0187b4695ccf44e3b71b54155fe405a36f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 8 Apr 2021 13:02:25 +0200 Subject: [PATCH 262/709] params: release go-ethereum v1.10.2 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index d88b4f8b88..3cf4941263 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 04dcc9378dc38d54eeb6a688d81e8b5e13446519 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 8 Apr 2021 13:04:30 +0200 Subject: [PATCH 263/709] params: begin v1.10.3 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 3cf4941263..d350c5ff4a 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From fe1586b094048d7f661be93a04117e92e5ebeaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 8 Apr 2021 18:06:03 +0300 Subject: [PATCH 264/709] eth, les: drop support for eth/64, fix eth/66 tests --- eth/downloader/downloader.go | 5 +- eth/downloader/downloader_test.go | 327 +++++++++++++--------------- eth/downloader/peer.go | 8 +- eth/handler.go | 6 +- eth/handler_eth_test.go | 42 ++-- eth/protocols/eth/handler.go | 51 ++--- eth/protocols/eth/handler_test.go | 115 ++++++++-- eth/protocols/eth/handshake_test.go | 4 +- eth/protocols/eth/protocol.go | 5 +- eth/sync_test.go | 4 +- les/client_handler.go | 3 +- les/server_handler.go | 1 - 12 files changed, 286 insertions(+), 285 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index a5ed3761b1..b8cb48914e 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -459,8 +460,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < 64 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, 64) + if p.version < eth.ETH65 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 2917116144..1140a444c1 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/trie" @@ -515,16 +516,13 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } -func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonSync(t, 64, FullSync) } -func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonSync(t, 64, FastSync) } +func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } +func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } +func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } -func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, 65, FullSync) } -func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, 65, FastSync) } -func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, 65, LightSync) } - -func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, 66, FullSync) } -func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, 66, FastSync) } -func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, 66, LightSync) } +func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } +func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } +func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -545,14 +543,11 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling64Full(t *testing.T) { testThrottling(t, 64, FullSync) } -func TestThrottling64Fast(t *testing.T) { testThrottling(t, 64, FastSync) } - -func TestThrottling65Full(t *testing.T) { testThrottling(t, 65, FullSync) } -func TestThrottling65Fast(t *testing.T) { testThrottling(t, 65, FastSync) } +func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } +func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } -func TestThrottling66Full(t *testing.T) { testThrottling(t, 66, FullSync) } -func TestThrottling66Fast(t *testing.T) { testThrottling(t, 66, FastSync) } +func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } +func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -633,16 +628,13 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync64Full(t *testing.T) { testForkedSync(t, 64, FullSync) } -func TestForkedSync64Fast(t *testing.T) { testForkedSync(t, 64, FastSync) } - -func TestForkedSync65Full(t *testing.T) { testForkedSync(t, 65, FullSync) } -func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, 65, FastSync) } -func TestForkedSync65Light(t *testing.T) { testForkedSync(t, 65, LightSync) } +func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } +func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } +func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } -func TestForkedSync66Full(t *testing.T) { testForkedSync(t, 66, FullSync) } -func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, 66, FastSync) } -func TestForkedSync66Light(t *testing.T) { testForkedSync(t, 66, LightSync) } +func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } +func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } +func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -669,16 +661,13 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync64Full(t *testing.T) { testHeavyForkedSync(t, 64, FullSync) } -func TestHeavyForkedSync64Fast(t *testing.T) { testHeavyForkedSync(t, 64, FastSync) } +func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } +func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } +func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } -func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, 65, FullSync) } -func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, 65, FastSync) } -func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, 65, LightSync) } - -func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, 66, FullSync) } -func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, 66, FastSync) } -func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, 66, LightSync) } +func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } +func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } +func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -707,16 +696,13 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync64Full(t *testing.T) { testBoundedForkedSync(t, 64, FullSync) } -func TestBoundedForkedSync64Fast(t *testing.T) { testBoundedForkedSync(t, 64, FastSync) } - -func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, 65, FullSync) } -func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, 65, FastSync) } -func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, 65, LightSync) } +func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } +func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } +func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } -func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, 66, FullSync) } -func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, 66, FastSync) } -func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, 66, LightSync) } +func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } +func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } +func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -744,16 +730,25 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync64Full(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FullSync) } -func TestBoundedHeavyForkedSync64Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FastSync) } - -func TestBoundedHeavyForkedSync65Full(t *testing.T) { testBoundedHeavyForkedSync(t, 65, FullSync) } -func TestBoundedHeavyForkedSync65Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 65, FastSync) } -func TestBoundedHeavyForkedSync65Light(t *testing.T) { testBoundedHeavyForkedSync(t, 65, LightSync) } +func TestBoundedHeavyForkedSync65Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) +} +func TestBoundedHeavyForkedSync65Fast(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) +} +func TestBoundedHeavyForkedSync65Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) +} -func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, 66, FullSync) } -func TestBoundedHeavyForkedSync66Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 66, FastSync) } -func TestBoundedHeavyForkedSync66Light(t *testing.T) { testBoundedHeavyForkedSync(t, 66, LightSync) } +func TestBoundedHeavyForkedSync66Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) +} +func TestBoundedHeavyForkedSync66Fast(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH66, FastSync) +} +func TestBoundedHeavyForkedSync66Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH66, LightSync) +} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -799,16 +794,13 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) } -func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) } - -func TestCancel65Full(t *testing.T) { testCancel(t, 65, FullSync) } -func TestCancel65Fast(t *testing.T) { testCancel(t, 65, FastSync) } -func TestCancel65Light(t *testing.T) { testCancel(t, 65, LightSync) } +func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } +func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } +func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } -func TestCancel66Full(t *testing.T) { testCancel(t, 66, FullSync) } -func TestCancel66Fast(t *testing.T) { testCancel(t, 66, FastSync) } -func TestCancel66Light(t *testing.T) { testCancel(t, 66, LightSync) } +func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } +func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } +func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -835,16 +827,13 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, 64, FullSync) } -func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) } +func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } +func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } +func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } -func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, 65, FullSync) } -func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, 65, FastSync) } -func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, 65, LightSync) } - -func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, 66, FullSync) } -func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, 66, FastSync) } -func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, 66, LightSync) } +func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } +func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } +func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -868,16 +857,13 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, 64, FullSync) } -func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) } - -func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, 65, FullSync) } -func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, 65, FastSync) } -func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, 65, LightSync) } +func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } +func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } +func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } -func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, 66, FullSync) } -func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, 66, FastSync) } -func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, 66, LightSync) } +func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } +func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } +func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -889,9 +875,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 64", 64, chain) - tester.newPeer("peer 65", 65, chain) - tester.newPeer("peer 66", 66, chain) + tester.newPeer("peer 65", eth.ETH65, chain) + tester.newPeer("peer 66", eth.ETH66, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -900,7 +885,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{64, 65, 66} { + for _, version := range []int{65, 66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -910,16 +895,13 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, FullSync) } -func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) } +func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } +func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } +func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } -func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, 65, FullSync) } -func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, 65, FastSync) } -func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, 65, LightSync) } - -func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, 66, FullSync) } -func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, 66, FastSync) } -func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, 66, LightSync) } +func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } +func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } +func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -967,16 +949,13 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 64, FullSync) } -func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) } - -func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, 65, FullSync) } -func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, 65, FastSync) } -func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, 65, LightSync) } +func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } +func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } +func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } -func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, 66, FullSync) } -func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, 66, FastSync) } -func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, 66, LightSync) } +func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } +func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } +func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1002,16 +981,13 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 64, FullSync) } -func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) } +func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } +func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } +func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } -func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, 65, FullSync) } -func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, 65, FastSync) } -func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, 65, LightSync) } - -func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, 66, FullSync) } -func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, 66, FastSync) } -func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, 66, LightSync) } +func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } +func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } +func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1042,9 +1018,8 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. -func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) } -func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, 65, FastSync) } -func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, 66, FastSync) } +func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } +func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1134,16 +1109,25 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttack(t, 64, FullSync) } -func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } - -func TestHighTDStarvationAttack65Full(t *testing.T) { testHighTDStarvationAttack(t, 65, FullSync) } -func TestHighTDStarvationAttack65Fast(t *testing.T) { testHighTDStarvationAttack(t, 65, FastSync) } -func TestHighTDStarvationAttack65Light(t *testing.T) { testHighTDStarvationAttack(t, 65, LightSync) } +func TestHighTDStarvationAttack65Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, FullSync) +} +func TestHighTDStarvationAttack65Fast(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, FastSync) +} +func TestHighTDStarvationAttack65Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, LightSync) +} -func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, 66, FullSync) } -func TestHighTDStarvationAttack66Fast(t *testing.T) { testHighTDStarvationAttack(t, 66, FastSync) } -func TestHighTDStarvationAttack66Light(t *testing.T) { testHighTDStarvationAttack(t, 66, LightSync) } +func TestHighTDStarvationAttack66Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH66, FullSync) +} +func TestHighTDStarvationAttack66Fast(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH66, FastSync) +} +func TestHighTDStarvationAttack66Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH66, LightSync) +} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1159,9 +1143,8 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } -func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, 65) } -func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, 66) } +func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } +func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { t.Parallel() @@ -1213,16 +1196,13 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) } -func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) } +func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } +func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } +func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } -func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, 65, FullSync) } -func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, 65, FastSync) } -func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, 65, LightSync) } - -func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, 66, FullSync) } -func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, 66, FastSync) } -func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, 66, LightSync) } +func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } +func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } +func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1300,16 +1280,13 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, FullSync) } -func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) } - -func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, 65, FullSync) } -func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, 65, FastSync) } -func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, 65, LightSync) } +func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } +func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } +func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } -func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, 66, FullSync) } -func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, 66, FastSync) } -func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, 66, LightSync) } +func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } +func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } +func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1379,16 +1356,13 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, FullSync) } -func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) } +func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } +func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } +func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } -func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, 65, FullSync) } -func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, 65, FastSync) } -func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, 65, LightSync) } - -func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, 66, FullSync) } -func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, 66, FastSync) } -func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, 66, LightSync) } +func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } +func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } +func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1455,16 +1429,13 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, FullSync) } -func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) } - -func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, 65, FullSync) } -func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, 65, FastSync) } -func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, 65, LightSync) } +func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } +func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } +func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } -func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, 66, FullSync) } -func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, 66, FastSync) } -func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, 66, LightSync) } +func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } +func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } +func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1535,16 +1506,13 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang64Full(t *testing.T) { testDeliverHeadersHang(t, 64, FullSync) } -func TestDeliverHeadersHang64Fast(t *testing.T) { testDeliverHeadersHang(t, 64, FastSync) } +func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } +func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } +func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } -func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, 65, FullSync) } -func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, 65, FastSync) } -func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, 65, LightSync) } - -func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, 66, FullSync) } -func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, 66, FastSync) } -func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, 66, LightSync) } +func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } +func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } +func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() @@ -1699,16 +1667,17 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement64Full(t *testing.T) { testCheckpointEnforcement(t, 64, FullSync) } -func TestCheckpointEnforcement64Fast(t *testing.T) { testCheckpointEnforcement(t, 64, FastSync) } - -func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, 65, FullSync) } -func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, 65, FastSync) } -func TestCheckpointEnforcement65Light(t *testing.T) { testCheckpointEnforcement(t, 65, LightSync) } +func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } +func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } +func TestCheckpointEnforcement65Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH65, LightSync) +} -func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, 66, FullSync) } -func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, 66, FastSync) } -func TestCheckpointEnforcement66Light(t *testing.T) { testCheckpointEnforcement(t, 66, LightSync) } +func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } +func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } +func TestCheckpointEnforcement66Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH66, LightSync) +} func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 7852569d8e..b3b6cc95a0 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -458,7 +458,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.headerThroughput } - return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -472,7 +472,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.blockThroughput } - return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -486,7 +486,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.receiptThroughput } - return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -500,7 +500,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.stateThroughput } - return ps.idlePeers(eth.ETH64, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index 11c8565de1..3f10750abf 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -496,11 +496,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { for peer, hashes := range annos { annoPeers++ annoCount += len(hashes) - if peer.Version() >= eth.ETH65 { - peer.AsyncSendPooledTransactionHashes(hashes) - } else { - peer.AsyncSendTransactions(hashes) - } + peer.AsyncSendPooledTransactionHashes(hashes) } log.Debug("Transaction broadcast", "txs", len(txs), "announce packs", annoPeers, "announced hashes", annoCount, diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 5f5d4e9e82..1d38e3b666 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,8 +80,8 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit64(t *testing.T) { testForkIDSplit(t, 64) } -func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, 65) } +func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } +func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { t.Parallel() @@ -236,8 +236,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) } -func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, 65) } +func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } +func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { t.Parallel() @@ -294,8 +294,8 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) } -func TestSendTransactions65(t *testing.T) { testSendTransactions(t, 65) } +func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } +func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { t.Parallel() @@ -354,19 +354,7 @@ func testSendTransactions(t *testing.T, protocol uint) { seen := make(map[common.Hash]struct{}) for len(seen) < len(insert) { switch protocol { - case 63, 64: - select { - case <-anns: - t.Errorf("tx announce received on pre eth/65") - case txs := <-bcasts: - for _, tx := range txs { - if _, ok := seen[tx.Hash()]; ok { - t.Errorf("duplicate transaction announced: %x", tx.Hash()) - } - seen[tx.Hash()] = struct{}{} - } - } - case 65: + case 65, 66: select { case hashes := <-anns: for _, hash := range hashes { @@ -392,8 +380,8 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation64(t *testing.T) { testTransactionPropagation(t, 64) } -func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, 65) } +func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } +func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { t.Parallel() @@ -530,8 +518,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{1}, "", nil), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{2}, "", nil), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{1}, "", nil), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{2}, "", nil), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -620,8 +608,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{byte(i)}, "", nil), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH64, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{byte(i)}, "", nil), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -672,8 +660,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock64(t *testing.T) { testBroadcastMalformedBlock(t, 64) } -func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, 65) } +func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } +func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { t.Parallel() diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 0dc3de9898..52dcf94011 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,44 +171,27 @@ type Decoder interface { Time() time.Time } -var eth64 = map[uint64]msgHandler{ - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetNodeDataMsg: handleGetNodeData, - NodeDataMsg: handleNodeData, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, -} var eth65 = map[uint64]msgHandler{ - // old 64 messages - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetNodeDataMsg: handleGetNodeData, - NodeDataMsg: handleNodeData, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - // New eth65 messages + GetBlockHeadersMsg: handleGetBlockHeaders, + BlockHeadersMsg: handleBlockHeaders, + GetBlockBodiesMsg: handleGetBlockBodies, + BlockBodiesMsg: handleBlockBodies, + GetNodeDataMsg: handleGetNodeData, + NodeDataMsg: handleNodeData, + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, GetPooledTransactionsMsg: handleGetPooledTransactions, PooledTransactionsMsg: handlePooledTransactions, } var eth66 = map[uint64]msgHandler{ - // eth64 announcement messages (no id) - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - // eth65 announcement messages (no id) + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, // eth66 messages with request-id GetBlockHeadersMsg: handleGetBlockHeaders66, @@ -236,10 +219,8 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth64 - if peer.Version() == ETH65 { - handlers = eth65 - } else if peer.Version() >= ETH66 { + var handlers = eth65 + if peer.Version() >= ETH66 { handlers = eth66 } // Track the emount of time it takes to serve the request and run the handler diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 30beae931b..2dd2446e3d 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,8 +110,8 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders64(t *testing.T) { testGetBlockHeaders(t, 64) } -func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, 65) } +func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } +func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { t.Parallel() @@ -254,18 +254,44 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } } } } @@ -273,8 +299,8 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies64(t *testing.T) { testGetBlockBodies(t, 64) } -func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, 65) } +func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } +func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { t.Parallel() @@ -343,16 +369,29 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockBodiesMsg, hashes) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockBodiesMsg, hashes) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } } } } // Tests that the state trie nodes can be retrieved based on hashes. -func TestGetNodeData64(t *testing.T) { testGetNodeData(t, 64) } -func TestGetNodeData65(t *testing.T) { testGetNodeData(t, 65) } +func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } +func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { t.Parallel() @@ -410,7 +449,14 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - p2p.Send(peer.app, GetNodeDataMsg, hashes) + if protocol <= ETH65 { + p2p.Send(peer.app, GetNodeDataMsg, hashes) + } else { + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) + } msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -419,8 +465,16 @@ func testGetNodeData(t *testing.T, protocol uint) { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } var data [][]byte - if err := msg.Decode(&data); err != nil { - t.Fatalf("failed to decode response node data: %v", err) + if protocol <= ETH65 { + if err := msg.Decode(&data); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + } else { + var res NodeDataPacket66 + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + data = res.NodeDataPacket } // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { @@ -452,8 +506,8 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts64(t *testing.T) { testGetBlockReceipts(t, 64) } -func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, 65) } +func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } +func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { t.Parallel() @@ -503,7 +557,7 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { // Collect the hashes to request, and the response to expect var ( hashes []common.Hash - receipts []types.Receipts + receipts [][]*types.Receipt ) for i := uint64(0); i <= backend.chain.CurrentBlock().NumberU64(); i++ { block := backend.chain.GetBlockByNumber(i) @@ -512,8 +566,21 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - p2p.Send(peer.app, GetReceiptsMsg, hashes) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { - t.Errorf("receipts mismatch: %v", err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetReceiptsMsg, hashes) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { + t.Errorf("receipts mismatch: %v", err) + } + } else { + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) + } } } diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 65f9a00064..3bebda2dcc 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,8 +27,8 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake64(t *testing.T) { testHandshake(t, 64) } -func TestHandshake65(t *testing.T) { testHandshake(t, 65) } +func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } +func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { t.Parallel() diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 7f1832754f..62c018ef8e 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH64 = 64 ETH65 = 65 ETH66 = 66 ) @@ -41,11 +40,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66, ETH65, ETH64} +var ProtocolVersions = []uint{ETH66, ETH65} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17, ETH64: 17} +var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 diff --git a/eth/sync_test.go b/eth/sync_test.go index 9cc806b18a..a0c6f86023 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,8 +28,8 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. -func TestFastSyncDisabling64(t *testing.T) { testFastSyncDisabling(t, 64) } -func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, 65) } +func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } +func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully // imported into the blockchain. diff --git a/les/client_handler.go b/les/client_handler.go index f8e9edc9fe..73149975c3 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" @@ -470,7 +471,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, ethVersion, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { diff --git a/les/server_handler.go b/les/server_handler.go index 5e12136d90..0a683c1b41 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -41,7 +41,6 @@ import ( const ( softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header - ethVersion = 64 // equivalent eth version for the downloader MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request MaxBodyFetch = 32 // Amount of block bodies to be fetched per retrieval request From 6c27d8f996ee8bc0dd7ed3e8ec195dd38e14acab Mon Sep 17 00:00:00 2001 From: Balaji Shetty Pachai <32358081+balajipachai@users.noreply.github.com> Date: Tue, 13 Apr 2021 13:30:48 +0530 Subject: [PATCH 265/709] accounts: documentation fixes (#22645) * replaces `an chance` with `a chance` * replaces `SignHashWithPassphrase` with `SignTextWithPassphrase` as there was no SignHashWithPasspharse function in the file --- accounts/accounts.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accounts/accounts.go b/accounts/accounts.go index 08a1f0f2b1..7178578091 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -113,7 +113,7 @@ type Wallet interface { SignData(account Account, mimeType string, data []byte) ([]byte, error) // SignDataWithPassphrase is identical to SignData, but also takes a password - // NOTE: there's an chance that an erroneous call might mistake the two strings, and + // NOTE: there's a chance that an erroneous call might mistake the two strings, and // supply password in the mimetype field, or vice versa. Thus, an implementation // should never echo the mimetype or return the mimetype in the error-response SignDataWithPassphrase(account Account, passphrase, mimeType string, data []byte) ([]byte, error) @@ -127,7 +127,7 @@ type Wallet interface { // a password to decrypt the account, or a PIN code o verify the transaction), // an AuthNeededError instance will be returned, containing infos for the user // about which fields or actions are needed. The user may retry by providing - // the needed details via SignHashWithPassphrase, or by other means (e.g. unlock + // the needed details via SignTextWithPassphrase, or by other means (e.g. unlock // the account in a keystore). // // This method should return the signature in 'canonical' format, with v 0 or 1 From 271e5b7fc921709bd57ec672912f63a4586717dd Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 13 Apr 2021 15:45:30 +0200 Subject: [PATCH 266/709] cmd/geth: add db-command to inspect freezer index (#22633) This PR makes it easier to inspect the freezer index, which could be useful to investigate things like #22111 --- cmd/geth/dbcmd.go | 58 ++++++++++++++++++++++++++++++++ core/rawdb/freezer.go | 2 +- core/rawdb/freezer_table.go | 19 +++++------ core/rawdb/freezer_table_test.go | 4 +-- core/rawdb/schema.go | 4 +-- 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index db1fb0b801..4c70373e9a 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strconv" "time" @@ -60,6 +61,7 @@ Remove blockchain and state databases`, dbDeleteCmd, dbPutCmd, dbGetSlotsCmd, + dbDumpFreezerIndex, }, } dbInspectCmd = cli.Command{ @@ -177,6 +179,22 @@ WARNING: This is a low-level operation which may cause database corruption!`, }, Description: "This command looks up the specified database key from the database.", } + dbDumpFreezerIndex = cli.Command{ + Action: utils.MigrateFlags(freezerInspect), + Name: "freezer-index", + Usage: "Dump out the index of a given freezer type", + ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.YoloV3Flag, + }, + Description: "This command displays information about the freezer index.", + } ) func removeDB(ctx *cli.Context) error { @@ -449,3 +467,43 @@ func dbDumpTrie(ctx *cli.Context) error { } return it.Err } + +func freezerInspect(ctx *cli.Context) error { + var ( + start, end int64 + disableSnappy bool + err error + ) + if ctx.NArg() < 3 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + kind := ctx.Args().Get(0) + if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok { + var options []string + for opt := range rawdb.FreezerNoSnappy { + options = append(options, opt) + } + sort.Strings(options) + return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options) + } else { + disableSnappy = noSnap + } + if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil { + log.Info("Could read start-param", "error", err) + return err + } + if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { + log.Info("Could read count param", "error", err) + return err + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") + log.Info("Opening freezer", "location", path, "name", kind) + if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy); err != nil { + return err + } else { + f.DumpIndex(start, end) + } + return nil +} diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 4e5ae4284e..94b99a64eb 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -118,7 +118,7 @@ func newFreezer(datadir string, namespace string, readonly bool) (*freezer, erro trigger: make(chan chan struct{}), quit: make(chan struct{}), } - for name, disableSnappy := range freezerNoSnappy { + for name, disableSnappy := range FreezerNoSnappy { table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, disableSnappy) if err != nil { for _, table := range freezer.tables { diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index cd273222b1..b614c10d37 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -636,25 +636,24 @@ func (t *freezerTable) Sync() error { return t.head.Sync() } -// printIndex is a debug print utility function for testing -func (t *freezerTable) printIndex() { +// DumpIndex is a debug print utility function, mainly for testing. It can also +// be used to analyse a live freezer table index. +func (t *freezerTable) DumpIndex(start, stop int64) { buf := make([]byte, indexEntrySize) - fmt.Printf("|-----------------|\n") - fmt.Printf("| fileno | offset |\n") - fmt.Printf("|--------+--------|\n") + fmt.Printf("| number | fileno | offset |\n") + fmt.Printf("|--------|--------|--------|\n") - for i := uint64(0); ; i++ { + for i := uint64(start); ; i++ { if _, err := t.index.ReadAt(buf, int64(i*indexEntrySize)); err != nil { break } var entry indexEntry entry.unmarshalBinary(buf) - fmt.Printf("| %03d | %03d | \n", entry.filenum, entry.offset) - if i > 100 { - fmt.Printf(" ... \n") + fmt.Printf("| %03d | %03d | %03d | \n", i, entry.filenum, entry.offset) + if stop > 0 && i >= uint64(stop) { break } } - fmt.Printf("|-----------------|\n") + fmt.Printf("|--------------------------|\n") } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index b22c58e138..b8d3170c62 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -525,7 +525,7 @@ func TestOffset(t *testing.T) { f.Append(4, getChunk(20, 0xbb)) f.Append(5, getChunk(20, 0xaa)) - f.printIndex() + f.DumpIndex(0, 100) f.Close() } // Now crop it. @@ -572,7 +572,7 @@ func TestOffset(t *testing.T) { if err != nil { t.Fatal(err) } - f.printIndex() + f.DumpIndex(0, 100) // It should allow writing item 6 f.Append(numDeleted+2, getChunk(20, 0x99)) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 0b411057f8..7a97389106 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -114,9 +114,9 @@ const ( freezerDifficultyTable = "diffs" ) -// freezerNoSnappy configures whether compression is disabled for the ancient-tables. +// FreezerNoSnappy configures whether compression is disabled for the ancient-tables. // Hashes and difficulties don't compress well. -var freezerNoSnappy = map[string]bool{ +var FreezerNoSnappy = map[string]bool{ freezerHeaderTable: false, freezerHashTable: true, freezerBodiesTable: false, From 72e37942f3f777129b6cfa2d28b1ba7bbdf6437a Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Wed, 14 Apr 2021 03:21:46 +0530 Subject: [PATCH 267/709] cmd/faucet: support testnet flags in the faucet (#22545) Co-authored-by: Felix Lange Co-authored-by: Martin Holst Swende --- cmd/faucet/faucet.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index e839f1c886..bb5375384f 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -85,6 +85,9 @@ var ( twitterTokenFlag = flag.String("twitter.token", "", "Bearer token to authenticate with the v2 Twitter API") twitterTokenV1Flag = flag.String("twitter.token.v1", "", "Bearer token to authenticate with the v1.1 Twitter API") + + goerliFlag = flag.Bool("goerli", false, "Initializes the faucet with Görli network config") + rinkebyFlag = flag.Bool("rinkeby", false, "Initializes the faucet with Rinkeby network config") ) var ( @@ -144,13 +147,9 @@ func main() { log.Crit("Failed to render the faucet template", "err", err) } // Load and parse the genesis block requested by the user - blob, err := ioutil.ReadFile(*genesisFlag) + genesis, err := getGenesis(genesisFlag, *goerliFlag, *rinkebyFlag) if err != nil { - log.Crit("Failed to read genesis block contents", "genesis", *genesisFlag, "err", err) - } - genesis := new(core.Genesis) - if err = json.Unmarshal(blob, genesis); err != nil { - log.Crit("Failed to parse genesis block json", "err", err) + log.Crit("Failed to parse genesis config", "err", err) } // Convert the bootnodes to internal enode representations var enodes []*enode.Node @@ -162,7 +161,8 @@ func main() { } } // Load up the account key and decrypt its password - if blob, err = ioutil.ReadFile(*accPassFlag); err != nil { + blob, err := ioutil.ReadFile(*accPassFlag) + if err != nil { log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err) } pass := strings.TrimSuffix(string(blob), "\n") @@ -884,3 +884,19 @@ func authNoAuth(url string) (string, string, common.Address, error) { } return address.Hex() + "@noauth", "", address, nil } + +// getGenesis returns a genesis based on input args +func getGenesis(genesisFlag *string, goerliFlag bool, rinkebyFlag bool) (*core.Genesis, error) { + switch { + case genesisFlag != nil: + var genesis core.Genesis + err := common.LoadJSON(*genesisFlag, &genesis) + return &genesis, err + case goerliFlag: + return core.DefaultGoerliGenesisBlock(), nil + case rinkebyFlag: + return core.DefaultRinkebyGenesisBlock(), nil + default: + return nil, fmt.Errorf("no genesis flag provided") + } +} From a50251e6cbfecfaf26040d42c70d2812bc422a4a Mon Sep 17 00:00:00 2001 From: xD AKA Rapper King Of cn background diablo & revelations <33193253+r1cs@users.noreply.github.com> Date: Wed, 14 Apr 2021 18:44:32 +0800 Subject: [PATCH 268/709] eth/fetcher: avoid spurious timer events at startup (#22652) Co-authored-by: Felix Lange --- eth/fetcher/block_fetcher.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 5ea8a128d9..3177a877ed 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -331,8 +331,12 @@ func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transac // events. func (f *BlockFetcher) loop() { // Iterate the block fetching until a quit is requested - fetchTimer := time.NewTimer(0) - completeTimer := time.NewTimer(0) + var ( + fetchTimer = time.NewTimer(0) + completeTimer = time.NewTimer(0) + ) + <-fetchTimer.C // clear out the channel + <-completeTimer.C defer fetchTimer.Stop() defer completeTimer.Stop() From 7088f1e81456bdc0bbe801e266ea6ca6c15c1835 Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 15 Apr 2021 04:23:11 +0800 Subject: [PATCH 269/709] core, eth: faster snapshot generation (#22504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/protocols: persist received state segments * core: initial implementation * core/state/snapshot: add tests * core, eth: updates * eth/protocols/snapshot: count flat state size * core/state: add metrics * core/state/snapshot: skip unnecessary deletion * core/state/snapshot: rename * core/state/snapshot: use the global batch * core/state/snapshot: add logs and fix wiping * core/state/snapshot: fix * core/state/snapshot: save generation progress even if the batch is empty * core/state/snapshot: fixes * core/state/snapshot: fix initial account range length * core/state/snapshot: fix initial account range * eth/protocols/snap: store flat states during the healing * eth/protocols/snap: print logs * core/state/snapshot: refactor (#4) * core/state/snapshot: refactor * core/state/snapshot: tiny fix and polish Co-authored-by: rjl493456442 * core, eth: fixes * core, eth: fix healing writer * core, trie, eth: fix paths * eth/protocols/snap: fix encoding * eth, core: add debug log * core/state/generate: release iterator asap (#5) core/state/snapshot: less copy core/state/snapshot: revert split loop core/state/snapshot: handle storage becoming empty, improve test robustness core/state: test modified codehash core/state/snapshot: polish * core/state/snapshot: optimize stats counter * core, eth: add metric * core/state/snapshot: update comments * core/state/snapshot: improve tests * core/state/snapshot: replace secure trie with standard trie * core/state/snapshot: wrap return as the struct * core/state/snapshot: skip wiping correct states * core/state/snapshot: updates * core/state/snapshot: fixes * core/state/snapshot: fix panic due to reference flaw in closure * core/state/snapshot: fix errors in state generation logic + fix log output * core/state/snapshot: remove an error case * core/state/snapshot: fix condition-check for exhausted snap state * core/state/snapshot: use stackTrie for small tries * core/state/snapshot: don't resolve small storage tries in vain * core/state/snapshot: properly clean up storage of deleted accounts * core/state/snapshot: avoid RLP-encoding in some cases + minor nitpicks * core/state/snapshot: fix error (+testcase) * core/state/snapshot: clean up tests a bit * core/state/snapshot: work in progress on better tests * core/state/snapshot: polish code * core/state/snapshot: fix trie iteration abortion trigger * core/state/snapshot: fixes flaws * core/state/snapshot: remove panic * core/state/snapshot: fix abort * core/state/snapshot: more tests (plus failing testcase) * core/state/snapshot: more testcases + fix for failing test * core/state/snapshot: testcase for malformed data * core/state/snapshot: some test nitpicks * core/state/snapshot: improvements to logging * core/state/snapshot: testcase to demo error in abortion * core/state/snapshot: fix abortion * cmd/geth: make verify-state report the root * trie: fix failing test * core/state/snapshot: add timer metrics * core/state/snapshot: fix metrics * core/state/snapshot: udpate tests * eth/protocols/snap: write snapshot account even if code or state is needed * core/state/snapshot: fix diskmore check * core/state/snapshot: review fixes * core/state/snapshot: improve error message * cmd/geth: rename 'error' to 'err' in logs * core/state/snapshot: fix some review concerns * core/state/snapshot, eth/protocols/snap: clear snapshot marker when starting/resuming snap sync * core: add error log * core/state/snapshot: use proper timers for metrics collection * core/state/snapshot: address some review concerns * eth/protocols/snap: improved log message * eth/protocols/snap: fix heal logs to condense infos * core/state/snapshot: wait for generator termination before restarting * core/state/snapshot: revert timers to counters to track total time Co-authored-by: Martin Holst Swende Co-authored-by: Péter Szilágyi --- cmd/geth/snapshot.go | 38 +- core/state/snapshot/conversion.go | 2 +- core/state/snapshot/generate.go | 679 +++++++++++++++++++++------ core/state/snapshot/generate_test.go | 649 ++++++++++++++++++++++++- core/state/snapshot/journal.go | 15 +- core/state/snapshot/snapshot.go | 10 +- core/state/snapshot/wipe.go | 32 +- core/state/statedb.go | 2 +- core/state/sync.go | 24 +- core/state/sync_test.go | 12 +- eth/downloader/statesync.go | 2 +- eth/protocols/snap/sync.go | 81 +++- trie/committer.go | 4 +- trie/sync.go | 9 +- trie/trie.go | 17 +- trie/trie_test.go | 2 +- 16 files changed, 1369 insertions(+), 209 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index e8f6a35438..1af458af20 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -155,7 +155,7 @@ func pruneState(ctx *cli.Context) error { chaindb := utils.MakeChainDatabase(ctx, stack, false) pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) if err != nil { - log.Error("Failed to open snapshot tree", "error", err) + log.Error("Failed to open snapshot tree", "err", err) return err } if ctx.NArg() > 1 { @@ -166,12 +166,12 @@ func pruneState(ctx *cli.Context) error { if ctx.NArg() == 1 { targetRoot, err = parseRoot(ctx.Args()[0]) if err != nil { - log.Error("Failed to resolve state root", "error", err) + log.Error("Failed to resolve state root", "err", err) return err } } if err = pruner.Prune(targetRoot); err != nil { - log.Error("Failed to prune state", "error", err) + log.Error("Failed to prune state", "err", err) return err } return nil @@ -189,7 +189,7 @@ func verifyState(ctx *cli.Context) error { } snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) if err != nil { - log.Error("Failed to open snapshot tree", "error", err) + log.Error("Failed to open snapshot tree", "err", err) return err } if ctx.NArg() > 1 { @@ -200,15 +200,15 @@ func verifyState(ctx *cli.Context) error { if ctx.NArg() == 1 { root, err = parseRoot(ctx.Args()[0]) if err != nil { - log.Error("Failed to resolve state root", "error", err) + log.Error("Failed to resolve state root", "err", err) return err } } if err := snaptree.Verify(root); err != nil { - log.Error("Failed to verfiy state", "error", err) + log.Error("Failed to verfiy state", "root", root, "err", err) return err } - log.Info("Verified the state") + log.Info("Verified the state", "root", root) return nil } @@ -236,7 +236,7 @@ func traverseState(ctx *cli.Context) error { if ctx.NArg() == 1 { root, err = parseRoot(ctx.Args()[0]) if err != nil { - log.Error("Failed to resolve state root", "error", err) + log.Error("Failed to resolve state root", "err", err) return err } log.Info("Start traversing the state", "root", root) @@ -247,7 +247,7 @@ func traverseState(ctx *cli.Context) error { triedb := trie.NewDatabase(chaindb) t, err := trie.NewSecure(root, triedb) if err != nil { - log.Error("Failed to open trie", "root", root, "error", err) + log.Error("Failed to open trie", "root", root, "err", err) return err } var ( @@ -262,13 +262,13 @@ func traverseState(ctx *cli.Context) error { accounts += 1 var acc state.Account if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { - log.Error("Invalid account encountered during traversal", "error", err) + log.Error("Invalid account encountered during traversal", "err", err) return err } if acc.Root != emptyRoot { storageTrie, err := trie.NewSecure(acc.Root, triedb) if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "error", err) + log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err } storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) @@ -276,7 +276,7 @@ func traverseState(ctx *cli.Context) error { slots += 1 } if storageIter.Err != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err) + log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err) return storageIter.Err } } @@ -294,7 +294,7 @@ func traverseState(ctx *cli.Context) error { } } if accIter.Err != nil { - log.Error("Failed to traverse state trie", "root", root, "error", accIter.Err) + log.Error("Failed to traverse state trie", "root", root, "err", accIter.Err) return accIter.Err } log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) @@ -326,7 +326,7 @@ func traverseRawState(ctx *cli.Context) error { if ctx.NArg() == 1 { root, err = parseRoot(ctx.Args()[0]) if err != nil { - log.Error("Failed to resolve state root", "error", err) + log.Error("Failed to resolve state root", "err", err) return err } log.Info("Start traversing the state", "root", root) @@ -337,7 +337,7 @@ func traverseRawState(ctx *cli.Context) error { triedb := trie.NewDatabase(chaindb) t, err := trie.NewSecure(root, triedb) if err != nil { - log.Error("Failed to open trie", "root", root, "error", err) + log.Error("Failed to open trie", "root", root, "err", err) return err } var ( @@ -368,13 +368,13 @@ func traverseRawState(ctx *cli.Context) error { accounts += 1 var acc state.Account if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { - log.Error("Invalid account encountered during traversal", "error", err) + log.Error("Invalid account encountered during traversal", "err", err) return errors.New("invalid account") } if acc.Root != emptyRoot { storageTrie, err := trie.NewSecure(acc.Root, triedb) if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "error", err) + log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") } storageIter := storageTrie.NodeIterator(nil) @@ -397,7 +397,7 @@ func traverseRawState(ctx *cli.Context) error { } } if storageIter.Error() != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error()) + log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error()) return storageIter.Error() } } @@ -416,7 +416,7 @@ func traverseRawState(ctx *cli.Context) error { } } if accIter.Error() != nil { - log.Error("Failed to traverse state trie", "root", root, "error", accIter.Error()) + log.Error("Failed to traverse state trie", "root", root, "err", accIter.Error()) return accIter.Error() } log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index bb87ecddf1..f70cbf1e68 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -322,7 +322,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return } if !bytes.Equal(account.Root, subroot.Bytes()) { - results <- fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot) + results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot) return } results <- nil diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 2b41dd5513..98c8d42a1a 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -19,17 +19,20 @@ package snapshot import ( "bytes" "encoding/binary" + "errors" "fmt" "math/big" "time" "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -40,17 +43,63 @@ var ( // emptyCode is the known hash of the empty EVM bytecode. emptyCode = crypto.Keccak256Hash(nil) + + // accountCheckRange is the upper limit of the number of accounts involved in + // each range check. This is a value estimated based on experience. If this + // value is too large, the failure rate of range prove will increase. Otherwise + // the the value is too small, the efficiency of the state recovery will decrease. + accountCheckRange = 128 + + // storageCheckRange is the upper limit of the number of storage slots involved + // in each range check. This is a value estimated based on experience. If this + // value is too large, the failure rate of range prove will increase. Otherwise + // the the value is too small, the efficiency of the state recovery will decrease. + storageCheckRange = 1024 + + // errMissingTrie is returned if the target trie is missing while the generation + // is running. In this case the generation is aborted and wait the new signal. + errMissingTrie = errors.New("missing trie") +) + +// Metrics in generation +var ( + snapGeneratedAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/generated", nil) + snapRecoveredAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/recovered", nil) + snapWipedAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/wiped", nil) + snapMissallAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/missall", nil) + snapGeneratedStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/generated", nil) + snapRecoveredStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/recovered", nil) + snapWipedStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/wiped", nil) + snapMissallStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/missall", nil) + snapSuccessfulRangeProofMeter = metrics.NewRegisteredMeter("state/snapshot/generation/proof/success", nil) + snapFailedRangeProofMeter = metrics.NewRegisteredMeter("state/snapshot/generation/proof/failure", nil) + + // snapAccountProveCounter measures time spent on the account proving + snapAccountProveCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/prove", nil) + // snapAccountTrieReadCounter measures time spent on the account trie iteration + snapAccountTrieReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/trieread", nil) + // snapAccountSnapReadCounter measues time spent on the snapshot account iteration + snapAccountSnapReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/snapread", nil) + // snapAccountWriteCounter measures time spent on writing/updating/deleting accounts + snapAccountWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/write", nil) + // snapStorageProveCounter measures time spent on storage proving + snapStorageProveCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/prove", nil) + // snapStorageTrieReadCounter measures time spent on the storage trie iteration + snapStorageTrieReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/trieread", nil) + // snapStorageSnapReadCounter measures time spent on the snapshot storage iteration + snapStorageSnapReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/snapread", nil) + // snapStorageWriteCounter measures time spent on writing/updating/deleting storages + snapStorageWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/write", nil) ) // generatorStats is a collection of statistics gathered by the snapshot generator // for logging purposes. type generatorStats struct { - wiping chan struct{} // Notification channel if wiping is in progress origin uint64 // Origin prefix where generation started start time.Time // Timestamp when generation started - accounts uint64 // Number of accounts indexed - slots uint64 // Number of storage slots indexed - storage common.StorageSize // Account and storage slot size + accounts uint64 // Number of accounts indexed(generated or recovered) + slots uint64 // Number of storage slots indexed(generated or recovered) + storage common.StorageSize // Total account and storage slot size(generation or recovery) } // Log creates an contextual log with the given message and the context pulled @@ -91,25 +140,30 @@ func (gs *generatorStats) Log(msg string, root common.Hash, marker []byte) { log.Info(msg, ctx...) } +// ClearSnapshotMarker sets the snapshot marker to zero, meaning that snapshots +// are not usable. +func ClearSnapshotMarker(diskdb ethdb.KeyValueStore) { + batch := diskdb.NewBatch() + journalProgress(batch, []byte{}, nil) + if err := batch.Write(); err != nil { + log.Crit("Failed to write initialized state marker", "err", err) + } +} + // generateSnapshot regenerates a brand new snapshot based on an existing state // database and head block asynchronously. The snapshot is returned immediately // and generation is continued in the background until done. -func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, wiper chan struct{}) *diskLayer { - // Wipe any previously existing snapshot from the database if no wiper is - // currently in progress. - if wiper == nil { - wiper = wipeSnapshot(diskdb, true) - } +func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) *diskLayer { // Create a new disk layer with an initialized state marker at zero var ( - stats = &generatorStats{wiping: wiper, start: time.Now()} + stats = &generatorStats{start: time.Now()} batch = diskdb.NewBatch() genMarker = []byte{} // Initialized but empty! ) rawdb.WriteSnapshotRoot(batch, root) journalProgress(batch, genMarker, stats) if err := batch.Write(); err != nil { - log.Crit("Failed to write initialized state marker", "error", err) + log.Crit("Failed to write initialized state marker", "err", err) } base := &diskLayer{ diskdb: diskdb, @@ -135,7 +189,6 @@ func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorSta Marker: marker, } if stats != nil { - entry.Wiping = (stats.wiping != nil) entry.Accounts = stats.accounts entry.Slots = stats.slots entry.Storage = uint64(stats.storage) @@ -159,169 +212,521 @@ func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorSta rawdb.WriteSnapshotGenerator(db, blob) } -// generate is a background thread that iterates over the state and storage tries, -// constructing the state snapshot. All the arguments are purely for statistics -// gathering and logging, since the method surfs the blocks as they arrive, often -// being restarted. -func (dl *diskLayer) generate(stats *generatorStats) { - // If a database wipe is in operation, wait until it's done - if stats.wiping != nil { - stats.Log("Wiper running, state snapshotting paused", common.Hash{}, dl.genMarker) - select { - // If wiper is done, resume normal mode of operation - case <-stats.wiping: - stats.wiping = nil - stats.start = time.Now() +// proofResult contains the output of range proving which can be used +// for further processing regardless if it is successful or not. +type proofResult struct { + keys [][]byte // The key set of all elements being iterated, even proving is failed + vals [][]byte // The val set of all elements being iterated, even proving is failed + diskMore bool // Set when the database has extra snapshot states since last iteration + trieMore bool // Set when the trie has extra snapshot states(only meaningful for successful proving) + proofErr error // Indicator whether the given state range is valid or not + tr *trie.Trie // The trie, in case the trie was resolved by the prover (may be nil) +} - // If generator was aborted during wipe, return - case abort := <-dl.genAbort: - abort <- stats - return +// valid returns the indicator that range proof is successful or not. +func (result *proofResult) valid() bool { + return result.proofErr == nil +} + +// last returns the last verified element key regardless of whether the range proof is +// successful or not. Nil is returned if nothing involved in the proving. +func (result *proofResult) last() []byte { + var last []byte + if len(result.keys) > 0 { + last = result.keys[len(result.keys)-1] + } + return last +} + +// forEach iterates all the visited elements and applies the given callback on them. +// The iteration is aborted if the callback returns non-nil error. +func (result *proofResult) forEach(callback func(key []byte, val []byte) error) error { + for i := 0; i < len(result.keys); i++ { + key, val := result.keys[i], result.vals[i] + if err := callback(key, val); err != nil { + return err + } + } + return nil +} + +// proveRange proves the snapshot segment with particular prefix is "valid". +// The iteration start point will be assigned if the iterator is restored from +// the last interruption. Max will be assigned in order to limit the maximum +// amount of data involved in each iteration. +// +// The proof result will be returned if the range proving is finished, otherwise +// the error will be returned to abort the entire procedure. +func (dl *diskLayer) proveRange(stats *generatorStats, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { + var ( + keys [][]byte + vals [][]byte + proof = rawdb.NewMemoryDatabase() + diskMore = false + ) + iter := dl.diskdb.NewIterator(prefix, origin) + defer iter.Release() + + var start = time.Now() + for iter.Next() { + key := iter.Key() + if len(key) != len(prefix)+common.HashLength { + continue + } + if len(keys) == max { + // Break if we've reached the max size, and signal that we're not + // done yet. + diskMore = true + break + } + keys = append(keys, common.CopyBytes(key[len(prefix):])) + + if valueConvertFn == nil { + vals = append(vals, common.CopyBytes(iter.Value())) + } else { + val, err := valueConvertFn(iter.Value()) + if err != nil { + // Special case, the state data is corrupted (invalid slim-format account), + // don't abort the entire procedure directly. Instead, let the fallback + // generation to heal the invalid data. + // + // Here append the original value to ensure that the number of key and + // value are the same. + vals = append(vals, common.CopyBytes(iter.Value())) + log.Error("Failed to convert account state data", "err", err) + } else { + vals = append(vals, val) + } + } + } + // Update metrics for database iteration and merkle proving + if kind == "storage" { + snapStorageSnapReadCounter.Inc(time.Since(start).Nanoseconds()) + } else { + snapAccountSnapReadCounter.Inc(time.Since(start).Nanoseconds()) + } + defer func(start time.Time) { + if kind == "storage" { + snapStorageProveCounter.Inc(time.Since(start).Nanoseconds()) + } else { + snapAccountProveCounter.Inc(time.Since(start).Nanoseconds()) + } + }(time.Now()) + + // The snap state is exhausted, pass the entire key/val set for verification + if origin == nil && !diskMore { + stackTr := trie.NewStackTrie(nil) + for i, key := range keys { + stackTr.TryUpdate(key, common.CopyBytes(vals[i])) + } + if gotRoot := stackTr.Hash(); gotRoot != root { + return &proofResult{ + keys: keys, + vals: vals, + proofErr: fmt.Errorf("wrong root: have %#x want %#x", gotRoot, root), + }, nil } + return &proofResult{keys: keys, vals: vals}, nil } - // Create an account and state iterator pointing to the current generator marker - accTrie, err := trie.NewSecure(dl.root, dl.triedb) + // Snap state is chunked, generate edge proofs for verification. + tr, err := trie.New(root, dl.triedb) if err != nil { - // The account trie is missing (GC), surf the chain until one becomes available stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) + return nil, errMissingTrie + } + // Firstly find out the key of last iterated element. + var last []byte + if len(keys) > 0 { + last = keys[len(keys)-1] + } + // Generate the Merkle proofs for the first and last element + if origin == nil { + origin = common.Hash{}.Bytes() + } + if err := tr.Prove(origin, 0, proof); err != nil { + log.Debug("Failed to prove range", "kind", kind, "origin", origin, "err", err) + return &proofResult{ + keys: keys, + vals: vals, + diskMore: diskMore, + proofErr: err, + tr: tr, + }, nil + } + if last != nil { + if err := tr.Prove(last, 0, proof); err != nil { + log.Debug("Failed to prove range", "kind", kind, "last", last, "err", err) + return &proofResult{ + keys: keys, + vals: vals, + diskMore: diskMore, + proofErr: err, + tr: tr, + }, nil + } + } + // Verify the snapshot segment with range prover, ensure that all flat states + // in this range correspond to merkle trie. + _, _, _, cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) + return &proofResult{ + keys: keys, + vals: vals, + diskMore: diskMore, + trieMore: cont, + proofErr: err, + tr: tr}, + nil +} - abort := <-dl.genAbort - abort <- stats - return +// onStateCallback is a function that is called by generateRange, when processing a range of +// accounts or storage slots. For each element, the callback is invoked. +// If 'delete' is true, then this element (and potential slots) needs to be deleted from the snapshot. +// If 'write' is true, then this element needs to be updated with the 'val'. +// If 'write' is false, then this element is already correct, and needs no update. However, +// for accounts, the storage trie of the account needs to be checked. +// The 'val' is the canonical encoding of the value (not the slim format for accounts) +type onStateCallback func(key []byte, val []byte, write bool, delete bool) error + +// generateRange generates the state segment with particular prefix. Generation can +// either verify the correctness of existing state through rangeproof and skip +// generation, or iterate trie to regenerate state on demand. +func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, origin []byte, max int, stats *generatorStats, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { + // Use range prover to check the validity of the flat state in the range + result, err := dl.proveRange(stats, root, prefix, kind, origin, max, valueConvertFn) + if err != nil { + return false, nil, err } - stats.Log("Resuming state snapshot generation", dl.root, dl.genMarker) + last := result.last() + + // Construct contextual logger + logCtx := []interface{}{"kind", kind, "prefix", hexutil.Encode(prefix)} + if len(origin) > 0 { + logCtx = append(logCtx, "origin", hexutil.Encode(origin)) + } + logger := log.New(logCtx...) + + // The range prover says the range is correct, skip trie iteration + if result.valid() { + snapSuccessfulRangeProofMeter.Mark(1) + logger.Trace("Proved state range", "last", hexutil.Encode(last)) - var accMarker []byte + // The verification is passed, process each state with the given + // callback function. If this state represents a contract, the + // corresponding storage check will be performed in the callback + if err := result.forEach(func(key []byte, val []byte) error { return onState(key, val, false, false) }); err != nil { + return false, nil, err + } + // Only abort the iteration when both database and trie are exhausted + return !result.diskMore && !result.trieMore, last, nil + } + logger.Trace("Detected outdated state range", "last", hexutil.Encode(last), "err", result.proofErr) + snapFailedRangeProofMeter.Mark(1) + + // Special case, the entire trie is missing. In the original trie scheme, + // all the duplicated subtries will be filter out(only one copy of data + // will be stored). While in the snapshot model, all the storage tries + // belong to different contracts will be kept even they are duplicated. + // Track it to a certain extent remove the noise data used for statistics. + if origin == nil && last == nil { + meter := snapMissallAccountMeter + if kind == "storage" { + meter = snapMissallStorageMeter + } + meter.Mark(1) + } + tr := result.tr + if tr == nil { + tr, err = trie.New(root, dl.triedb) + if err != nil { + stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) + return false, nil, errMissingTrie + } + } + var ( + trieMore bool + iter = trie.NewIterator(tr.NodeIterator(origin)) + kvkeys, kvvals = result.keys, result.vals + + // counters + count = 0 // number of states delivered by iterator + created = 0 // states created from the trie + updated = 0 // states updated from the trie + deleted = 0 // states not in trie, but were in snapshot + untouched = 0 // states already correct + + // timers + start = time.Now() + internal time.Duration + ) + for iter.Next() { + if last != nil && bytes.Compare(iter.Key, last) > 0 { + trieMore = true + break + } + count++ + write := true + created++ + for len(kvkeys) > 0 { + if cmp := bytes.Compare(kvkeys[0], iter.Key); cmp < 0 { + // delete the key + istart := time.Now() + if err := onState(kvkeys[0], nil, false, true); err != nil { + return false, nil, err + } + kvkeys = kvkeys[1:] + kvvals = kvvals[1:] + deleted++ + internal += time.Since(istart) + continue + } else if cmp == 0 { + // the snapshot key can be overwritten + created-- + if write = !bytes.Equal(kvvals[0], iter.Value); write { + updated++ + } else { + untouched++ + } + kvkeys = kvkeys[1:] + kvvals = kvvals[1:] + } + break + } + istart := time.Now() + if err := onState(iter.Key, iter.Value, write, false); err != nil { + return false, nil, err + } + internal += time.Since(istart) + } + if iter.Err != nil { + return false, nil, iter.Err + } + // Delete all stale snapshot states remaining + istart := time.Now() + for _, key := range kvkeys { + if err := onState(key, nil, false, true); err != nil { + return false, nil, err + } + deleted += 1 + } + internal += time.Since(istart) + + // Update metrics for counting trie iteration + if kind == "storage" { + snapStorageTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds()) + } else { + snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds()) + } + logger.Debug("Regenerated state range", "root", root, "last", hexutil.Encode(last), + "count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted) + + // If there are either more trie items, or there are more snap items + // (in the next segment), then we need to keep working + return !trieMore && !result.diskMore, last, nil +} + +// generate is a background thread that iterates over the state and storage tries, +// constructing the state snapshot. All the arguments are purely for statistics +// gathering and logging, since the method surfs the blocks as they arrive, often +// being restarted. +func (dl *diskLayer) generate(stats *generatorStats) { + var ( + accMarker []byte + accountRange = accountCheckRange + ) if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that - accMarker = dl.genMarker[:common.HashLength] + // Always reset the initial account range as 1 + // whenever recover from the interruption. + accMarker, accountRange = dl.genMarker[:common.HashLength], 1 } - accIt := trie.NewIterator(accTrie.NodeIterator(accMarker)) - batch := dl.diskdb.NewBatch() + var ( + batch = dl.diskdb.NewBatch() + logged = time.Now() + accOrigin = common.CopyBytes(accMarker) + abort chan *generatorStats + ) + stats.Log("Resuming state snapshot generation", dl.root, dl.genMarker) - // Iterate from the previous marker and continue generating the state snapshot - logged := time.Now() - for accIt.Next() { - // Retrieve the current account and flatten it into the internal format - accountHash := common.BytesToHash(accIt.Key) + checkAndFlush := func(currentLocation []byte) error { + select { + case abort = <-dl.genAbort: + default: + } + if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil { + // Flush out the batch anyway no matter it's empty or not. + // It's possible that all the states are recovered and the + // generation indeed makes progress. + journalProgress(batch, currentLocation, stats) + + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + + dl.lock.Lock() + dl.genMarker = currentLocation + dl.lock.Unlock() + + if abort != nil { + stats.Log("Aborting state snapshot generation", dl.root, currentLocation) + return errors.New("aborted") + } + } + if time.Since(logged) > 8*time.Second { + stats.Log("Generating state snapshot", dl.root, currentLocation) + logged = time.Now() + } + return nil + } + onAccount := func(key []byte, val []byte, write bool, delete bool) error { + var ( + start = time.Now() + accountHash = common.BytesToHash(key) + ) + if delete { + rawdb.DeleteAccountSnapshot(batch, accountHash) + snapWipedAccountMeter.Mark(1) + + // Ensure that any previous snapshot storage values are cleared + prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...) + keyLen := len(rawdb.SnapshotStoragePrefix) + 2*common.HashLength + if err := wipeKeyRange(dl.diskdb, "storage", prefix, nil, nil, keyLen, snapWipedStorageMeter, false); err != nil { + return err + } + snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds()) + return nil + } + // Retrieve the current account and flatten it into the internal format var acc struct { Nonce uint64 Balance *big.Int Root common.Hash CodeHash []byte } - if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil { + if err := rlp.DecodeBytes(val, &acc); err != nil { log.Crit("Invalid account encountered during snapshot creation", "err", err) } - data := SlimAccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash) - // If the account is not yet in-progress, write it out if accMarker == nil || !bytes.Equal(accountHash[:], accMarker) { - rawdb.WriteAccountSnapshot(batch, accountHash, data) - stats.storage += common.StorageSize(1 + common.HashLength + len(data)) + dataLen := len(val) // Approximate size, saves us a round of RLP-encoding + if !write { + if bytes.Equal(acc.CodeHash, emptyCode[:]) { + dataLen -= 32 + } + if acc.Root == emptyRoot { + dataLen -= 32 + } + snapRecoveredAccountMeter.Mark(1) + } else { + data := SlimAccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash) + dataLen = len(data) + rawdb.WriteAccountSnapshot(batch, accountHash, data) + snapGeneratedAccountMeter.Mark(1) + } + stats.storage += common.StorageSize(1 + common.HashLength + dataLen) stats.accounts++ } // If we've exceeded our batch allowance or termination was requested, flush to disk - var abort chan *generatorStats - select { - case abort = <-dl.genAbort: - default: - } - if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil { - // Only write and set the marker if we actually did something useful - if batch.ValueSize() > 0 { - // Ensure the generator entry is in sync with the data - marker := accountHash[:] - journalProgress(batch, marker, stats) - - batch.Write() - batch.Reset() - - dl.lock.Lock() - dl.genMarker = marker - dl.lock.Unlock() - } - if abort != nil { - stats.Log("Aborting state snapshot generation", dl.root, accountHash[:]) - abort <- stats - return - } + if err := checkAndFlush(accountHash[:]); err != nil { + return err } - // If the account is in-progress, continue where we left off (otherwise iterate all) - if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(acc.Root, dl.triedb) - if err != nil { - log.Error("Generator failed to access storage trie", "root", dl.root, "account", accountHash, "stroot", acc.Root, "err", err) - abort := <-dl.genAbort - abort <- stats - return + // If the iterated account is the contract, create a further loop to + // verify or regenerate the contract storage. + if acc.Root == emptyRoot { + // If the root is empty, we still need to ensure that any previous snapshot + // storage values are cleared + // TODO: investigate if this can be avoided, this will be very costly since it + // affects every single EOA account + // - Perhaps we can avoid if where codeHash is emptyCode + prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...) + keyLen := len(rawdb.SnapshotStoragePrefix) + 2*common.HashLength + if err := wipeKeyRange(dl.diskdb, "storage", prefix, nil, nil, keyLen, snapWipedStorageMeter, false); err != nil { + return err } + snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds()) + } else { + snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds()) + var storeMarker []byte if accMarker != nil && bytes.Equal(accountHash[:], accMarker) && len(dl.genMarker) > common.HashLength { storeMarker = dl.genMarker[common.HashLength:] } - storeIt := trie.NewIterator(storeTrie.NodeIterator(storeMarker)) - for storeIt.Next() { - rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(storeIt.Key), storeIt.Value) - stats.storage += common.StorageSize(1 + 2*common.HashLength + len(storeIt.Value)) + onStorage := func(key []byte, val []byte, write bool, delete bool) error { + defer func(start time.Time) { + snapStorageWriteCounter.Inc(time.Since(start).Nanoseconds()) + }(time.Now()) + + if delete { + rawdb.DeleteStorageSnapshot(batch, accountHash, common.BytesToHash(key)) + snapWipedStorageMeter.Mark(1) + return nil + } + if write { + rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(key), val) + snapGeneratedStorageMeter.Mark(1) + } else { + snapRecoveredStorageMeter.Mark(1) + } + stats.storage += common.StorageSize(1 + 2*common.HashLength + len(val)) stats.slots++ // If we've exceeded our batch allowance or termination was requested, flush to disk - var abort chan *generatorStats - select { - case abort = <-dl.genAbort: - default: - } - if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil { - // Only write and set the marker if we actually did something useful - if batch.ValueSize() > 0 { - // Ensure the generator entry is in sync with the data - marker := append(accountHash[:], storeIt.Key...) - journalProgress(batch, marker, stats) - - batch.Write() - batch.Reset() - - dl.lock.Lock() - dl.genMarker = marker - dl.lock.Unlock() - } - if abort != nil { - stats.Log("Aborting state snapshot generation", dl.root, append(accountHash[:], storeIt.Key...)) - abort <- stats - return - } - if time.Since(logged) > 8*time.Second { - stats.Log("Generating state snapshot", dl.root, append(accountHash[:], storeIt.Key...)) - logged = time.Now() - } + if err := checkAndFlush(append(accountHash[:], key...)); err != nil { + return err } + return nil } - if err := storeIt.Err; err != nil { - log.Error("Generator failed to iterate storage trie", "accroot", dl.root, "acchash", common.BytesToHash(accIt.Key), "stroot", acc.Root, "err", err) - abort := <-dl.genAbort - abort <- stats - return + var storeOrigin = common.CopyBytes(storeMarker) + for { + exhausted, last, err := dl.generateRange(acc.Root, append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...), "storage", storeOrigin, storageCheckRange, stats, onStorage, nil) + if err != nil { + return err + } + if exhausted { + break + } + if storeOrigin = increaseKey(last); storeOrigin == nil { + break // special case, the last is 0xffffffff...fff + } } } - if time.Since(logged) > 8*time.Second { - stats.Log("Generating state snapshot", dl.root, accIt.Key) - logged = time.Now() - } // Some account processed, unmark the marker accMarker = nil + return nil } - if err := accIt.Err; err != nil { - log.Error("Generator failed to iterate account trie", "root", dl.root, "err", err) - abort := <-dl.genAbort - abort <- stats - return + + // Global loop for regerating the entire state trie + all layered storage tries. + for { + exhausted, last, err := dl.generateRange(dl.root, rawdb.SnapshotAccountPrefix, "account", accOrigin, accountRange, stats, onAccount, FullAccountRLP) + // The procedure it aborted, either by external signal or internal error + if err != nil { + if abort == nil { // aborted by internal error, wait the signal + abort = <-dl.genAbort + } + abort <- stats + return + } + // Abort the procedure if the entire snapshot is generated + if exhausted { + break + } + if accOrigin = increaseKey(last); accOrigin == nil { + break // special case, the last is 0xffffffff...fff + } + accountRange = accountCheckRange } // Snapshot fully generated, set the marker to nil. // Note even there is nothing to commit, persist the // generator anyway to mark the snapshot is complete. journalProgress(batch, nil, stats) - batch.Write() + if err := batch.Write(); err != nil { + log.Error("Failed to flush batch", "err", err) + + abort = <-dl.genAbort + abort <- stats + return + } + batch.Reset() log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots, "storage", stats.storage, "elapsed", common.PrettyDuration(time.Since(stats.start))) @@ -332,6 +737,18 @@ func (dl *diskLayer) generate(stats *generatorStats) { dl.lock.Unlock() // Someone will be looking for us, wait it out - abort := <-dl.genAbort + abort = <-dl.genAbort abort <- nil } + +// increaseKey increase the input key by one bit. Return nil if the entire +// addition operation overflows, +func increaseKey(key []byte) []byte { + for i := len(key) - 1; i >= 0; i-- { + key[i]++ + if key[i] != 0x0 { + return key + } + } + return nil +} diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 03263f3976..3a669085f7 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -17,16 +17,361 @@ package snapshot import ( + "fmt" "math/big" + "os" "testing" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/crypto/sha3" ) +// Tests that snapshot generation from an empty database. +func TestGeneration(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // two of which also has the same 3-slot storage trie attached. + var ( + diskdb = memorydb.New() + triedb = trie.NewDatabase(diskdb) + ) + stTrie, _ := trie.NewSecure(common.Hash{}, triedb) + stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 + stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 + stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 + stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + + accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + + acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} + val, _ = rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + + acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ = rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + triedb.Commit(root, false, nil) + + if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { + t.Fatalf("have %#x want %#x", have, want) + } + snap := generateSnapshot(diskdb, triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +func hashData(input []byte) common.Hash { + var hasher = sha3.NewLegacyKeccak256() + var hash common.Hash + hasher.Reset() + hasher.Write(input) + hasher.Sum(hash[:0]) + return hash +} + +// Tests that snapshot generation with existent flat state. +func TestGenerateExistentState(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // two of which also has the same 3-slot storage trie attached. + var ( + diskdb = memorydb.New() + triedb = trie.NewDatabase(diskdb) + ) + stTrie, _ := trie.NewSecure(common.Hash{}, triedb) + stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 + stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 + stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 + stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + + accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val) + rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3")) + + acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} + val, _ = rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + diskdb.Put(hashData([]byte("acc-2")).Bytes(), val) + rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val) + + acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ = rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val) + rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3")) + + root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + triedb.Commit(root, false, nil) + + snap := generateSnapshot(diskdb, triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { + t.Helper() + accIt := snap.AccountIterator(common.Hash{}) + defer accIt.Release() + snapRoot, err := generateTrieRoot(nil, accIt, common.Hash{}, stackTrieGenerate, + func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) + defer storageIt.Release() + + hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true) + + if err != nil { + t.Fatal(err) + } + if snapRoot != trieRoot { + t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) + } +} + +type testHelper struct { + diskdb *memorydb.Database + triedb *trie.Database + accTrie *trie.SecureTrie +} + +func newHelper() *testHelper { + diskdb := memorydb.New() + triedb := trie.NewDatabase(diskdb) + accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + return &testHelper{ + diskdb: diskdb, + triedb: triedb, + accTrie: accTrie, + } +} + +func (t *testHelper) addTrieAccount(acckey string, acc *Account) { + val, _ := rlp.EncodeToBytes(acc) + t.accTrie.Update([]byte(acckey), val) +} + +func (t *testHelper) addSnapAccount(acckey string, acc *Account) { + val, _ := rlp.EncodeToBytes(acc) + key := hashData([]byte(acckey)) + rawdb.WriteAccountSnapshot(t.diskdb, key, val) +} + +func (t *testHelper) addAccount(acckey string, acc *Account) { + t.addTrieAccount(acckey, acc) + t.addSnapAccount(acckey, acc) +} + +func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) { + accHash := hashData([]byte(accKey)) + for i, key := range keys { + rawdb.WriteStorageSnapshot(t.diskdb, accHash, hashData([]byte(key)), []byte(vals[i])) + } +} + +func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { + stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb) + for i, k := range keys { + stTrie.Update([]byte(k), []byte(vals[i])) + } + root, _ := stTrie.Commit(nil) + return root.Bytes() +} + +func (t *testHelper) Generate() (common.Hash, *diskLayer) { + root, _ := t.accTrie.Commit(nil) + t.triedb.Commit(root, false, nil) + snap := generateSnapshot(t.diskdb, t.triedb, 16, root) + return root, snap +} + +// Tests that snapshot generation with existent flat state, where the flat state +// contains some errors: +// - the contract with empty storage root but has storage entries in the disk +// - the contract with non empty storage root but empty storage slots +// - the contract(non-empty storage) misses some storage slots +// - miss in the beginning +// - miss in the middle +// - miss in the end +// - the contract(non-empty storage) has wrong storage slots +// - wrong slots in the beginning +// - wrong slots in the middle +// - wrong slots in the end +// - the contract(non-empty storage) has extra storage slots +// - extra slots in the beginning +// - extra slots in the middle +// - extra slots in the end +func TestGenerateExistentStateWithWrongStorage(t *testing.T) { + helper := newHelper() + stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Account one, empty root but non-empty database + helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Account two, non empty root but empty database + helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + + // Miss slots + { + // Account three, non empty root but misses slots in the beginning + helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) + + // Account four, non empty root but misses slots in the middle + helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) + + // Account five, non empty root but misses slots in the end + helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) + } + + // Wrong storage slots + { + // Account six, non empty root but wrong slots in the beginning + helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) + + // Account seven, non empty root but wrong slots in the middle + helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) + + // Account eight, non empty root but wrong slots in the end + helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) + + // Account 9, non empty root but rotated slots + helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) + } + + // Extra storage slots + { + // Account 10, non empty root but extra slots in the beginning + helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) + + // Account 11, non empty root but extra slots in the middle + helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) + + // Account 12, non empty root but extra slots in the end + helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) + } + + root, snap := helper.Generate() + t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation with existent flat state, where the flat state +// contains some errors: +// - miss accounts +// - wrong accounts +// - extra accounts +func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { + helper := newHelper() + stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] + // Extra accounts [acc-0, acc-5, acc-7] + + // Missing accounts, only in the trie + { + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Beginning + helper.addTrieAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Middle + helper.addTrieAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // End + } + + // Wrong accounts + { + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) + + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + } + + // Extra accounts, only in the snap + { + helper.addSnapAccount("acc-0", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyRoot.Bytes()}) // before the beginning + helper.addSnapAccount("acc-5", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle + helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end + } + + root, snap := helper.Generate() + t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + // Tests that snapshot generation errors out correctly in case of a missing trie // node in the account trie. func TestGenerateCorruptAccountTrie(t *testing.T) { @@ -55,7 +400,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil) diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), nil) + snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978")) select { case <-snap.genPending: // Snapshot generation succeeded @@ -115,7 +460,7 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // Delete a storage trie root and ensure the generator chokes diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), nil) + snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) select { case <-snap.genPending: // Snapshot generation succeeded @@ -174,7 +519,7 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // Delete a storage trie leaf and ensure the generator chokes diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), nil) + snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) select { case <-snap.genPending: // Snapshot generation succeeded @@ -188,3 +533,301 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { snap.genAbort <- stop <-stop } + +func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie { + stTrie, _ := trie.NewSecure(common.Hash{}, triedb) + for i := 0; i < n; i++ { + k := fmt.Sprintf("key-%d", i) + v := fmt.Sprintf("val-%d", i) + stTrie.Update([]byte(k), []byte(v)) + } + stTrie.Commit(nil) + return stTrie +} + +// Tests that snapshot generation when an extra account with storage exists in the snap state. +func TestGenerateWithExtraAccounts(t *testing.T) { + var ( + diskdb = memorydb.New() + triedb = trie.NewDatabase(diskdb) + stTrie = getStorageTrie(5, triedb) + ) + accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + { // Account one in the trie + acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap + key := hashData([]byte("acc-1")) + rawdb.WriteAccountSnapshot(diskdb, key, val) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5")) + } + { // Account two exists only in the snapshot + acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + key := hashData([]byte("acc-2")) + rawdb.WriteAccountSnapshot(diskdb, key, val) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) + } + root, _ := accTrie.Commit(nil) + t.Logf("root: %x", root) + triedb.Commit(root, false, nil) + // To verify the test: If we now inspect the snap db, there should exist extraneous storage items + if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { + t.Fatalf("expected snap storage to exist") + } + + snap := generateSnapshot(diskdb, triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop + // If we now inspect the snap db, there should exist no extraneous storage items + if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + t.Fatalf("expected slot to be removed, got %v", string(data)) + } +} + +func enableLogging() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) +} + +// Tests that snapshot generation when an extra account with storage exists in the snap state. +func TestGenerateWithManyExtraAccounts(t *testing.T) { + if false { + enableLogging() + } + var ( + diskdb = memorydb.New() + triedb = trie.NewDatabase(diskdb) + stTrie = getStorageTrie(3, triedb) + ) + accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + { // Account one in the trie + acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap + key := hashData([]byte("acc-1")) + rawdb.WriteAccountSnapshot(diskdb, key, val) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + } + { // 100 accounts exist only in snapshot + for i := 0; i < 1000; i++ { + //acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + key := hashData([]byte(fmt.Sprintf("acc-%d", i))) + rawdb.WriteAccountSnapshot(diskdb, key, val) + } + } + root, _ := accTrie.Commit(nil) + t.Logf("root: %x", root) + triedb.Commit(root, false, nil) + + snap := generateSnapshot(diskdb, triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests this case +// maxAccountRange 3 +// snapshot-accounts: 01, 02, 03, 04, 05, 06, 07 +// trie-accounts: 03, 07 +// +// We iterate three snapshot storage slots (max = 3) from the database. They are 0x01, 0x02, 0x03. +// The trie has a lot of deletions. +// So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. +// But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. +func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { + accountCheckRange = 3 + if false { + enableLogging() + } + var ( + diskdb = memorydb.New() + triedb = trie.NewDatabase(diskdb) + ) + accTrie, _ := trie.New(common.Hash{}, triedb) + { + acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + accTrie.Update(common.HexToHash("0x03").Bytes(), val) + accTrie.Update(common.HexToHash("0x07").Bytes(), val) + + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) + } + + root, _ := accTrie.Commit(nil) + t.Logf("root: %x", root) + triedb.Commit(root, false, nil) + + snap := generateSnapshot(diskdb, triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// TestGenerateWithMalformedSnapdata tests what happes if we have some junk +// in the snapshot database, which cannot be parsed back to an account +func TestGenerateWithMalformedSnapdata(t *testing.T) { + accountCheckRange = 3 + if false { + enableLogging() + } + var ( + diskdb = memorydb.New() + triedb = trie.NewDatabase(diskdb) + ) + accTrie, _ := trie.New(common.Hash{}, triedb) + { + acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + accTrie.Update(common.HexToHash("0x03").Bytes(), val) + + junk := make([]byte, 100) + copy(junk, []byte{0xde, 0xad}) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk) + rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) + } + + root, _ := accTrie.Commit(nil) + t.Logf("root: %x", root) + triedb.Commit(root, false, nil) + + snap := generateSnapshot(diskdb, triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop + // If we now inspect the snap db, there should exist no extraneous storage items + if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + t.Fatalf("expected slot to be removed, got %v", string(data)) + } +} + +func TestGenerateFromEmptySnap(t *testing.T) { + //enableLogging() + accountCheckRange = 10 + storageCheckRange = 20 + helper := newHelper() + stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + // Add 1K accounts to the trie + for i := 0; i < 400; i++ { + helper.addTrieAccount(fmt.Sprintf("acc-%d", i), + &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + } + root, snap := helper.Generate() + t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(1 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation with existent flat state, where the flat state +// storage is correct, but incomplete. +// The incomplete part is on the second range +// snap: [ 0x01, 0x02, 0x03, 0x04] , [ 0x05, 0x06, 0x07, {missing}] (with storageCheck = 4) +// trie: 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +// This hits a case where the snap verification passes, but there are more elements in the trie +// which we must also add. +func TestGenerateWithIncompleteStorage(t *testing.T) { + storageCheckRange = 4 + helper := newHelper() + stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} + stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} + stRoot := helper.makeStorageTrie(stKeys, stVals) + // We add 8 accounts, each one is missing exactly one of the storage slots. This means + // we don't have to order the keys and figure out exactly which hash-key winds up + // on the sensitive spots at the boundaries + for i := 0; i < 8; i++ { + accKey := fmt.Sprintf("acc-%d", i) + helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()}) + var moddedKeys []string + var moddedVals []string + for ii := 0; ii < 8; ii++ { + if ii != i { + moddedKeys = append(moddedKeys, stKeys[ii]) + moddedVals = append(moddedVals, stVals[ii]) + } + } + helper.addSnapStorage(accKey, moddedKeys, moddedVals) + } + + root, snap := helper.Generate() + t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(250 * time.Millisecond): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index d7e454cceb..b31e921ca9 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -37,7 +37,10 @@ const journalVersion uint64 = 0 // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { - Wiping bool // Whether the database was in progress of being wiped + // Indicator that whether the database was in progress of being wiped. + // It's deprecated but keep it here for background compatibility. + Wiping bool + Done bool // Whether the generator finished creating the snapshot Marker []byte Accounts uint64 @@ -193,14 +196,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, } // Everything loaded correctly, resume any suspended operations if !generator.Done { - // If the generator was still wiping, restart one from scratch (fine for - // now as it's rare and the wiper deletes the stuff it touches anyway, so - // restarting won't incur a lot of extra database hops. - var wiper chan struct{} - if generator.Wiping { - log.Info("Resuming previous snapshot wipe") - wiper = wipeSnapshot(diskdb, false) - } // Whether or not wiping was in progress, load any generator progress too base.genMarker = generator.Marker if base.genMarker == nil { @@ -214,7 +209,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, origin = binary.BigEndian.Uint64(generator.Marker) } go base.generate(&generatorStats{ - wiping: wiper, origin: origin, start: time.Now(), accounts: generator.Accounts, @@ -381,7 +375,6 @@ func (dl *diskLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) { Marker: dl.genMarker, } if stats != nil { - entry.Wiping = (stats.wiping != nil) entry.Accounts = stats.accounts entry.Slots = stats.slots entry.Storage = uint64(stats.storage) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index eccf377264..710ba4d4c2 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -656,9 +656,6 @@ func (t *Tree) Rebuild(root common.Hash) { // building a brand new snapshot. rawdb.DeleteSnapshotRecoveryNumber(t.diskdb) - // Track whether there's a wipe currently running and keep it alive if so - var wiper chan struct{} - // Iterate over and mark all layers stale for _, layer := range t.layers { switch layer := layer.(type) { @@ -667,10 +664,7 @@ func (t *Tree) Rebuild(root common.Hash) { if layer.genAbort != nil { abort := make(chan *generatorStats) layer.genAbort <- abort - - if stats := <-abort; stats != nil { - wiper = stats.wiping - } + <-abort } // Layer should be inactive now, mark it as stale layer.lock.Lock() @@ -691,7 +685,7 @@ func (t *Tree) Rebuild(root common.Hash) { // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") t.layers = map[common.Hash]snapshot{ - root: generateSnapshot(t.diskdb, t.triedb, t.cache, root, wiper), + root: generateSnapshot(t.diskdb, t.triedb, t.cache, root), } } diff --git a/core/state/snapshot/wipe.go b/core/state/snapshot/wipe.go index 14b63031a5..2cab57393b 100644 --- a/core/state/snapshot/wipe.go +++ b/core/state/snapshot/wipe.go @@ -24,10 +24,11 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) // wipeSnapshot starts a goroutine to iterate over the entire key-value database -// and delete all the data associated with the snapshot (accounts, storage, +// and delete all the data associated with the snapshot (accounts, storage, // metadata). After all is done, the snapshot range of the database is compacted // to free up unused data blocks. func wipeSnapshot(db ethdb.KeyValueStore, full bool) chan struct{} { @@ -53,10 +54,10 @@ func wipeSnapshot(db ethdb.KeyValueStore, full bool) chan struct{} { // removed in sync to avoid data races. After all is done, the snapshot range of // the database is compacted to free up unused data blocks. func wipeContent(db ethdb.KeyValueStore) error { - if err := wipeKeyRange(db, "accounts", rawdb.SnapshotAccountPrefix, len(rawdb.SnapshotAccountPrefix)+common.HashLength); err != nil { + if err := wipeKeyRange(db, "accounts", rawdb.SnapshotAccountPrefix, nil, nil, len(rawdb.SnapshotAccountPrefix)+common.HashLength, snapWipedAccountMeter, true); err != nil { return err } - if err := wipeKeyRange(db, "storage", rawdb.SnapshotStoragePrefix, len(rawdb.SnapshotStoragePrefix)+2*common.HashLength); err != nil { + if err := wipeKeyRange(db, "storage", rawdb.SnapshotStoragePrefix, nil, nil, len(rawdb.SnapshotStoragePrefix)+2*common.HashLength, snapWipedStorageMeter, true); err != nil { return err } // Compact the snapshot section of the database to get rid of unused space @@ -82,8 +83,11 @@ func wipeContent(db ethdb.KeyValueStore) error { } // wipeKeyRange deletes a range of keys from the database starting with prefix -// and having a specific total key length. -func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int) error { +// and having a specific total key length. The start and limit is optional for +// specifying a particular key range for deletion. +// +// Origin is included for wiping and limit is excluded if they are specified. +func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, origin []byte, limit []byte, keylen int, meter metrics.Meter, report bool) error { // Batch deletions together to avoid holding an iterator for too long var ( batch = db.NewBatch() @@ -92,7 +96,11 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int // Iterate over the key-range and delete all of them start, logged := time.Now(), time.Now() - it := db.NewIterator(prefix, nil) + it := db.NewIterator(prefix, origin) + var stop []byte + if limit != nil { + stop = append(prefix, limit...) + } for it.Next() { // Skip any keys with the correct prefix but wrong length (trie nodes) key := it.Key() @@ -102,6 +110,9 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int if len(key) != keylen { continue } + if stop != nil && bytes.Compare(key, stop) >= 0 { + break + } // Delete the key and periodically recreate the batch and iterator batch.Delete(key) items++ @@ -116,7 +127,7 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int seekPos := key[len(prefix):] it = db.NewIterator(prefix, seekPos) - if time.Since(logged) > 8*time.Second { + if time.Since(logged) > 8*time.Second && report { log.Info("Deleting state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } @@ -126,6 +137,11 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int if err := batch.Write(); err != nil { return err } - log.Info("Deleted state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start))) + if meter != nil { + meter.Mark(int64(items)) + } + if report { + log.Info("Deleted state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start))) + } return nil } diff --git a/core/state/statedb.go b/core/state/statedb.go index 2e5d6e47c8..90f4709bfc 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -948,7 +948,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. var account Account - root, err := s.trie.Commit(func(path []byte, leaf []byte, parent common.Hash) error { + root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil } diff --git a/core/state/sync.go b/core/state/sync.go index 1018b78e5e..7a5852fb19 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -26,17 +26,31 @@ import ( ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.SyncBloom) *trie.Sync { +func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.SyncBloom, onLeaf func(paths [][]byte, leaf []byte) error) *trie.Sync { + // Register the storage slot callback if the external callback is specified. + var onSlot func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error + if onLeaf != nil { + onSlot = func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { + return onLeaf(paths, leaf) + } + } + // Register the account callback to connect the state trie and the storage + // trie belongs to the contract. var syncer *trie.Sync - callback := func(path []byte, leaf []byte, parent common.Hash) error { + onAccount := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { + if onLeaf != nil { + if err := onLeaf(paths, leaf); err != nil { + return err + } + } var obj Account if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } - syncer.AddSubTrie(obj.Root, path, parent, nil) - syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent) + syncer.AddSubTrie(obj.Root, hexpath, parent, onSlot) + syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), hexpath, parent) return nil } - syncer = trie.NewSync(root, database, callback, bloom) + syncer = trie.NewSync(root, database, onAccount, bloom) return syncer } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 9c4867093d..a13fcf56a3 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -133,7 +133,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New())) + sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New()), nil) if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes) } @@ -170,7 +170,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb)) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil) nodes, paths, codes := sched.Missing(count) var ( @@ -249,7 +249,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb)) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil) nodes, _, codes := sched.Missing(0) queue := append(append([]common.Hash{}, nodes...), codes...) @@ -297,7 +297,7 @@ func testIterativeRandomStateSync(t *testing.T, count int) { // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb)) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil) queue := make(map[common.Hash]struct{}) nodes, _, codes := sched.Missing(count) @@ -347,7 +347,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb)) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil) queue := make(map[common.Hash]struct{}) nodes, _, codes := sched.Missing(0) @@ -414,7 +414,7 @@ func TestIncompleteStateSync(t *testing.T) { // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb)) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil) var added []common.Hash diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 6231588ad2..ff84a3a8f0 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -298,7 +298,7 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { return &stateSync{ d: d, root: root, - sched: state.NewStateSync(root, d.stateDB, d.stateBloom), + sched: state.NewStateSync(root, d.stateDB, d.stateBloom, nil), keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), trieTasks: make(map[common.Hash]*trieTask), codeTasks: make(map[common.Hash]*codeTask), diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 2924fa0802..22b0c8604d 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -51,7 +52,7 @@ const ( // maxRequestSize is the maximum number of bytes to request from a remote peer. maxRequestSize = 512 * 1024 - // maxStorageSetRequestCountis th maximum number of contracts to request the + // maxStorageSetRequestCount is the maximum number of contracts to request the // storage of in a single query. If this number is too low, we're not filling // responses fully and waste round trip times. If it's too high, we're capping // responses and waste bandwidth. @@ -435,9 +436,14 @@ type Syncer struct { bytecodeHealDups uint64 // Number of bytecodes already processed bytecodeHealNops uint64 // Number of bytecodes not requested - startTime time.Time // Time instance when snapshot sync started - startAcc common.Hash // Account hash where sync started from - logTime time.Time // Time instance when status was last reported + stateWriter ethdb.Batch // Shared batch writer used for persisting raw states + accountHealed uint64 // Number of accounts downloaded during the healing stage + accountHealedBytes common.StorageSize // Number of raw account bytes persisted to disk during the healing stage + storageHealed uint64 // Number of storage slots downloaded during the healing stage + storageHealedBytes common.StorageSize // Number of raw storage bytes persisted to disk during the healing stage + + startTime time.Time // Time instance when snapshot sync started + logTime time.Time // Time instance when status was last reported pend sync.WaitGroup // Tracks network request goroutines for graceful shutdown lock sync.RWMutex // Protects fields that can change outside of sync (peers, reqs, root) @@ -477,6 +483,7 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { bytecodeHealReqFails: make(chan *bytecodeHealRequest), trienodeHealResps: make(chan *trienodeHealResponse), bytecodeHealResps: make(chan *bytecodeHealResponse), + stateWriter: db.NewBatch(), } } @@ -544,7 +551,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { s.lock.Lock() s.root = root s.healer = &healTask{ - scheduler: state.NewStateSync(root, s.db, nil), + scheduler: state.NewStateSync(root, s.db, nil, s.onHealState), trieTasks: make(map[common.Hash]trie.SyncPath), codeTasks: make(map[common.Hash]struct{}), } @@ -560,6 +567,11 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { log.Debug("Snapshot sync already completed") return nil } + // If sync is still not finished, we need to ensure that any marker is wiped. + // Otherwise, it may happen that requests for e.g. genesis-data is delivered + // from the snapshot data, instead of from the trie + snapshot.ClearSnapshotMarker(s.db) + defer func() { // Persist any progress, independent of failure for _, task := range s.tasks { s.forwardAccountTask(task) @@ -569,6 +581,14 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { }() log.Debug("Starting snapshot sync cycle", "root", root) + + // Flush out the last committed raw states + defer func() { + if s.stateWriter.ValueSize() > 0 { + s.stateWriter.Write() + s.stateWriter.Reset() + } + }() defer s.report(true) // Whether sync completed or not, disregard any future packets @@ -1694,7 +1714,7 @@ func (s *Syncer) processBytecodeResponse(res *bytecodeResponse) { // processStorageResponse integrates an already validated storage response // into the account tasks. func (s *Syncer) processStorageResponse(res *storageResponse) { - // Switch the suntask from pending to idle + // Switch the subtask from pending to idle if res.subTask != nil { res.subTask.req = nil } @@ -1826,6 +1846,14 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { nodes++ } it.Release() + + // Persist the received storage segements. These flat state maybe + // outdated during the sync, but it can be fixed later during the + // snapshot generation. + for j := 0; j < len(res.hashes[i]); j++ { + rawdb.WriteStorageSnapshot(batch, account, res.hashes[i][j], res.slots[i][j]) + bytes += common.StorageSize(1 + 2*common.HashLength + len(res.slots[i][j])) + } } if err := batch.Write(); err != nil { log.Crit("Failed to persist storage slots", "err", err) @@ -1983,6 +2011,14 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } it.Release() + // Persist the received account segements. These flat state maybe + // outdated during the sync, but it can be fixed later during the + // snapshot generation. + for i, hash := range res.hashes { + blob := snapshot.SlimAccountRLP(res.accounts[i].Nonce, res.accounts[i].Balance, res.accounts[i].Root, res.accounts[i].CodeHash) + rawdb.WriteAccountSnapshot(batch, hash, blob) + bytes += common.StorageSize(1 + common.HashLength + len(blob)) + } if err := batch.Write(); err != nil { log.Crit("Failed to persist accounts", "err", err) } @@ -2569,6 +2605,33 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e return nil } +// onHealState is a callback method to invoke when a flat state(account +// or storage slot) is downloded during the healing stage. The flat states +// can be persisted blindly and can be fixed later in the generation stage. +// Note it's not concurrent safe, please handle the concurrent issue outside. +func (s *Syncer) onHealState(paths [][]byte, value []byte) error { + if len(paths) == 1 { + var account state.Account + if err := rlp.DecodeBytes(value, &account); err != nil { + return nil + } + blob := snapshot.SlimAccountRLP(account.Nonce, account.Balance, account.Root, account.CodeHash) + rawdb.WriteAccountSnapshot(s.stateWriter, common.BytesToHash(paths[0]), blob) + s.accountHealed += 1 + s.accountHealedBytes += common.StorageSize(1 + common.HashLength + len(blob)) + } + if len(paths) == 2 { + rawdb.WriteStorageSnapshot(s.stateWriter, common.BytesToHash(paths[0]), common.BytesToHash(paths[1]), value) + s.storageHealed += 1 + s.storageHealedBytes += common.StorageSize(1 + 2*common.HashLength + len(value)) + } + if s.stateWriter.ValueSize() > ethdb.IdealBatchSize { + s.stateWriter.Write() // It's fine to ignore the error here + s.stateWriter.Reset() + } + return nil +} + // hashSpace is the total size of the 256 bit hash space for accounts. var hashSpace = new(big.Int).Exp(common.Big2, common.Big256, nil) @@ -2632,7 +2695,9 @@ func (s *Syncer) reportHealProgress(force bool) { var ( trienode = fmt.Sprintf("%d@%v", s.trienodeHealSynced, s.trienodeHealBytes.TerminalString()) bytecode = fmt.Sprintf("%d@%v", s.bytecodeHealSynced, s.bytecodeHealBytes.TerminalString()) + accounts = fmt.Sprintf("%d@%v", s.accountHealed, s.accountHealedBytes.TerminalString()) + storage = fmt.Sprintf("%d@%v", s.storageHealed, s.storageHealedBytes.TerminalString()) ) - log.Info("State heal in progress", "nodes", trienode, "codes", bytecode, - "pending", s.healer.scheduler.Pending()) + log.Info("State heal in progress", "accounts", accounts, "slots", storage, + "codes", bytecode, "nodes", trienode, "pending", s.healer.scheduler.Pending()) } diff --git a/trie/committer.go b/trie/committer.go index 33fd9e9823..ce4065f5fd 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -220,13 +220,13 @@ func (c *committer) commitLoop(db *Database) { switch n := n.(type) { case *shortNode: if child, ok := n.Val.(valueNode); ok { - c.onleaf(nil, child, hash) + c.onleaf(nil, nil, child, hash) } case *fullNode: // For children in range [0, 15], it's impossible // to contain valuenode. Only check the 17th child. if n.Children[16] != nil { - c.onleaf(nil, n.Children[16].(valueNode), hash) + c.onleaf(nil, nil, n.Children[16].(valueNode), hash) } } } diff --git a/trie/sync.go b/trie/sync.go index dd8279b665..3a6076ff8f 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -398,7 +398,14 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // Notify any external watcher of a new key/value node if req.callback != nil { if node, ok := (child.node).(valueNode); ok { - if err := req.callback(child.path, node, req.hash); err != nil { + var paths [][]byte + if len(child.path) == 2*common.HashLength { + paths = append(paths, hexToKeybytes(child.path)) + } else if len(child.path) == 4*common.HashLength { + paths = append(paths, hexToKeybytes(child.path[:2*common.HashLength])) + paths = append(paths, hexToKeybytes(child.path[2*common.HashLength:])) + } + if err := req.callback(paths, child.path, node, req.hash); err != nil { return nil, err } } diff --git a/trie/trie.go b/trie/trie.go index 87b72ecf17..7ed235fa8a 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -37,9 +37,20 @@ var ( ) // LeafCallback is a callback type invoked when a trie operation reaches a leaf -// node. It's used by state sync and commit to allow handling external references -// between account and storage tries. -type LeafCallback func(path []byte, leaf []byte, parent common.Hash) error +// node. +// +// The paths is a path tuple identifying a particular trie node either in a single +// trie (account) or a layered trie (account -> storage). Each path in the tuple +// is in the raw format(32 bytes). +// +// The hexpath is a composite hexary path identifying the trie node. All the key +// bytes are converted to the hexary nibbles and composited with the parent path +// if the trie node is in a layered trie. +// +// It's used by state sync and commit to allow handling external references +// between account and storage tries. And also it's used in the state healing +// for extracting the raw states(leaf nodes) with corresponding paths. +type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error // Trie is a Merkle Patricia Trie. // The zero value is an empty trie with no database. diff --git a/trie/trie_test.go b/trie/trie_test.go index 3aa4098d14..d6930fdee7 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -569,7 +569,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { benchmarkCommitAfterHash(b, nil) }) var a account - onleaf := func(path []byte, leaf []byte, parent common.Hash) error { + onleaf := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { rlp.DecodeBytes(leaf, &a) return nil } From d5e57948d1d889b129be87e2cbcdbfaa727ef5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Apr 2021 23:39:42 +0300 Subject: [PATCH 270/709] core/types: drop some relice data types --- core/types/block.go | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/core/types/block.go b/core/types/block.go index 553db003bb..a3318f8779 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -166,19 +166,6 @@ type Block struct { ReceivedFrom interface{} } -// DeprecatedTd is an old relic for extracting the TD of a block. It is in the -// code solely to facilitate upgrading the database from the old format to the -// new, after which it should be deleted. Do not use! -func (b *Block) DeprecatedTd() *big.Int { - return b.td -} - -// [deprecated by eth/63] -// StorageBlock defines the RLP encoding of a Block stored in the -// state database. The StorageBlock encoding contains fields that -// would otherwise need to be recomputed. -type StorageBlock Block - // "external" block encoding. used for eth protocol, etc. type extblock struct { Header *Header @@ -186,15 +173,6 @@ type extblock struct { Uncles []*Header } -// [deprecated by eth/63] -// "storage" block encoding. used for database. -type storageblock struct { - Header *Header - Txs []*Transaction - Uncles []*Header - TD *big.Int -} - // NewBlock creates a new block. The input data is copied, // changes to header and to the field values will not affect the // block. @@ -279,16 +257,6 @@ func (b *Block) EncodeRLP(w io.Writer) error { }) } -// [deprecated by eth/63] -func (b *StorageBlock) DecodeRLP(s *rlp.Stream) error { - var sb storageblock - if err := s.Decode(&sb); err != nil { - return err - } - b.header, b.uncles, b.transactions, b.td = sb.Header, sb.Uncles, sb.Txs, sb.TD - return nil -} - // TODO: copies func (b *Block) Uncles() []*Header { return b.uncles } From 1e207342b545fb5992e2e8f38e60f219f379e5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 15 Apr 2021 20:35:00 +0300 Subject: [PATCH 271/709] all: make logs a bit easier on the eye to digest (#22665) * all: add thousandths separators for big numbers on log messages * p2p/sentry: drop accidental file * common, log: add fast number formatter * common, eth/protocols/snap: simplifty fancy num types * log: handle nil big ints --- accounts/url.go | 2 +- cmd/evm/README.md | 6 +- cmd/evm/testdata/8/readme.md | 6 +- common/types.go | 2 +- core/blockchain.go | 10 ++-- core/blockchain_test.go | 4 +- core/chain_indexer.go | 2 +- core/headerchain.go | 2 +- eth/protocols/snap/sync.go | 14 ++--- log/format.go | 106 ++++++++++++++++++++++++++++++++++- log/format_test.go | 75 +++++++++++++++++++++++++ 11 files changed, 202 insertions(+), 27 deletions(-) create mode 100644 log/format_test.go diff --git a/accounts/url.go b/accounts/url.go index a5add10216..12a84414a0 100644 --- a/accounts/url.go +++ b/accounts/url.go @@ -64,7 +64,7 @@ func (u URL) String() string { func (u URL) TerminalString() string { url := u.String() if len(url) > 32 { - return url[:31] + "…" + return url[:31] + ".." } return url } diff --git a/cmd/evm/README.md b/cmd/evm/README.md index 7742ccbbb7..cdff41f904 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -256,9 +256,9 @@ Error code: 4 Another thing that can be done, is to chain invocations: ``` ./evm t8n --input.alloc=./testdata/1/alloc.json --input.txs=./testdata/1/txs.json --input.env=./testdata/1/env.json --output.alloc=stdout | ./evm t8n --input.alloc=stdin --input.env=./testdata/1/env.json --input.txs=./testdata/1/txs.json -INFO [01-21|22:41:22.963] rejected tx index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" -INFO [01-21|22:41:22.966] rejected tx index=0 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" -INFO [01-21|22:41:22.967] rejected tx index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [01-21|22:41:22.963] rejected tx index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [01-21|22:41:22.966] rejected tx index=0 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [01-21|22:41:22.967] rejected tx index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" ``` What happened here, is that we first applied two identical transactions, so the second one was rejected. diff --git a/cmd/evm/testdata/8/readme.md b/cmd/evm/testdata/8/readme.md index 778fc6151a..e021cd7e2e 100644 --- a/cmd/evm/testdata/8/readme.md +++ b/cmd/evm/testdata/8/readme.md @@ -56,8 +56,8 @@ dir=./testdata/8 \ If we try to execute it on older rules: ``` dir=./testdata/8 && ./evm t8n --state.fork=Istanbul --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json -INFO [01-21|23:21:51.265] rejected tx index=0 hash="d2818d…6ab3da" error="tx type not supported" -INFO [01-21|23:21:51.265] rejected tx index=1 hash="26ea00…81c01b" from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0" -INFO [01-21|23:21:51.265] rejected tx index=2 hash="698d01…369cee" error="tx type not supported" +INFO [01-21|23:21:51.265] rejected tx index=0 hash=d2818d..6ab3da error="tx type not supported" +INFO [01-21|23:21:51.265] rejected tx index=1 hash=26ea00..81c01b from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0" +INFO [01-21|23:21:51.265] rejected tx index=2 hash=698d01..369cee error="tx type not supported" ``` Number `1` and `3` are not applicable, and therefore number `2` has wrong nonce. \ No newline at end of file diff --git a/common/types.go b/common/types.go index d920e8b1f1..d715356692 100644 --- a/common/types.go +++ b/common/types.go @@ -76,7 +76,7 @@ func (h Hash) Hex() string { return hexutil.Encode(h[:]) } // TerminalString implements log.TerminalStringer, formatting a string for console // output during logging. func (h Hash) TerminalString() string { - return fmt.Sprintf("%x…%x", h[:3], h[29:]) + return fmt.Sprintf("%x..%x", h[:3], h[29:]) } // String implements the stringer interface and is used also by the logger when diff --git a/core/blockchain.go b/core/blockchain.go index d65ce4f048..8c3d940a69 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -630,7 +630,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { // Make sure that both the block as well at its state trie exists block := bc.GetBlockByHash(hash) if block == nil { - return fmt.Errorf("non existent block [%x…]", hash[:4]) + return fmt.Errorf("non existent block [%x..]", hash[:4]) } if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil { return err @@ -1147,7 +1147,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() { log.Error("Non contiguous receipt insert", "number", blockChain[i].Number(), "hash", blockChain[i].Hash(), "parent", blockChain[i].ParentHash(), "prevnumber", blockChain[i-1].Number(), "prevhash", blockChain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, blockChain[i-1].NumberU64(), + return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, blockChain[i-1].NumberU64(), blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4]) } } @@ -1212,7 +1212,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Short circuit if the owner header is unknown if !bc.HasHeader(block.Hash(), block.NumberU64()) { - return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) + return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4]) } var ( start = time.Now() @@ -1356,7 +1356,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Short circuit if the owner header is unknown if !bc.HasHeader(block.Hash(), block.NumberU64()) { - return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) + return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4]) } if !skipPresenceCheck { // Ignore if the entire data is already known @@ -1679,7 +1679,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { log.Error("Non contiguous block insert", "number", block.Number(), "hash", block.Hash(), "parent", block.ParentHash(), "prevnumber", prev.Number(), "prevhash", prev.Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, prev.NumberU64(), + return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, prev.NumberU64(), prev.Hash().Bytes()[:4], i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4]) } } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index fd7f1aea1b..5004abd1c7 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1462,13 +1462,13 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { t.Fatalf("block %d: failed to insert into chain: %v", i, err) } if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() { - t.Errorf("block %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) + t.Errorf("block %d: current block/header mismatch: block #%d [%x..], header #%d [%x..]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) } if _, err := chain.InsertChain(forks[i : i+1]); err != nil { t.Fatalf(" fork %d: failed to insert into chain: %v", i, err) } if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() { - t.Errorf(" fork %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) + t.Errorf(" fork %d: current block/header mismatch: block #%d [%x..], header #%d [%x..]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) } } } diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 4b326c970b..95901a0eaa 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -401,7 +401,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com } header := rawdb.ReadHeader(c.chainDb, hash, number) if header == nil { - return common.Hash{}, fmt.Errorf("block #%d [%x…] not found", number, hash[:4]) + return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4]) } else if header.ParentHash != lastHead { return common.Hash{}, fmt.Errorf("chain reorged during section processing") } diff --git a/core/headerchain.go b/core/headerchain.go index 9aee660eba..1dbf958786 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -306,7 +306,7 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) log.Error("Non contiguous header insert", "number", chain[i].Number, "hash", hash, "parent", chain[i].ParentHash, "prevnumber", chain[i-1].Number, "prevhash", parentHash) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].Number, + return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, chain[i-1].Number, parentHash.Bytes()[:4], i, chain[i].Number, hash.Bytes()[:4], chain[i].ParentHash[:4]) } // If the header is a banned one, straight out abort diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 22b0c8604d..1ea589c7b2 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2675,9 +2675,9 @@ func (s *Syncer) reportSyncProgress(force bool) { // Create a mega progress report var ( progress = fmt.Sprintf("%.2f%%", float64(synced)*100/estBytes) - accounts = fmt.Sprintf("%d@%v", s.accountSynced, s.accountBytes.TerminalString()) - storage = fmt.Sprintf("%d@%v", s.storageSynced, s.storageBytes.TerminalString()) - bytecode = fmt.Sprintf("%d@%v", s.bytecodeSynced, s.bytecodeBytes.TerminalString()) + accounts = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.accountSynced), s.accountBytes.TerminalString()) + storage = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.storageSynced), s.storageBytes.TerminalString()) + bytecode = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.bytecodeSynced), s.bytecodeBytes.TerminalString()) ) log.Info("State sync in progress", "synced", progress, "state", synced, "accounts", accounts, "slots", storage, "codes", bytecode, "eta", common.PrettyDuration(estTime-elapsed)) @@ -2693,10 +2693,10 @@ func (s *Syncer) reportHealProgress(force bool) { // Create a mega progress report var ( - trienode = fmt.Sprintf("%d@%v", s.trienodeHealSynced, s.trienodeHealBytes.TerminalString()) - bytecode = fmt.Sprintf("%d@%v", s.bytecodeHealSynced, s.bytecodeHealBytes.TerminalString()) - accounts = fmt.Sprintf("%d@%v", s.accountHealed, s.accountHealedBytes.TerminalString()) - storage = fmt.Sprintf("%d@%v", s.storageHealed, s.storageHealedBytes.TerminalString()) + trienode = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.trienodeHealSynced), s.trienodeHealBytes.TerminalString()) + bytecode = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.bytecodeHealSynced), s.bytecodeHealBytes.TerminalString()) + accounts = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.accountHealed), s.accountHealedBytes.TerminalString()) + storage = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.storageHealed), s.storageHealedBytes.TerminalString()) ) log.Info("State heal in progress", "accounts", accounts, "slots", storage, "codes", bytecode, "nodes", trienode, "pending", s.healer.scheduler.Pending()) diff --git a/log/format.go b/log/format.go index 421384cc1d..0667921528 100644 --- a/log/format.go +++ b/log/format.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "math/big" "reflect" "strconv" "strings" @@ -329,11 +330,20 @@ func formatLogfmtValue(value interface{}, term bool) string { return "nil" } - if t, ok := value.(time.Time); ok { + switch v := value.(type) { + case time.Time: // Performance optimization: No need for escaping since the provided // timeFormat doesn't have any escape characters, and escaping is // expensive. - return t.Format(timeFormat) + return v.Format(timeFormat) + + case *big.Int: + // Big ints get consumed by the Stringer clause so we need to handle + // them earlier on. + if v == nil { + return "" + } + return formatLogfmtBigInt(v) } if term { if s, ok := value.(TerminalStringer); ok { @@ -349,8 +359,24 @@ func formatLogfmtValue(value interface{}, term bool) string { return strconv.FormatFloat(float64(v), floatFormat, 3, 64) case float64: return strconv.FormatFloat(v, floatFormat, 3, 64) - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + case int8, uint8: return fmt.Sprintf("%d", value) + case int: + return FormatLogfmtInt64(int64(v)) + case int16: + return FormatLogfmtInt64(int64(v)) + case int32: + return FormatLogfmtInt64(int64(v)) + case int64: + return FormatLogfmtInt64(v) + case uint: + return FormatLogfmtUint64(uint64(v)) + case uint16: + return FormatLogfmtUint64(uint64(v)) + case uint32: + return FormatLogfmtUint64(uint64(v)) + case uint64: + return FormatLogfmtUint64(v) case string: return escapeString(v) default: @@ -358,6 +384,80 @@ func formatLogfmtValue(value interface{}, term bool) string { } } +// FormatLogfmtInt64 formats a potentially big number in a friendlier split format. +func FormatLogfmtInt64(n int64) string { + if n < 0 { + return formatLogfmtUint64(uint64(-n), true) + } + return formatLogfmtUint64(uint64(n), false) +} + +// FormatLogfmtUint64 formats a potentially big number in a friendlier split format. +func FormatLogfmtUint64(n uint64) string { + return formatLogfmtUint64(n, false) +} + +func formatLogfmtUint64(n uint64, neg bool) string { + // Small numbers are fine as is + if n < 100000 { + if neg { + return strconv.Itoa(-int(n)) + } else { + return strconv.Itoa(int(n)) + } + } + // Large numbers should be split + const maxLength = 26 + + var ( + out = make([]byte, maxLength) + i = maxLength - 1 + comma = 0 + ) + for ; n > 0; i-- { + if comma == 3 { + comma = 0 + out[i] = ',' + } else { + comma++ + out[i] = '0' + byte(n%10) + n /= 10 + } + } + if neg { + out[i] = '-' + i-- + } + return string(out[i+1:]) +} + +var big1000 = big.NewInt(1000) + +// formatLogfmtBigInt formats a potentially gigantic number in a friendlier split +// format. +func formatLogfmtBigInt(n *big.Int) string { + // Most number don't need fancy handling, just downcast + if n.IsUint64() { + return FormatLogfmtUint64(n.Uint64()) + } + if n.IsInt64() { + return FormatLogfmtInt64(n.Int64()) + } + // Ok, huge number needs huge effort + groups := make([]string, 0, 8) // random initial size to cover most cases + for n.Cmp(big1000) >= 0 { + _, mod := n.DivMod(n, big1000, nil) + groups = append(groups, fmt.Sprintf("%03d", mod)) + } + groups = append(groups, n.String()) + + last := len(groups) - 1 + for i := 0; i < len(groups)/2; i++ { + groups[i], groups[last-i] = groups[last-i], groups[i] + } + return strings.Join(groups, ",") +} + // escapeString checks if the provided string needs escaping/quoting, and // calls strconv.Quote if needed func escapeString(s string) string { diff --git a/log/format_test.go b/log/format_test.go new file mode 100644 index 0000000000..348b265c9b --- /dev/null +++ b/log/format_test.go @@ -0,0 +1,75 @@ +package log + +import ( + "math" + "math/rand" + "testing" +) + +func TestPrettyInt64(t *testing.T) { + tests := []struct { + n int64 + s string + }{ + {0, "0"}, + {10, "10"}, + {-10, "-10"}, + {100, "100"}, + {-100, "-100"}, + {1000, "1000"}, + {-1000, "-1000"}, + {10000, "10000"}, + {-10000, "-10000"}, + {99999, "99999"}, + {-99999, "-99999"}, + {100000, "100,000"}, + {-100000, "-100,000"}, + {1000000, "1,000,000"}, + {-1000000, "-1,000,000"}, + {math.MaxInt64, "9,223,372,036,854,775,807"}, + {math.MinInt64, "-9,223,372,036,854,775,808"}, + } + for i, tt := range tests { + if have := FormatLogfmtInt64(tt.n); have != tt.s { + t.Errorf("test %d: format mismatch: have %s, want %s", i, have, tt.s) + } + } +} + +func TestPrettyUint64(t *testing.T) { + tests := []struct { + n uint64 + s string + }{ + {0, "0"}, + {10, "10"}, + {100, "100"}, + {1000, "1000"}, + {10000, "10000"}, + {99999, "99999"}, + {100000, "100,000"}, + {1000000, "1,000,000"}, + {math.MaxUint64, "18,446,744,073,709,551,615"}, + } + for i, tt := range tests { + if have := FormatLogfmtUint64(tt.n); have != tt.s { + t.Errorf("test %d: format mismatch: have %s, want %s", i, have, tt.s) + } + } +} + +var sink string + +func BenchmarkPrettyInt64Logfmt(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + sink = FormatLogfmtInt64(rand.Int63()) + } +} + +func BenchmarkPrettyUint64Logfmt(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + sink = FormatLogfmtUint64(rand.Uint64()) + } +} From 9553c98de8f730e77b612a642089824f1c35bf3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 15 Apr 2021 21:01:16 +0300 Subject: [PATCH 272/709] eth/protocols/snap: use ephemeral channels to avoid cross-sync delveries --- eth/protocols/snap/sync.go | 213 +++++++++++++++++++------------------ 1 file changed, 112 insertions(+), 101 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 1ea589c7b2..cff1a77e6c 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -106,9 +106,11 @@ type accountRequest struct { peer string // Peer to which this request is assigned id uint64 // Request ID of this request - cancel chan struct{} // Channel to track sync cancellation - timeout *time.Timer // Timer to track delivery timeout - stale chan struct{} // Channel to signal the request was dropped + deliver chan *accountResponse // Channel to deliver successful response on + revert chan *accountRequest // Channel to deliver request failure on + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped origin common.Hash // First account requested to allow continuation checks limit common.Hash // Last account requested to allow non-overlapping chunking @@ -147,9 +149,11 @@ type bytecodeRequest struct { peer string // Peer to which this request is assigned id uint64 // Request ID of this request - cancel chan struct{} // Channel to track sync cancellation - timeout *time.Timer // Timer to track delivery timeout - stale chan struct{} // Channel to signal the request was dropped + deliver chan *bytecodeResponse // Channel to deliver successful response on + revert chan *bytecodeRequest // Channel to deliver request failure on + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped hashes []common.Hash // Bytecode hashes to validate responses task *accountTask // Task which this request is filling (only access fields through the runloop!!) @@ -176,9 +180,11 @@ type storageRequest struct { peer string // Peer to which this request is assigned id uint64 // Request ID of this request - cancel chan struct{} // Channel to track sync cancellation - timeout *time.Timer // Timer to track delivery timeout - stale chan struct{} // Channel to signal the request was dropped + deliver chan *storageResponse // Channel to deliver successful response on + revert chan *storageRequest // Channel to deliver request failure on + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped accounts []common.Hash // Account hashes to validate responses roots []common.Hash // Storage roots to validate responses @@ -224,9 +230,11 @@ type trienodeHealRequest struct { peer string // Peer to which this request is assigned id uint64 // Request ID of this request - cancel chan struct{} // Channel to track sync cancellation - timeout *time.Timer // Timer to track delivery timeout - stale chan struct{} // Channel to signal the request was dropped + deliver chan *trienodeHealResponse // Channel to deliver successful response on + revert chan *trienodeHealRequest // Channel to deliver request failure on + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped hashes []common.Hash // Trie node hashes to validate responses paths []trie.SyncPath // Trie node paths requested for rescheduling @@ -256,9 +264,11 @@ type bytecodeHealRequest struct { peer string // Peer to which this request is assigned id uint64 // Request ID of this request - cancel chan struct{} // Channel to track sync cancellation - timeout *time.Timer // Timer to track delivery timeout - stale chan struct{} // Channel to signal the request was dropped + deliver chan *bytecodeHealResponse // Channel to deliver successful response on + revert chan *bytecodeHealRequest // Channel to deliver request failure on + cancel chan struct{} // Channel to track sync cancellation + timeout *time.Timer // Timer to track delivery timeout + stale chan struct{} // Channel to signal the request was dropped hashes []common.Hash // Bytecode hashes to validate responses task *healTask // Task which this request is filling (only access fields through the runloop!!) @@ -399,14 +409,6 @@ type Syncer struct { bytecodeReqs map[uint64]*bytecodeRequest // Bytecode requests currently running storageReqs map[uint64]*storageRequest // Storage requests currently running - accountReqFails chan *accountRequest // Failed account range requests to revert - bytecodeReqFails chan *bytecodeRequest // Failed bytecode requests to revert - storageReqFails chan *storageRequest // Failed storage requests to revert - - accountResps chan *accountResponse // Account sub-tries to integrate into the database - bytecodeResps chan *bytecodeResponse // Bytecodes to integrate into the database - storageResps chan *storageResponse // Storage sub-tries to integrate into the database - accountSynced uint64 // Number of accounts downloaded accountBytes common.StorageSize // Number of account trie bytes persisted to disk bytecodeSynced uint64 // Number of bytecodes downloaded @@ -421,12 +423,6 @@ type Syncer struct { trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running - trienodeHealReqFails chan *trienodeHealRequest // Failed trienode requests to revert - bytecodeHealReqFails chan *bytecodeHealRequest // Failed bytecode requests to revert - - trienodeHealResps chan *trienodeHealResponse // Trie nodes to integrate into the database - bytecodeHealResps chan *bytecodeHealResponse // Bytecodes to integrate into the database - trienodeHealSynced uint64 // Number of state trie nodes downloaded trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk trienodeHealDups uint64 // Number of state trie nodes already processed @@ -464,26 +460,16 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { storageIdlers: make(map[string]struct{}), bytecodeIdlers: make(map[string]struct{}), - accountReqs: make(map[uint64]*accountRequest), - storageReqs: make(map[uint64]*storageRequest), - bytecodeReqs: make(map[uint64]*bytecodeRequest), - accountReqFails: make(chan *accountRequest), - storageReqFails: make(chan *storageRequest), - bytecodeReqFails: make(chan *bytecodeRequest), - accountResps: make(chan *accountResponse), - storageResps: make(chan *storageResponse), - bytecodeResps: make(chan *bytecodeResponse), + accountReqs: make(map[uint64]*accountRequest), + storageReqs: make(map[uint64]*storageRequest), + bytecodeReqs: make(map[uint64]*bytecodeRequest), trienodeHealIdlers: make(map[string]struct{}), bytecodeHealIdlers: make(map[string]struct{}), - trienodeHealReqs: make(map[uint64]*trienodeHealRequest), - bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), - trienodeHealReqFails: make(chan *trienodeHealRequest), - bytecodeHealReqFails: make(chan *bytecodeHealRequest), - trienodeHealResps: make(chan *trienodeHealResponse), - bytecodeHealResps: make(chan *bytecodeHealResponse), - stateWriter: db.NewBatch(), + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + stateWriter: db.NewBatch(), } } @@ -611,6 +597,21 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { peerDropSub := s.peerDrop.Subscribe(peerDrop) defer peerDropSub.Unsubscribe() + // Create a set of unique channels for this sync cycle. We need these to be + // ephemeral so a data race doesn't accidentally deliver something stale on + // a persistent channel across syncs (yup, this happened) + var ( + accountReqFails = make(chan *accountRequest) + storageReqFails = make(chan *storageRequest) + bytecodeReqFails = make(chan *bytecodeRequest) + accountResps = make(chan *accountResponse) + storageResps = make(chan *storageResponse) + bytecodeResps = make(chan *bytecodeResponse) + trienodeHealReqFails = make(chan *trienodeHealRequest) + bytecodeHealReqFails = make(chan *bytecodeHealRequest) + trienodeHealResps = make(chan *trienodeHealResponse) + bytecodeHealResps = make(chan *bytecodeHealResponse) + ) for { // Remove all completed tasks and terminate sync if everything's done s.cleanStorageTasks() @@ -619,14 +620,14 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { return nil } // Assign all the data retrieval tasks to any free peers - s.assignAccountTasks(cancel) - s.assignBytecodeTasks(cancel) - s.assignStorageTasks(cancel) + s.assignAccountTasks(accountResps, accountReqFails, cancel) + s.assignBytecodeTasks(bytecodeResps, bytecodeReqFails, cancel) + s.assignStorageTasks(storageResps, storageReqFails, cancel) if len(s.tasks) == 0 { // Sync phase done, run heal phase - s.assignTrienodeHealTasks(cancel) - s.assignBytecodeHealTasks(cancel) + s.assignTrienodeHealTasks(trienodeHealResps, trienodeHealReqFails, cancel) + s.assignBytecodeHealTasks(bytecodeHealResps, bytecodeHealReqFails, cancel) } // Wait for something to happen select { @@ -639,26 +640,26 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { case <-cancel: return ErrCancelled - case req := <-s.accountReqFails: + case req := <-accountReqFails: s.revertAccountRequest(req) - case req := <-s.bytecodeReqFails: + case req := <-bytecodeReqFails: s.revertBytecodeRequest(req) - case req := <-s.storageReqFails: + case req := <-storageReqFails: s.revertStorageRequest(req) - case req := <-s.trienodeHealReqFails: + case req := <-trienodeHealReqFails: s.revertTrienodeHealRequest(req) - case req := <-s.bytecodeHealReqFails: + case req := <-bytecodeHealReqFails: s.revertBytecodeHealRequest(req) - case res := <-s.accountResps: + case res := <-accountResps: s.processAccountResponse(res) - case res := <-s.bytecodeResps: + case res := <-bytecodeResps: s.processBytecodeResponse(res) - case res := <-s.storageResps: + case res := <-storageResps: s.processStorageResponse(res) - case res := <-s.trienodeHealResps: + case res := <-trienodeHealResps: s.processTrienodeHealResponse(res) - case res := <-s.bytecodeHealResps: + case res := <-bytecodeHealResps: s.processBytecodeHealResponse(res) } // Report stats if something meaningful happened @@ -801,7 +802,7 @@ func (s *Syncer) cleanStorageTasks() { // assignAccountTasks attempts to match idle peers to pending account range // retrievals. -func (s *Syncer) assignAccountTasks(cancel chan struct{}) { +func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *accountRequest, cancel chan struct{}) { s.lock.Lock() defer s.lock.Unlock() @@ -847,13 +848,15 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { } // Generate the network query and send it to the peer req := &accountRequest{ - peer: idle, - id: reqid, - cancel: cancel, - stale: make(chan struct{}), - origin: task.Next, - limit: task.Last, - task: task, + peer: idle, + id: reqid, + deliver: success, + revert: fail, + cancel: cancel, + stale: make(chan struct{}), + origin: task.Next, + limit: task.Last, + task: task, } req.timeout = time.AfterFunc(requestTimeout, func() { peer.Log().Debug("Account range request timed out", "reqid", reqid) @@ -879,7 +882,7 @@ func (s *Syncer) assignAccountTasks(cancel chan struct{}) { } // assignBytecodeTasks attempts to match idle peers to pending code retrievals. -func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { +func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan *bytecodeRequest, cancel chan struct{}) { s.lock.Lock() defer s.lock.Unlock() @@ -937,12 +940,14 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { } } req := &bytecodeRequest{ - peer: idle, - id: reqid, - cancel: cancel, - stale: make(chan struct{}), - hashes: hashes, - task: task, + peer: idle, + id: reqid, + deliver: success, + revert: fail, + cancel: cancel, + stale: make(chan struct{}), + hashes: hashes, + task: task, } req.timeout = time.AfterFunc(requestTimeout, func() { peer.Log().Debug("Bytecode request timed out", "reqid", reqid) @@ -966,7 +971,7 @@ func (s *Syncer) assignBytecodeTasks(cancel chan struct{}) { // assignStorageTasks attempts to match idle peers to pending storage range // retrievals. -func (s *Syncer) assignStorageTasks(cancel chan struct{}) { +func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *storageRequest, cancel chan struct{}) { s.lock.Lock() defer s.lock.Unlock() @@ -1059,6 +1064,8 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { req := &storageRequest{ peer: idle, id: reqid, + deliver: success, + revert: fail, cancel: cancel, stale: make(chan struct{}), accounts: accounts, @@ -1101,7 +1108,7 @@ func (s *Syncer) assignStorageTasks(cancel chan struct{}) { // assignTrienodeHealTasks attempts to match idle peers to trie node requests to // heal any trie errors caused by the snap sync's chunked retrieval model. -func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { +func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fail chan *trienodeHealRequest, cancel chan struct{}) { s.lock.Lock() defer s.lock.Unlock() @@ -1179,13 +1186,15 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { } } req := &trienodeHealRequest{ - peer: idle, - id: reqid, - cancel: cancel, - stale: make(chan struct{}), - hashes: hashes, - paths: paths, - task: s.healer, + peer: idle, + id: reqid, + deliver: success, + revert: fail, + cancel: cancel, + stale: make(chan struct{}), + hashes: hashes, + paths: paths, + task: s.healer, } req.timeout = time.AfterFunc(requestTimeout, func() { peer.Log().Debug("Trienode heal request timed out", "reqid", reqid) @@ -1209,7 +1218,7 @@ func (s *Syncer) assignTrienodeHealTasks(cancel chan struct{}) { // assignBytecodeHealTasks attempts to match idle peers to bytecode requests to // heal any trie errors caused by the snap sync's chunked retrieval model. -func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { +func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fail chan *bytecodeHealRequest, cancel chan struct{}) { s.lock.Lock() defer s.lock.Unlock() @@ -1280,12 +1289,14 @@ func (s *Syncer) assignBytecodeHealTasks(cancel chan struct{}) { } } req := &bytecodeHealRequest{ - peer: idle, - id: reqid, - cancel: cancel, - stale: make(chan struct{}), - hashes: hashes, - task: s.healer, + peer: idle, + id: reqid, + deliver: success, + revert: fail, + cancel: cancel, + stale: make(chan struct{}), + hashes: hashes, + task: s.healer, } req.timeout = time.AfterFunc(requestTimeout, func() { peer.Log().Debug("Bytecode heal request timed out", "reqid", reqid) @@ -1366,7 +1377,7 @@ func (s *Syncer) revertRequests(peer string) { // request and return all failed retrieval tasks to the scheduler for reassignment. func (s *Syncer) scheduleRevertAccountRequest(req *accountRequest) { select { - case s.accountReqFails <- req: + case req.revert <- req: // Sync event loop notified case <-req.cancel: // Sync cycle got cancelled @@ -1407,7 +1418,7 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { // and return all failed retrieval tasks to the scheduler for reassignment. func (s *Syncer) scheduleRevertBytecodeRequest(req *bytecodeRequest) { select { - case s.bytecodeReqFails <- req: + case req.revert <- req: // Sync event loop notified case <-req.cancel: // Sync cycle got cancelled @@ -1448,7 +1459,7 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { // request and return all failed retrieval tasks to the scheduler for reassignment. func (s *Syncer) scheduleRevertStorageRequest(req *storageRequest) { select { - case s.storageReqFails <- req: + case req.revert <- req: // Sync event loop notified case <-req.cancel: // Sync cycle got cancelled @@ -1493,7 +1504,7 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { // request and return all failed retrieval tasks to the scheduler for reassignment. func (s *Syncer) scheduleRevertTrienodeHealRequest(req *trienodeHealRequest) { select { - case s.trienodeHealReqFails <- req: + case req.revert <- req: // Sync event loop notified case <-req.cancel: // Sync cycle got cancelled @@ -1534,7 +1545,7 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { // request and return all failed retrieval tasks to the scheduler for reassignment. func (s *Syncer) scheduleRevertBytecodeHealRequest(req *bytecodeHealRequest) { select { - case s.bytecodeHealReqFails <- req: + case req.revert <- req: // Sync event loop notified case <-req.cancel: // Sync cycle got cancelled @@ -2147,7 +2158,7 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco cont: cont, } select { - case s.accountResps <- response: + case req.deliver <- response: case <-req.cancel: case <-req.stale: } @@ -2253,7 +2264,7 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error codes: codes, } select { - case s.bytecodeResps <- response: + case req.deliver <- response: case <-req.cancel: case <-req.stale: } @@ -2411,7 +2422,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo cont: cont, } select { - case s.storageResps <- response: + case req.deliver <- response: case <-req.cancel: case <-req.stale: } @@ -2505,7 +2516,7 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error nodes: nodes, } select { - case s.trienodeHealResps <- response: + case req.deliver <- response: case <-req.cancel: case <-req.stale: } @@ -2598,7 +2609,7 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e codes: codes, } select { - case s.bytecodeHealResps <- response: + case req.deliver <- response: case <-req.cancel: case <-req.stale: } From 3cfd0fe7a8d8e175f5d0bff1e926d5d92a7ca6ce Mon Sep 17 00:00:00 2001 From: meowsbits Date: Thu, 15 Apr 2021 17:32:16 -0500 Subject: [PATCH 273/709] core: add TestGenesisHashes and fix YoloV3 (#22559) This adds simple unit test checking if the hard-coded genesis hash values in package params match the actual genesis block hashes. --- core/genesis_test.go | 35 +++++++++++++++++++++++++++++++++++ params/config.go | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/core/genesis_test.go b/core/genesis_test.go index 3470d0aa01..44c1ef253a 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -162,3 +162,38 @@ func TestSetupGenesis(t *testing.T) { } } } + +// TestGenesisHashes checks the congruity of default genesis data to corresponding hardcoded genesis hash values. +func TestGenesisHashes(t *testing.T) { + cases := []struct { + genesis *Genesis + hash common.Hash + }{ + { + genesis: DefaultGenesisBlock(), + hash: params.MainnetGenesisHash, + }, + { + genesis: DefaultGoerliGenesisBlock(), + hash: params.GoerliGenesisHash, + }, + { + genesis: DefaultRopstenGenesisBlock(), + hash: params.RopstenGenesisHash, + }, + { + genesis: DefaultRinkebyGenesisBlock(), + hash: params.RinkebyGenesisHash, + }, + { + genesis: DefaultYoloV3GenesisBlock(), + hash: params.YoloV3GenesisHash, + }, + } + for i, c := range cases { + b := c.genesis.MustCommit(rawdb.NewMemoryDatabase()) + if got := b.Hash(); got != c.hash { + t.Errorf("case: %d, want: %s, got: %s", i, c.hash.Hex(), got.Hex()) + } + } +} diff --git a/params/config.go b/params/config.go index 143e2e2a36..9fca534c79 100644 --- a/params/config.go +++ b/params/config.go @@ -31,7 +31,7 @@ var ( RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - YoloV3GenesisHash = common.HexToHash("0x374f07cc7fa7c251fc5f36849f574b43db43600526410349efdca2bcea14101a") + YoloV3GenesisHash = common.HexToHash("0xf1f2876e8500c77afcc03228757b39477eceffccf645b734967fe3c7e16967b7") ) // TrustedCheckpoints associates each known checkpoint with the genesis hash of From fda93f643efa5dd6aafde1df7f086862a5dad9e3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 16 Apr 2021 08:27:16 +0200 Subject: [PATCH 274/709] log: fix formatting of big.Int (#22679) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * log: fix formatting of big.Int The implementation of formatLogfmtBigInt had two issues: it crashed when the number was actually large enough to hit the big integer case, and modified the big.Int while formatting it. * log: don't call FormatLogfmtInt64 for int16 * log: separate from decimals back, not front Co-authored-by: Péter Szilágyi --- log/format.go | 58 +++++++++++++++++++++++++++------------------- log/format_test.go | 20 ++++++++++++++++ 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/log/format.go b/log/format.go index 0667921528..baf8fddac0 100644 --- a/log/format.go +++ b/log/format.go @@ -359,11 +359,16 @@ func formatLogfmtValue(value interface{}, term bool) string { return strconv.FormatFloat(float64(v), floatFormat, 3, 64) case float64: return strconv.FormatFloat(v, floatFormat, 3, 64) - case int8, uint8: - return fmt.Sprintf("%d", value) - case int: - return FormatLogfmtInt64(int64(v)) + case int8: + return strconv.FormatInt(int64(v), 10) + case uint8: + return strconv.FormatInt(int64(v), 10) case int16: + return strconv.FormatInt(int64(v), 10) + case uint16: + return strconv.FormatInt(int64(v), 10) + // Larger integers get thousands separators. + case int: return FormatLogfmtInt64(int64(v)) case int32: return FormatLogfmtInt64(int64(v)) @@ -371,8 +376,6 @@ func formatLogfmtValue(value interface{}, term bool) string { return FormatLogfmtInt64(v) case uint: return FormatLogfmtUint64(uint64(v)) - case uint16: - return FormatLogfmtUint64(uint64(v)) case uint32: return FormatLogfmtUint64(uint64(v)) case uint64: @@ -384,7 +387,7 @@ func formatLogfmtValue(value interface{}, term bool) string { } } -// FormatLogfmtInt64 formats a potentially big number in a friendlier split format. +// FormatLogfmtInt64 formats n with thousand separators. func FormatLogfmtInt64(n int64) string { if n < 0 { return formatLogfmtUint64(uint64(-n), true) @@ -392,7 +395,7 @@ func FormatLogfmtInt64(n int64) string { return formatLogfmtUint64(uint64(n), false) } -// FormatLogfmtUint64 formats a potentially big number in a friendlier split format. +// FormatLogfmtUint64 formats n with thousand separators. func FormatLogfmtUint64(n uint64) string { return formatLogfmtUint64(n, false) } @@ -431,31 +434,38 @@ func formatLogfmtUint64(n uint64, neg bool) string { return string(out[i+1:]) } -var big1000 = big.NewInt(1000) - -// formatLogfmtBigInt formats a potentially gigantic number in a friendlier split -// format. +// formatLogfmtBigInt formats n with thousand separators. func formatLogfmtBigInt(n *big.Int) string { - // Most number don't need fancy handling, just downcast if n.IsUint64() { return FormatLogfmtUint64(n.Uint64()) } if n.IsInt64() { return FormatLogfmtInt64(n.Int64()) } - // Ok, huge number needs huge effort - groups := make([]string, 0, 8) // random initial size to cover most cases - for n.Cmp(big1000) >= 0 { - _, mod := n.DivMod(n, big1000, nil) - groups = append(groups, fmt.Sprintf("%03d", mod)) - } - groups = append(groups, n.String()) - last := len(groups) - 1 - for i := 0; i < len(groups)/2; i++ { - groups[i], groups[last-i] = groups[last-i], groups[i] + var ( + text = n.String() + buf = make([]byte, len(text)+len(text)/3) + comma = 0 + i = len(buf) - 1 + ) + for j := len(text) - 1; j >= 0; j, i = j-1, i-1 { + c := text[j] + + switch { + case c == '-': + buf[i] = c + case comma == 3: + buf[i] = ',' + i-- + comma = 0 + fallthrough + default: + buf[i] = c + comma++ + } } - return strings.Join(groups, ",") + return string(buf[i+1:]) } // escapeString checks if the provided string needs escaping/quoting, and diff --git a/log/format_test.go b/log/format_test.go index 348b265c9b..d7e0a95768 100644 --- a/log/format_test.go +++ b/log/format_test.go @@ -2,6 +2,7 @@ package log import ( "math" + "math/big" "math/rand" "testing" ) @@ -58,6 +59,25 @@ func TestPrettyUint64(t *testing.T) { } } +func TestPrettyBigInt(t *testing.T) { + tests := []struct { + int string + s string + }{ + {"111222333444555678999", "111,222,333,444,555,678,999"}, + {"-111222333444555678999", "-111,222,333,444,555,678,999"}, + {"11122233344455567899900", "11,122,233,344,455,567,899,900"}, + {"-11122233344455567899900", "-11,122,233,344,455,567,899,900"}, + } + + for _, tt := range tests { + v, _ := new(big.Int).SetString(tt.int, 10) + if have := formatLogfmtBigInt(v); have != tt.s { + t.Errorf("invalid output %s, want %s", have, tt.s) + } + } +} + var sink string func BenchmarkPrettyInt64Logfmt(b *testing.B) { From 65689e7fcea8f3a813aea659e1b49f9052883ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 16 Apr 2021 09:52:33 +0200 Subject: [PATCH 275/709] les/vflux/server: fix priority cornercase causing fuzzer timeout (#22650) * les/vflux/server: fix estimatePriority corner case * les/vflux/server: simplify inactiveAllowance == 0 case --- les/vflux/server/balance.go | 10 +++++++--- les/vflux/server/balance_test.go | 6 +++--- les/vflux/server/clientpool.go | 9 +-------- les/vflux/server/clientpool_test.go | 11 ++++++++--- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index 01e645a16a..2bc1ddd189 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -358,11 +358,15 @@ func (n *nodeBalance) estimatePriority(capacity uint64, addBalance int64, future if bias > 0 { b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) } - // Note: we subtract one from the estimated priority in order to ensure that biased - // estimates are always lower than actual priorities, even if the bias is very small. + pri := n.balanceToPriority(now, b, capacity) + // Ensure that biased estimates are always lower than actual priorities, even if + // the bias is very small. // This ensures that two nodes will not ping-pong update signals forever if both of // them have zero estimated priority drop in the projected future. - pri := n.balanceToPriority(now, b, capacity) - 1 + current := n.balanceToPriority(now, n.balance, capacity) + if pri >= current { + pri = current - 1 + } if update { n.addCallback(balanceCallbackUpdate, pri, n.signalPriorityUpdate) } diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 66f0d1f301..5af89c18ab 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -283,7 +283,7 @@ func TestEstimatedPriority(t *testing.T) { {time.Second, 3 * time.Second, 1000000000, 48}, // All positive balance is used up - {time.Second * 55, 0, 0, 0}, + {time.Second * 55, 0, 0, -1}, // 1 minute estimated time cost, 4/58 * 10^9 estimated request cost per sec. {0, time.Minute, 0, -int64(time.Minute) - int64(time.Second)*120/29}, @@ -292,8 +292,8 @@ func TestEstimatedPriority(t *testing.T) { b.clock.Run(i.runTime) node.RequestServed(i.reqCost) priority := node.estimatePriority(1000000000, 0, i.futureTime, 0, false) - if priority != i.priority-1 { - t.Fatalf("Estimated priority mismatch, want %v, got %v", i.priority-1, priority) + if priority != i.priority { + t.Fatalf("Estimated priority mismatch, want %v, got %v", i.priority, priority) } } } diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go index 2e5fdd0ee7..079d511704 100644 --- a/les/vflux/server/clientpool.go +++ b/les/vflux/server/clientpool.go @@ -103,14 +103,7 @@ func NewClientPool(balanceDb ethdb.KeyValueStore, minCap uint64, connectedBias t if c, ok := ns.GetField(node, setup.clientField).(clientPeer); ok { timeout = c.InactiveAllowance() } - if timeout > 0 { - ns.AddTimeout(node, setup.inactiveFlag, timeout) - } else { - // Note: if capacity is immediately available then priorityPool will set the active - // flag simultaneously with removing the inactive flag and therefore this will not - // initiate disconnection - ns.SetStateSub(node, nodestate.Flags{}, setup.inactiveFlag, 0) - } + ns.AddTimeout(node, setup.inactiveFlag, timeout) } if oldState.Equals(setup.inactiveFlag) && newState.Equals(setup.inactiveFlag.Or(setup.priorityFlag)) { ns.SetStateSub(node, setup.inactiveFlag, nodestate.Flags{}, 0) // priority gained; remove timeout diff --git a/les/vflux/server/clientpool_test.go b/les/vflux/server/clientpool_test.go index 9503121697..0953e9c800 100644 --- a/les/vflux/server/clientpool_test.go +++ b/les/vflux/server/clientpool_test.go @@ -326,12 +326,13 @@ func TestPaidClientKickedOut(t *testing.T) { if cap := connect(pool, newPoolTestPeer(11, kickedCh)); cap == 0 { t.Fatalf("Free client should be accepted") } + clock.Run(0) select { case id := <-kickedCh: if id != 0 { t.Fatalf("Kicked client mismatch, want %v, got %v", 0, id) } - case <-time.NewTimer(time.Second).C: + default: t.Fatalf("timeout") } } @@ -399,23 +400,27 @@ func TestFreeClientKickedOut(t *testing.T) { if cap := connect(pool, newPoolTestPeer(10, kicked)); cap != 0 { t.Fatalf("New free client should be rejected") } + clock.Run(0) select { case <-kicked: - case <-time.NewTimer(time.Second).C: + default: t.Fatalf("timeout") } disconnect(pool, newPoolTestPeer(10, kicked)) clock.Run(5 * time.Minute) for i := 0; i < 10; i++ { connect(pool, newPoolTestPeer(i+10, kicked)) + } + clock.Run(0) + for i := 0; i < 10; i++ { select { case id := <-kicked: if id >= 10 { t.Fatalf("Old client should be kicked, now got: %d", id) } - case <-time.NewTimer(time.Second).C: + default: t.Fatalf("timeout") } } From 4f3ba6742fbf1cc585048fcc45072523c6de173d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Apr 2021 14:21:01 +0200 Subject: [PATCH 276/709] trie: make stacktrie not mutate input values (#22673) The stacktrie is a bit un-untuitive, API-wise: since it mutates input values. Such behaviour is dangerous, and easy to get wrong if the calling code 'forgets' this quirk. The behaviour is fixed by this PR, so that the input values are not modified by the stacktrie. Note: just as with the Trie, the stacktrie still references the live input objects, so it's still _not_ safe to mutate the values form the callsite. --- trie/stacktrie.go | 16 +++++-------- trie/stacktrie_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index a198eb204b..c34ac6c78c 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -346,8 +346,7 @@ func (st *StackTrie) hash() { panic(err) } case emptyNode: - st.val = st.val[:0] - st.val = append(st.val, emptyRoot[:]...) + st.val = emptyRoot.Bytes() st.key = st.key[:0] st.nodeType = hashedNode return @@ -357,17 +356,12 @@ func (st *StackTrie) hash() { st.key = st.key[:0] st.nodeType = hashedNode if len(h.tmp) < 32 { - st.val = st.val[:0] - st.val = append(st.val, h.tmp...) + st.val = common.CopyBytes(h.tmp) return } - // Going to write the hash to the 'val'. Need to ensure it's properly sized first - // Typically, 'branchNode's will have no 'val', and require this allocation - if required := 32 - len(st.val); required > 0 { - buf := make([]byte, required) - st.val = append(st.val, buf...) - } - st.val = st.val[:32] + // Write the hash to the 'val'. We allocate a new val here to not mutate + // input values + st.val = make([]byte, 32) h.sha.Reset() h.sha.Write(h.tmp) h.sha.Read(st.val) diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 29706f2e9d..50a4eda323 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -1,9 +1,12 @@ package trie import ( + "bytes" + "math/big" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb/memorydb" ) @@ -119,3 +122,51 @@ func TestUpdateVariableKeys(t *testing.T) { t.Fatalf("error %x != %x", st.Hash(), nt.Hash()) } } + +// TestStacktrieNotModifyValues checks that inserting blobs of data into the +// stacktrie does not mutate the blobs +func TestStacktrieNotModifyValues(t *testing.T) { + st := NewStackTrie(nil) + { // Test a very small trie + // Give it the value as a slice with large backing alloc, + // so if the stacktrie tries to append, it won't have to realloc + value := make([]byte, 1, 100) + value[0] = 0x2 + want := common.CopyBytes(value) + st.TryUpdate([]byte{0x01}, value) + st.Hash() + if have := value; !bytes.Equal(have, want) { + t.Fatalf("tiny trie: have %#x want %#x", have, want) + } + st = NewStackTrie(nil) + } + // Test with a larger trie + keyB := big.NewInt(1) + keyDelta := big.NewInt(1) + var vals [][]byte + getValue := func(i int) []byte { + if i%2 == 0 { // large + return crypto.Keccak256(big.NewInt(int64(i)).Bytes()) + } else { //small + return big.NewInt(int64(i)).Bytes() + } + } + + for i := 0; i < 1000; i++ { + key := common.BigToHash(keyB) + value := getValue(i) + st.TryUpdate(key.Bytes(), value) + vals = append(vals, value) + keyB = keyB.Add(keyB, keyDelta) + keyDelta.Add(keyDelta, common.Big1) + } + st.Hash() + for i := 0; i < 1000; i++ { + want := getValue(i) + + have := vals[i] + if !bytes.Equal(have, want) { + t.Fatalf("item %d, have %#x want %#x", i, have, want) + } + } +} From 09d44e9925a5a3f983ae1861f7ecdf69f49ab888 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Apr 2021 14:58:23 +0200 Subject: [PATCH 277/709] core/state/snapshot: avoid copybytes for stacktrie --- core/state/snapshot/generate.go | 2 +- trie/trie_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 98c8d42a1a..ed431fcb3d 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -317,7 +317,7 @@ func (dl *diskLayer) proveRange(stats *generatorStats, root common.Hash, prefix if origin == nil && !diskMore { stackTr := trie.NewStackTrie(nil) for i, key := range keys { - stackTr.TryUpdate(key, common.CopyBytes(vals[i])) + stackTr.TryUpdate(key, vals[i]) } if gotRoot := stackTr.Hash(); gotRoot != root { return &proofResult{ diff --git a/trie/trie_test.go b/trie/trie_test.go index d6930fdee7..492b423c2f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -830,8 +830,8 @@ func TestCommitSequenceStackTrie(t *testing.T) { val = make([]byte, 1+prng.Intn(1024)) } prng.Read(val) - trie.TryUpdate(key, common.CopyBytes(val)) - stTrie.TryUpdate(key, common.CopyBytes(val)) + trie.TryUpdate(key, val) + stTrie.TryUpdate(key, val) } // Flush trie -> database root, _ := trie.Commit(nil) From f79cce5de98332e6d54eb298fce85c69d1082ee2 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 16 Apr 2021 21:29:22 +0200 Subject: [PATCH 278/709] eth/catalyst: add catalyst API prototype (#22641) This change adds the --catalyst flag, enabling an RPC API for eth2 integration. In this initial version, catalyst mode also disables all peer-to-peer networking. Co-authored-by: Mikhail Kalinin Co-authored-by: Felix Lange --- cmd/geth/config.go | 13 +- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 18 +- consensus/ethash/consensus.go | 2 + core/blockchain.go | 16 ++ eth/backend.go | 1 + eth/catalyst/api.go | 302 ++++++++++++++++++++++++++++++++ eth/catalyst/api_test.go | 229 ++++++++++++++++++++++++ eth/catalyst/api_types.go | 70 ++++++++ eth/catalyst/gen_blockparams.go | 46 +++++ eth/catalyst/gen_ed.go | 117 +++++++++++++ params/config.go | 19 +- 13 files changed, 823 insertions(+), 12 deletions(-) create mode 100644 eth/catalyst/api.go create mode 100644 eth/catalyst/api_test.go create mode 100644 eth/catalyst/api_types.go create mode 100644 eth/catalyst/gen_blockparams.go create mode 100644 eth/catalyst/gen_ed.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 6fc75363c6..c867877ee6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -28,6 +28,7 @@ import ( "gopkg.in/urfave/cli.v1" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/metrics" @@ -143,7 +144,17 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if ctx.GlobalIsSet(utils.OverrideBerlinFlag.Name) { cfg.Eth.OverrideBerlin = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideBerlinFlag.Name)) } - backend := utils.RegisterEthService(stack, &cfg.Eth) + backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + + // Configure catalyst. + if ctx.GlobalBool(utils.CatalystFlag.Name) { + if eth == nil { + utils.Fatalf("Catalyst does not work in light client mode.") + } + if err := catalyst.Register(stack, eth); err != nil { + utils.Fatalf("%v", err) + } + } // Configure GraphQL if requested if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 12a5ae9bfc..78e65161da 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -151,6 +151,7 @@ var ( utils.EVMInterpreterFlag, utils.MinerNotifyFullFlag, configFileFlag, + utils.CatalystFlag, } rpcFlags = []cli.Flag{ diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index d7f8fd7ab9..980794db73 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -235,6 +235,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.SnapshotFlag, utils.BloomFilterSizeFlag, cli.HelpFlag, + utils.CatalystFlag, }, }, } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d631b8e332..59cf32c983 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -755,6 +755,11 @@ var ( Usage: "External EVM configuration (default = built-in interpreter)", Value: "", } + + CatalystFlag = cli.BoolFlag{ + Name: "catalyst", + Usage: "Catalyst mode (eth2 integration testing)", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1186,10 +1191,11 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.NetRestrict = list } - if ctx.GlobalBool(DeveloperFlag.Name) { + if ctx.GlobalBool(DeveloperFlag.Name) || ctx.GlobalBool(CatalystFlag.Name) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 - cfg.ListenAddr = ":0" + cfg.ListenAddr = "" + cfg.NoDial = true cfg.NoDiscovery = true cfg.DiscoveryV5 = false } @@ -1693,14 +1699,16 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) { } // RegisterEthService adds an Ethereum client to the stack. -func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) ethapi.Backend { +// The second return value is the full node instance, which may be nil if the +// node is running as a light client. +func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend, *eth.Ethereum) { if cfg.SyncMode == downloader.LightSync { backend, err := les.New(stack, cfg) if err != nil { Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - return backend.ApiBackend + return backend.ApiBackend, nil } backend, err := eth.New(stack, cfg) if err != nil { @@ -1713,7 +1721,7 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) ethapi.Backend } } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) - return backend.APIBackend + return backend.APIBackend, backend } // RegisterEthStatsService configures the Ethereum Stats daemon and adds it to diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 011a5688ef..e23bd824af 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -315,6 +315,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainHeaderReader, time uin func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { + case config.IsCatalyst(next): + return big.NewInt(1) case config.IsMuirGlacier(next): return calcDifficultyEip2384(time, parent) case config.IsConstantinople(next): diff --git a/core/blockchain.go b/core/blockchain.go index 8c3d940a69..dfbc5ad0bb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1693,6 +1693,22 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return n, err } +// InsertChainWithoutSealVerification works exactly the same +// except for seal verification, seal verification is omitted +func (bc *BlockChain) InsertChainWithoutSealVerification(block *types.Block) (int, error) { + bc.blockProcFeed.Send(true) + defer bc.blockProcFeed.Send(false) + + // Pre-checks passed, start the full block imports + bc.wg.Add(1) + bc.chainmu.Lock() + n, err := bc.insertChain(types.Blocks([]*types.Block{block}), false) + bc.chainmu.Unlock() + bc.wg.Done() + + return n, err +} + // insertChain is the internal implementation of InsertChain, which assumes that // 1) chains are contiguous, and 2) The chain mutex is held. // diff --git a/eth/backend.go b/eth/backend.go index 9cf8b85663..4c7374612e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -223,6 +223,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { }); err != nil { return nil, err } + eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go new file mode 100644 index 0000000000..d6ea691d02 --- /dev/null +++ b/eth/catalyst/api.go @@ -0,0 +1,302 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package catalyst implements the temporary eth1/eth2 RPC integration. +package catalyst + +import ( + "errors" + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + chainParams "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" +) + +// Register adds catalyst APIs to the node. +func Register(stack *node.Node, backend *eth.Ethereum) error { + chainconfig := backend.BlockChain().Config() + if chainconfig.CatalystBlock == nil { + return errors.New("catalystBlock is not set in genesis config") + } else if chainconfig.CatalystBlock.Sign() != 0 { + return errors.New("catalystBlock of genesis config must be zero") + } + + log.Warn("Catalyst mode enabled") + stack.RegisterAPIs([]rpc.API{ + { + Namespace: "consensus", + Version: "1.0", + Service: newConsensusAPI(backend), + Public: true, + }, + }) + return nil +} + +type consensusAPI struct { + eth *eth.Ethereum +} + +func newConsensusAPI(eth *eth.Ethereum) *consensusAPI { + return &consensusAPI{eth: eth} +} + +// blockExecutionEnv gathers all the data required to execute +// a block, either when assembling it or when inserting it. +type blockExecutionEnv struct { + chain *core.BlockChain + state *state.StateDB + tcount int + gasPool *core.GasPool + + header *types.Header + txs []*types.Transaction + receipts []*types.Receipt +} + +func (env *blockExecutionEnv) commitTransaction(tx *types.Transaction, coinbase common.Address) error { + vmconfig := *env.chain.GetVMConfig() + receipt, err := core.ApplyTransaction(env.chain.Config(), env.chain, &coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vmconfig) + if err != nil { + return err + } + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) + return nil +} + +func (api *consensusAPI) makeEnv(parent *types.Block, header *types.Header) (*blockExecutionEnv, error) { + state, err := api.eth.BlockChain().StateAt(parent.Root()) + if err != nil { + return nil, err + } + env := &blockExecutionEnv{ + chain: api.eth.BlockChain(), + state: state, + header: header, + gasPool: new(core.GasPool).AddGas(header.GasLimit), + } + return env, nil +} + +// AssembleBlock creates a new block, inserts it into the chain, and returns the "execution +// data" required for eth2 clients to process the new block. +func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableData, error) { + log.Info("Producing block", "parentHash", params.ParentHash) + + bc := api.eth.BlockChain() + parent := bc.GetBlockByHash(params.ParentHash) + pool := api.eth.TxPool() + + if parent.Time() >= params.Timestamp { + return nil, fmt.Errorf("child timestamp lower than parent's: %d >= %d", parent.Time(), params.Timestamp) + } + if now := uint64(time.Now().Unix()); params.Timestamp > now+1 { + wait := time.Duration(params.Timestamp-now) * time.Second + log.Info("Producing block too far in the future", "wait", common.PrettyDuration(wait)) + time.Sleep(wait) + } + + pending, err := pool.Pending() + if err != nil { + return nil, err + } + + coinbase, err := api.eth.Etherbase() + if err != nil { + return nil, err + } + num := parent.Number() + header := &types.Header{ + ParentHash: parent.Hash(), + Number: num.Add(num, common.Big1), + Coinbase: coinbase, + GasLimit: parent.GasLimit(), // Keep the gas limit constant in this prototype + Extra: []byte{}, + Time: params.Timestamp, + } + err = api.eth.Engine().Prepare(bc, header) + if err != nil { + return nil, err + } + + env, err := api.makeEnv(parent, header) + if err != nil { + return nil, err + } + + var ( + signer = types.MakeSigner(bc.Config(), header.Number) + txHeap = types.NewTransactionsByPriceAndNonce(signer, pending) + transactions []*types.Transaction + ) + for { + if env.gasPool.Gas() < chainParams.TxGas { + log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", chainParams.TxGas) + break + } + tx := txHeap.Peek() + if tx == nil { + break + } + + // The sender is only for logging purposes, and it doesn't really matter if it's correct. + from, _ := types.Sender(signer, tx) + + // Execute the transaction + env.state.Prepare(tx.Hash(), common.Hash{}, env.tcount) + err = env.commitTransaction(tx, coinbase) + switch err { + case core.ErrGasLimitReached: + // Pop the current out-of-gas transaction without shifting in the next from the account + log.Trace("Gas limit exceeded for current block", "sender", from) + txHeap.Pop() + + case core.ErrNonceTooLow: + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + txHeap.Shift() + + case core.ErrNonceTooHigh: + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with high nonce", "sender", from, "nonce", tx.Nonce()) + txHeap.Pop() + + case nil: + // Everything ok, collect the logs and shift in the next transaction from the same account + env.tcount++ + txHeap.Shift() + transactions = append(transactions, tx) + + default: + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + txHeap.Shift() + } + } + + // Create the block. + block, err := api.eth.Engine().FinalizeAndAssemble(bc, header, env.state, transactions, nil /* uncles */, env.receipts) + if err != nil { + return nil, err + } + return &executableData{ + BlockHash: block.Hash(), + ParentHash: block.ParentHash(), + Miner: block.Coinbase(), + StateRoot: block.Root(), + Number: block.NumberU64(), + GasLimit: block.GasLimit(), + GasUsed: block.GasUsed(), + Timestamp: block.Time(), + ReceiptRoot: block.ReceiptHash(), + LogsBloom: block.Bloom().Bytes(), + Transactions: encodeTransactions(block.Transactions()), + }, nil +} + +func encodeTransactions(txs []*types.Transaction) [][]byte { + var enc = make([][]byte, len(txs)) + for i, tx := range txs { + enc[i], _ = tx.MarshalBinary() + } + return enc +} + +func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { + var txs = make([]*types.Transaction, len(enc)) + for i, encTx := range enc { + var tx types.Transaction + if err := tx.UnmarshalBinary(encTx); err != nil { + return nil, fmt.Errorf("invalid transaction %d: %v", i, err) + } + txs[i] = &tx + } + return txs, nil +} + +func insertBlockParamsToBlock(params executableData) (*types.Block, error) { + txs, err := decodeTransactions(params.Transactions) + if err != nil { + return nil, err + } + + number := big.NewInt(0) + number.SetUint64(params.Number) + header := &types.Header{ + ParentHash: params.ParentHash, + UncleHash: types.EmptyUncleHash, + Coinbase: params.Miner, + Root: params.StateRoot, + TxHash: types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)), + ReceiptHash: params.ReceiptRoot, + Bloom: types.BytesToBloom(params.LogsBloom), + Difficulty: big.NewInt(1), + Number: number, + GasLimit: params.GasLimit, + GasUsed: params.GasUsed, + Time: params.Timestamp, + } + block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */) + return block, nil +} + +// NewBlock creates an Eth1 block, inserts it in the chain, and either returns true, +// or false + an error. This is a bit redundant for go, but simplifies things on the +// eth2 side. +func (api *consensusAPI) NewBlock(params executableData) (*newBlockResponse, error) { + parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) + if parent == nil { + return &newBlockResponse{false}, fmt.Errorf("could not find parent %x", params.ParentHash) + } + block, err := insertBlockParamsToBlock(params) + if err != nil { + return nil, err + } + + _, err = api.eth.BlockChain().InsertChainWithoutSealVerification(block) + return &newBlockResponse{err == nil}, err +} + +// Used in tests to add a the list of transactions from a block to the tx pool. +func (api *consensusAPI) addBlockTxs(block *types.Block) error { + for _, tx := range block.Transactions() { + api.eth.TxPool().AddLocal(tx) + } + return nil +} + +// FinalizeBlock is called to mark a block as synchronized, so +// that data that is no longer needed can be removed. +func (api *consensusAPI) FinalizeBlock(blockHash common.Hash) (*genericResponse, error) { + return &genericResponse{true}, nil +} + +// SetHead is called to perform a force choice. +func (api *consensusAPI) SetHead(newHead common.Hash) (*genericResponse, error) { + return &genericResponse{true}, nil +} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go new file mode 100644 index 0000000000..456b6867bd --- /dev/null +++ b/eth/catalyst/api_test.go @@ -0,0 +1,229 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) + + testBalance = big.NewInt(2e10) +) + +func generateTestChain() (*core.Genesis, []*types.Block) { + db := rawdb.NewMemoryDatabase() + config := params.AllEthashProtocolChanges + genesis := &core.Genesis{ + Config: config, + Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, + ExtraData: []byte("test genesis"), + Timestamp: 9000, + } + generate := func(i int, g *core.BlockGen) { + g.OffsetTime(5) + g.SetExtra([]byte("test")) + } + gblock := genesis.ToBlock(db) + engine := ethash.NewFaker() + blocks, _ := core.GenerateChain(config, gblock, engine, db, 10, generate) + blocks = append([]*types.Block{gblock}, blocks...) + return genesis, blocks +} + +func generateTestChainWithFork(n int, fork int) (*core.Genesis, []*types.Block, []*types.Block) { + if fork >= n { + fork = n - 1 + } + db := rawdb.NewMemoryDatabase() + //nolint:composites + config := ¶ms.ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, big.NewInt(0), new(params.EthashConfig), nil} + genesis := &core.Genesis{ + Config: config, + Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, + ExtraData: []byte("test genesis"), + Timestamp: 9000, + } + generate := func(i int, g *core.BlockGen) { + g.OffsetTime(5) + g.SetExtra([]byte("test")) + } + generateFork := func(i int, g *core.BlockGen) { + g.OffsetTime(5) + g.SetExtra([]byte("testF")) + } + gblock := genesis.ToBlock(db) + engine := ethash.NewFaker() + blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) + blocks = append([]*types.Block{gblock}, blocks...) + forkedBlocks, _ := core.GenerateChain(config, blocks[fork], engine, db, n-fork, generateFork) + return genesis, blocks, forkedBlocks +} + +func TestEth2AssembleBlock(t *testing.T) { + genesis, blocks := generateTestChain() + n, ethservice := startEthService(t, genesis, blocks[1:9]) + defer n.Close() + + api := newConsensusAPI(ethservice) + signer := types.NewEIP155Signer(ethservice.BlockChain().Config().ChainID) + tx, err := types.SignTx(types.NewTransaction(0, blocks[8].Coinbase(), big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + if err != nil { + t.Fatalf("error signing transaction, err=%v", err) + } + ethservice.TxPool().AddLocal(tx) + blockParams := assembleBlockParams{ + ParentHash: blocks[8].ParentHash(), + Timestamp: blocks[8].Time(), + } + execData, err := api.AssembleBlock(blockParams) + + if err != nil { + t.Fatalf("error producing block, err=%v", err) + } + + if len(execData.Transactions) != 1 { + t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) + } +} + +func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { + genesis, blocks := generateTestChain() + n, ethservice := startEthService(t, genesis, blocks[1:9]) + defer n.Close() + + api := newConsensusAPI(ethservice) + + // Put the 10th block's tx in the pool and produce a new block + api.addBlockTxs(blocks[9]) + blockParams := assembleBlockParams{ + ParentHash: blocks[9].ParentHash(), + Timestamp: blocks[9].Time(), + } + execData, err := api.AssembleBlock(blockParams) + if err != nil { + t.Fatalf("error producing block, err=%v", err) + } + + if len(execData.Transactions) != blocks[9].Transactions().Len() { + t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) + } +} + +func TestEth2NewBlock(t *testing.T) { + genesis, blocks, forkedBlocks := generateTestChainWithFork(10, 4) + n, ethservice := startEthService(t, genesis, blocks[1:5]) + defer n.Close() + + api := newConsensusAPI(ethservice) + for i := 5; i < 10; i++ { + p := executableData{ + ParentHash: ethservice.BlockChain().CurrentBlock().Hash(), + Miner: blocks[i].Coinbase(), + StateRoot: blocks[i].Root(), + GasLimit: blocks[i].GasLimit(), + GasUsed: blocks[i].GasUsed(), + Transactions: encodeTransactions(blocks[i].Transactions()), + ReceiptRoot: blocks[i].ReceiptHash(), + LogsBloom: blocks[i].Bloom().Bytes(), + BlockHash: blocks[i].Hash(), + Timestamp: blocks[i].Time(), + Number: uint64(i), + } + success, err := api.NewBlock(p) + if err != nil || !success.Valid { + t.Fatalf("Failed to insert block: %v", err) + } + } + + exp := ethservice.BlockChain().CurrentBlock().Hash() + + // Introduce the fork point. + lastBlockNum := blocks[4].Number() + lastBlock := blocks[4] + for i := 0; i < 4; i++ { + lastBlockNum.Add(lastBlockNum, big.NewInt(1)) + p := executableData{ + ParentHash: lastBlock.Hash(), + Miner: forkedBlocks[i].Coinbase(), + StateRoot: forkedBlocks[i].Root(), + Number: lastBlockNum.Uint64(), + GasLimit: forkedBlocks[i].GasLimit(), + GasUsed: forkedBlocks[i].GasUsed(), + Transactions: encodeTransactions(blocks[i].Transactions()), + ReceiptRoot: forkedBlocks[i].ReceiptHash(), + LogsBloom: forkedBlocks[i].Bloom().Bytes(), + BlockHash: forkedBlocks[i].Hash(), + Timestamp: forkedBlocks[i].Time(), + } + success, err := api.NewBlock(p) + if err != nil || !success.Valid { + t.Fatalf("Failed to insert forked block #%d: %v", i, err) + } + lastBlock, err = insertBlockParamsToBlock(p) + if err != nil { + t.Fatal(err) + } + } + + if ethservice.BlockChain().CurrentBlock().Hash() != exp { + t.Fatalf("Wrong head after inserting fork %x != %x", exp, ethservice.BlockChain().CurrentBlock().Hash()) + } +} + +// startEthService creates a full node instance for testing. +func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { + t.Helper() + + n, err := node.New(&node.Config{}) + if err != nil { + t.Fatal("can't create node:", err) + } + + ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}} + ethservice, err := eth.New(n, ethcfg) + if err != nil { + t.Fatal("can't create eth service:", err) + } + if err := n.Start(); err != nil { + t.Fatal("can't start node:", err) + } + if _, err := ethservice.BlockChain().InsertChain(blocks); err != nil { + n.Close() + t.Fatal("can't import test blocks:", err) + } + ethservice.SetEtherbase(testAddr) + + return n, ethservice +} diff --git a/eth/catalyst/api_types.go b/eth/catalyst/api_types.go new file mode 100644 index 0000000000..d5d351a991 --- /dev/null +++ b/eth/catalyst/api_types.go @@ -0,0 +1,70 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +//go:generate go run github.com/fjl/gencodec -type assembleBlockParams -field-override assembleBlockParamsMarshaling -out gen_blockparams.go + +// Structure described at https://hackmd.io/T9x2mMA4S7us8tJwEB3FDQ +type assembleBlockParams struct { + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` +} + +// JSON type overrides for assembleBlockParams. +type assembleBlockParamsMarshaling struct { + Timestamp hexutil.Uint64 +} + +//go:generate go run github.com/fjl/gencodec -type executableData -field-override executableDataMarshaling -out gen_ed.go + +// Structure described at https://notes.ethereum.org/@n0ble/rayonism-the-merge-spec#Parameters1 +type executableData struct { + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Miner common.Address `json:"miner" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + Number uint64 `json:"number" gencodec:"required"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` + ReceiptRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom []byte `json:"logsBloom" gencodec:"required"` + Transactions [][]byte `json:"transactions" gencodec:"required"` +} + +// JSON type overrides for executableData. +type executableDataMarshaling struct { + Number hexutil.Uint64 + GasLimit hexutil.Uint64 + GasUsed hexutil.Uint64 + Timestamp hexutil.Uint64 + LogsBloom hexutil.Bytes + Transactions []hexutil.Bytes +} + +type newBlockResponse struct { + Valid bool `json:"valid"` +} + +type genericResponse struct { + Success bool `json:"success"` +} diff --git a/eth/catalyst/gen_blockparams.go b/eth/catalyst/gen_blockparams.go new file mode 100644 index 0000000000..a9a08ec3a8 --- /dev/null +++ b/eth/catalyst/gen_blockparams.go @@ -0,0 +1,46 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package catalyst + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*assembleBlockParamsMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (a assembleBlockParams) MarshalJSON() ([]byte, error) { + type assembleBlockParams struct { + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + } + var enc assembleBlockParams + enc.ParentHash = a.ParentHash + enc.Timestamp = hexutil.Uint64(a.Timestamp) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (a *assembleBlockParams) UnmarshalJSON(input []byte) error { + type assembleBlockParams struct { + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + } + var dec assembleBlockParams + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.ParentHash == nil { + return errors.New("missing required field 'parentHash' for assembleBlockParams") + } + a.ParentHash = *dec.ParentHash + if dec.Timestamp == nil { + return errors.New("missing required field 'timestamp' for assembleBlockParams") + } + a.Timestamp = uint64(*dec.Timestamp) + return nil +} diff --git a/eth/catalyst/gen_ed.go b/eth/catalyst/gen_ed.go new file mode 100644 index 0000000000..4c2e4c8ead --- /dev/null +++ b/eth/catalyst/gen_ed.go @@ -0,0 +1,117 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package catalyst + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*executableDataMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (e executableData) MarshalJSON() ([]byte, error) { + type executableData struct { + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Miner common.Address `json:"miner" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + Number hexutil.Uint64 `json:"number" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ReceiptRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + } + var enc executableData + enc.BlockHash = e.BlockHash + enc.ParentHash = e.ParentHash + enc.Miner = e.Miner + enc.StateRoot = e.StateRoot + enc.Number = hexutil.Uint64(e.Number) + enc.GasLimit = hexutil.Uint64(e.GasLimit) + enc.GasUsed = hexutil.Uint64(e.GasUsed) + enc.Timestamp = hexutil.Uint64(e.Timestamp) + enc.ReceiptRoot = e.ReceiptRoot + enc.LogsBloom = e.LogsBloom + if e.Transactions != nil { + enc.Transactions = make([]hexutil.Bytes, len(e.Transactions)) + for k, v := range e.Transactions { + enc.Transactions[k] = v + } + } + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (e *executableData) UnmarshalJSON(input []byte) error { + type executableData struct { + BlockHash *common.Hash `json:"blockHash" gencodec:"required"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + Miner *common.Address `json:"miner" gencodec:"required"` + StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` + Number *hexutil.Uint64 `json:"number" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ReceiptRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + } + var dec executableData + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.BlockHash == nil { + return errors.New("missing required field 'blockHash' for executableData") + } + e.BlockHash = *dec.BlockHash + if dec.ParentHash == nil { + return errors.New("missing required field 'parentHash' for executableData") + } + e.ParentHash = *dec.ParentHash + if dec.Miner == nil { + return errors.New("missing required field 'miner' for executableData") + } + e.Miner = *dec.Miner + if dec.StateRoot == nil { + return errors.New("missing required field 'stateRoot' for executableData") + } + e.StateRoot = *dec.StateRoot + if dec.Number == nil { + return errors.New("missing required field 'number' for executableData") + } + e.Number = uint64(*dec.Number) + if dec.GasLimit == nil { + return errors.New("missing required field 'gasLimit' for executableData") + } + e.GasLimit = uint64(*dec.GasLimit) + if dec.GasUsed == nil { + return errors.New("missing required field 'gasUsed' for executableData") + } + e.GasUsed = uint64(*dec.GasUsed) + if dec.Timestamp == nil { + return errors.New("missing required field 'timestamp' for executableData") + } + e.Timestamp = uint64(*dec.Timestamp) + if dec.ReceiptRoot == nil { + return errors.New("missing required field 'receiptsRoot' for executableData") + } + e.ReceiptRoot = *dec.ReceiptRoot + if dec.LogsBloom == nil { + return errors.New("missing required field 'logsBloom' for executableData") + } + e.LogsBloom = *dec.LogsBloom + if dec.Transactions == nil { + return errors.New("missing required field 'transactions' for executableData") + } + e.Transactions = make([][]byte, len(dec.Transactions)) + for k, v := range dec.Transactions { + e.Transactions[k] = v + } + return nil +} diff --git a/params/config.go b/params/config.go index 9fca534c79..f4e2f5ea67 100644 --- a/params/config.go +++ b/params/config.go @@ -244,16 +244,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -326,8 +326,9 @@ type ChainConfig struct { MuirGlacierBlock *big.Int `json:"muirGlacierBlock,omitempty"` // Eip-2384 (bomb delay) switch block (nil = no fork, 0 = already activated) BerlinBlock *big.Int `json:"berlinBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) - YoloV3Block *big.Int `json:"yoloV3Block,omitempty"` // YOLO v3: Gas repricings TODO @holiman add EIP references - EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) + YoloV3Block *big.Int `json:"yoloV3Block,omitempty"` // YOLO v3: Gas repricings TODO @holiman add EIP references + EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) + CatalystBlock *big.Int `json:"catalystBlock,omitempty"` // Catalyst switch block (nil = no fork, 0 = already on catalyst) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -440,6 +441,11 @@ func (c *ChainConfig) IsBerlin(num *big.Int) bool { return isForked(c.BerlinBlock, num) || isForked(c.YoloV3Block, num) } +// IsCatalyst returns whether num is either equal to the Merge fork block or greater. +func (c *ChainConfig) IsCatalyst(num *big.Int) bool { + return isForked(c.CatalystBlock, num) +} + // IsEWASM returns whether num represents a block number after the EWASM fork func (c *ChainConfig) IsEWASM(num *big.Int) bool { return isForked(c.EWASMBlock, num) @@ -623,7 +629,7 @@ type Rules struct { ChainID *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool - IsBerlin bool + IsBerlin, IsCatalyst bool } // Rules ensures c's ChainID is not nil. @@ -643,5 +649,6 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { IsPetersburg: c.IsPetersburg(num), IsIstanbul: c.IsIstanbul(num), IsBerlin: c.IsBerlin(num), + IsCatalyst: c.IsCatalyst(num), } } From 424656519a0bfc0841b112c935279c787cecd904 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 19 Apr 2021 14:54:38 +0200 Subject: [PATCH 279/709] cmd/devp2p: add support for -limit option in nodeset filter command (#22694) The new -limit option makes the filter operate on top N nodes by score. This also adds ENR attribute stats in the nodeset info command. Node set commands are now documented in README. --- cmd/devp2p/README.md | 27 +++++++++++++-- cmd/devp2p/nodeset.go | 28 +++++++++++++++- cmd/devp2p/nodesetcmd.go | 71 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/cmd/devp2p/README.md b/cmd/devp2p/README.md index e934ee25c9..7f816b602e 100644 --- a/cmd/devp2p/README.md +++ b/cmd/devp2p/README.md @@ -30,6 +30,29 @@ Run `devp2p dns to-route53 ` to publish a tree to Amazon Route53. You can find more information about these commands in the [DNS Discovery Setup Guide][dns-tutorial]. +### Node Set Utilities + +There are several commands for working with JSON node set files. These files are generated +by the discovery crawlers and DNS client commands. Node sets also used as the input of the +DNS deployer commands. + +Run `devp2p nodeset info ` to display statistics of a node set. + +Run `devp2p nodeset filter ` to write a new, filtered node +set to standard output. The following filters are supported: + +- `-limit ` limits the output set to N entries, taking the top N nodes by score +- `-ip ` filters nodes by IP subnet +- `-min-age ` filters nodes by 'first seen' time +- `-eth-network ` filters nodes by "eth" ENR entry +- `-les-server` filters nodes by LES server support +- `-snap` filters nodes by snap protocol support + +For example, given a node set in `nodes.json`, you could create a filtered set containing +up to 20 eth mainnet nodes which also support snap sync using this command: + + devp2p nodeset filter nodes.json -eth-network mainnet -snap -limit 20 + ### Discovery v4 Utilities The `devp2p discv4 ...` command family deals with the [Node Discovery v4][discv4] @@ -94,7 +117,7 @@ To run the eth protocol test suite against your implementation, the node needs t geth --datadir --nodiscover --nat=none --networkid 19763 --verbosity 5 ``` -Then, run the following command, replacing `` with the enode of the geth node: +Then, run the following command, replacing `` with the enode of the geth node: ``` devp2p rlpx eth-test cmd/devp2p/internal/ethtest/testdata/chain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json ``` @@ -103,7 +126,7 @@ Repeat the above process (re-initialising the node) in order to run the Eth Prot #### Eth66 Test Suite -The Eth66 test suite is also a conformance test suite for the eth 66 protocol version specifically. +The Eth66 test suite is also a conformance test suite for the eth 66 protocol version specifically. To run the eth66 protocol test suite, initialize a geth node as described above and run the following command, replacing `` with the enode of the geth node: diff --git a/cmd/devp2p/nodeset.go b/cmd/devp2p/nodeset.go index 2d86c3f65a..1d78e34c73 100644 --- a/cmd/devp2p/nodeset.go +++ b/cmd/devp2p/nodeset.go @@ -71,6 +71,7 @@ func writeNodesJSON(file string, nodes nodeSet) { } } +// nodes returns the node records contained in the set. func (ns nodeSet) nodes() []*enode.Node { result := make([]*enode.Node, 0, len(ns)) for _, n := range ns { @@ -83,12 +84,37 @@ func (ns nodeSet) nodes() []*enode.Node { return result } +// add ensures the given nodes are present in the set. func (ns nodeSet) add(nodes ...*enode.Node) { for _, n := range nodes { - ns[n.ID()] = nodeJSON{Seq: n.Seq(), N: n} + v := ns[n.ID()] + v.N = n + v.Seq = n.Seq() + ns[n.ID()] = v } } +// topN returns the top n nodes by score as a new set. +func (ns nodeSet) topN(n int) nodeSet { + if n >= len(ns) { + return ns + } + + byscore := make([]nodeJSON, 0, len(ns)) + for _, v := range ns { + byscore = append(byscore, v) + } + sort.Slice(byscore, func(i, j int) bool { + return byscore[i].Score >= byscore[j].Score + }) + result := make(nodeSet, n) + for _, v := range byscore[:n] { + result[v.N.ID()] = v + } + return result +} + +// verify performs integrity checks on the node set. func (ns nodeSet) verify() error { for id, n := range ns { if n.N.ID() != id { diff --git a/cmd/devp2p/nodesetcmd.go b/cmd/devp2p/nodesetcmd.go index 33de1fdf31..848288c9cf 100644 --- a/cmd/devp2p/nodesetcmd.go +++ b/cmd/devp2p/nodesetcmd.go @@ -17,8 +17,12 @@ package main import ( + "errors" "fmt" "net" + "sort" + "strconv" + "strings" "time" "github.com/ethereum/go-ethereum/core/forkid" @@ -60,25 +64,64 @@ func nodesetInfo(ctx *cli.Context) error { ns := loadNodesJSON(ctx.Args().First()) fmt.Printf("Set contains %d nodes.\n", len(ns)) + showAttributeCounts(ns) return nil } +// showAttributeCounts prints the distribution of ENR attributes in a node set. +func showAttributeCounts(ns nodeSet) { + attrcount := make(map[string]int) + var attrlist []interface{} + for _, n := range ns { + r := n.N.Record() + attrlist = r.AppendElements(attrlist[:0])[1:] + for i := 0; i < len(attrlist); i += 2 { + key := attrlist[i].(string) + attrcount[key]++ + } + } + + var keys []string + var maxlength int + for key := range attrcount { + keys = append(keys, key) + if len(key) > maxlength { + maxlength = len(key) + } + } + sort.Strings(keys) + fmt.Println("ENR attribute counts:") + for _, key := range keys { + fmt.Printf("%s%s: %d\n", strings.Repeat(" ", maxlength-len(key)+1), key, attrcount[key]) + } +} + func nodesetFilter(ctx *cli.Context) error { if ctx.NArg() < 1 { return fmt.Errorf("need nodes file as argument") } - ns := loadNodesJSON(ctx.Args().First()) + // Parse -limit. + limit, err := parseFilterLimit(ctx.Args().Tail()) + if err != nil { + return err + } + // Parse the filters. filter, err := andFilter(ctx.Args().Tail()) if err != nil { return err } + // Load nodes and apply filters. + ns := loadNodesJSON(ctx.Args().First()) result := make(nodeSet) for id, n := range ns { if filter(n) { result[id] = n } } + if limit >= 0 { + result = result.topN(limit) + } writeNodesJSON("-", result) return nil } @@ -91,6 +134,7 @@ type nodeFilterC struct { } var filterFlags = map[string]nodeFilterC{ + "-limit": {1, trueFilter}, // needed to skip over -limit "-ip": {1, ipFilter}, "-min-age": {1, minAgeFilter}, "-eth-network": {1, ethFilter}, @@ -98,6 +142,7 @@ var filterFlags = map[string]nodeFilterC{ "-snap": {0, snapFilter}, } +// parseFilters parses nodeFilters from args. func parseFilters(args []string) ([]nodeFilter, error) { var filters []nodeFilter for len(args) > 0 { @@ -118,6 +163,26 @@ func parseFilters(args []string) ([]nodeFilter, error) { return filters, nil } +// parseFilterLimit parses the -limit option in args. It returns -1 if there is no limit. +func parseFilterLimit(args []string) (int, error) { + limit := -1 + for i, arg := range args { + if arg == "-limit" { + if i == len(args)-1 { + return -1, errors.New("-limit requires an argument") + } + n, err := strconv.Atoi(args[i+1]) + if err != nil { + return -1, fmt.Errorf("invalid -limit %q", args[i+1]) + } + limit = n + } + } + return limit, nil +} + +// andFilter parses node filters in args and and returns a single filter that requires all +// of them to match. func andFilter(args []string) (nodeFilter, error) { checks, err := parseFilters(args) if err != nil { @@ -134,6 +199,10 @@ func andFilter(args []string) (nodeFilter, error) { return f, nil } +func trueFilter(args []string) (nodeFilter, error) { + return func(n nodeJSON) bool { return true }, nil +} + func ipFilter(args []string) (nodeFilter, error) { _, cidr, err := net.ParseCIDR(args[0]) if err != nil { From 653b7e959d57bea49ba3628e4336a74ef4363ce2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 19 Apr 2021 14:54:55 +0200 Subject: [PATCH 280/709] cmd/devp2p: add dns nuke-route53 command (#22695) --- cmd/devp2p/dns_route53.go | 55 +++++++++++++++++++++++++++++++-------- cmd/devp2p/dnscmd.go | 35 +++++++++++++++++++++---- 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 2b6a30e60f..1d4f975dda 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -107,22 +107,48 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { return err } log.Info(fmt.Sprintf("Found %d TXT records", len(existing))) - records := t.ToTXT(name) changes := c.computeChanges(name, records, existing) + + // Submit to API. + comment := fmt.Sprintf("enrtree update of %s at seq %d", name, t.Seq()) + return c.submitChanges(changes, comment) +} + +// deleteDomain removes all TXT records of the given domain. +func (c *route53Client) deleteDomain(name string) error { + if err := c.checkZone(name); err != nil { + return err + } + + // Compute DNS changes. + existing, err := c.collectRecords(name) + if err != nil { + return err + } + log.Info(fmt.Sprintf("Found %d TXT records", len(existing))) + changes := makeDeletionChanges(existing, nil) + + // Submit to API. + comment := "enrtree delete of " + name + return c.submitChanges(changes, comment) +} + +// submitChanges submits the given DNS changes to Route53. +func (c *route53Client) submitChanges(changes []types.Change, comment string) error { if len(changes) == 0 { log.Info("No DNS changes needed") return nil } - // Submit all change batches. + var err error batches := splitChanges(changes, route53ChangeSizeLimit, route53ChangeCountLimit) changesToCheck := make([]*route53.ChangeResourceRecordSetsOutput, len(batches)) for i, changes := range batches { log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes))) batch := &types.ChangeBatch{ Changes: changes, - Comment: aws.String(fmt.Sprintf("enrtree update %d/%d of %s at seq %d", i+1, len(batches), name, t.Seq())), + Comment: aws.String(fmt.Sprintf("%s (%d/%d)", comment, i+1, len(batches))), } req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch} changesToCheck[i], err = c.api.ChangeResourceRecordSets(context.TODO(), req) @@ -151,7 +177,6 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { time.Sleep(30 * time.Second) } } - return nil } @@ -186,7 +211,8 @@ func (c *route53Client) findZoneID(name string) (string, error) { return "", errors.New("can't find zone ID for " + name) } -// computeChanges creates DNS changes for the given record. +// computeChanges creates DNS changes for the given set of DNS discovery records. +// The 'existing' arg is the set of records that already exist on Route53. func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []types.Change { // Convert all names to lowercase. lrecords := make(map[string]string, len(records)) @@ -223,16 +249,23 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e } // Iterate over the old records and delete anything stale. - for path, set := range existing { - if _, ok := records[path]; ok { + changes = append(changes, makeDeletionChanges(existing, records)...) + + // Ensure changes are in the correct order. + sortChanges(changes) + return changes +} + +// makeDeletionChanges creates record changes which delete all records not contained in 'keep'. +func makeDeletionChanges(records map[string]recordSet, keep map[string]string) []types.Change { + var changes []types.Change + for path, set := range records { + if _, ok := keep[path]; ok { continue } - // Stale entry, nuke it. - log.Info(fmt.Sprintf("Deleting %s = %q", path, strings.Join(set.values, ""))) + log.Info(fmt.Sprintf("Deleting %s = %s", path, strings.Join(set.values, ""))) changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values...)) } - - sortChanges(changes) return changes } diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index 50ab7bf983..66deef56ea 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -43,6 +43,7 @@ var ( dnsTXTCommand, dnsCloudflareCommand, dnsRoute53Command, + dnsRoute53NukeCommand, }, } dnsSyncCommand = cli.Command{ @@ -84,6 +85,18 @@ var ( route53RegionFlag, }, } + dnsRoute53NukeCommand = cli.Command{ + Name: "nuke-route53", + Usage: "Deletes DNS TXT records of a subdomain on Amazon Route53", + ArgsUsage: "", + Action: dnsNukeRoute53, + Flags: []cli.Flag{ + route53AccessKeyFlag, + route53AccessSecretFlag, + route53ZoneIDFlag, + route53RegionFlag, + }, + } ) var ( @@ -174,6 +187,9 @@ func dnsSign(ctx *cli.Context) error { return nil } +// directoryName returns the directory name of the given path. +// For example, when dir is "foo/bar", it returns "bar". +// When dir is ".", and the working directory is "example/foo", it returns "foo". func directoryName(dir string) string { abs, err := filepath.Abs(dir) if err != nil { @@ -182,7 +198,7 @@ func directoryName(dir string) string { return filepath.Base(abs) } -// dnsToTXT peforms dnsTXTCommand. +// dnsToTXT performs dnsTXTCommand. func dnsToTXT(ctx *cli.Context) error { if ctx.NArg() < 1 { return fmt.Errorf("need tree definition directory as argument") @@ -199,9 +215,9 @@ func dnsToTXT(ctx *cli.Context) error { return nil } -// dnsToCloudflare peforms dnsCloudflareCommand. +// dnsToCloudflare performs dnsCloudflareCommand. func dnsToCloudflare(ctx *cli.Context) error { - if ctx.NArg() < 1 { + if ctx.NArg() != 1 { return fmt.Errorf("need tree definition directory as argument") } domain, t, err := loadTreeDefinitionForExport(ctx.Args().Get(0)) @@ -212,9 +228,9 @@ func dnsToCloudflare(ctx *cli.Context) error { return client.deploy(domain, t) } -// dnsToRoute53 peforms dnsRoute53Command. +// dnsToRoute53 performs dnsRoute53Command. func dnsToRoute53(ctx *cli.Context) error { - if ctx.NArg() < 1 { + if ctx.NArg() != 1 { return fmt.Errorf("need tree definition directory as argument") } domain, t, err := loadTreeDefinitionForExport(ctx.Args().Get(0)) @@ -225,6 +241,15 @@ func dnsToRoute53(ctx *cli.Context) error { return client.deploy(domain, t) } +// dnsNukeRoute53 performs dnsRoute53NukeCommand. +func dnsNukeRoute53(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return fmt.Errorf("need domain name as argument") + } + client := newRoute53Client(ctx) + return client.deleteDomain(ctx.Args().First()) +} + // loadSigningKey loads a private key in Ethereum keystore format. func loadSigningKey(keyfile string) *ecdsa.PrivateKey { keyjson, err := ioutil.ReadFile(keyfile) From d6ffa140359d24262903962f145a1236836aeeb4 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 20 Apr 2021 13:27:46 +0800 Subject: [PATCH 281/709] core: nuke legacy snapshot supporting (#22663) --- core/blockchain.go | 15 +- core/blockchain_snapshot_test.go | 320 +------------------------------ core/state/snapshot/journal.go | 122 +----------- core/state/snapshot/snapshot.go | 27 --- 4 files changed, 12 insertions(+), 472 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index dfbc5ad0bb..49aa1c3e86 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -207,9 +207,8 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config - shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. - writeLegacyJournal bool // Testing flag used to flush the snapshot journal in legacy format. + shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. } // NewBlockChain returns a fully initialised block chain using information @@ -1002,14 +1001,8 @@ func (bc *BlockChain) Stop() { var snapBase common.Hash if bc.snaps != nil { var err error - if bc.writeLegacyJournal { - if snapBase, err = bc.snaps.LegacyJournal(bc.CurrentBlock().Root()); err != nil { - log.Error("Failed to journal state snapshot", "err", err) - } - } else { - if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil { - log.Error("Failed to journal state snapshot", "err", err) - } + if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil { + log.Error("Failed to journal state snapshot", "err", err) } } // Ensure the state of a recent block is also stored to disk before exiting. diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 70ae31f7d8..75c09b421d 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -39,7 +39,6 @@ import ( // snapshotTestBasic wraps the common testing fields in the snapshot tests. type snapshotTestBasic struct { - legacy bool // Wether write the snapshot journal in legacy format chainBlocks int // Number of blocks to generate for the canonical chain snapshotBlock uint64 // Block number of the relevant snapshot disk layer commitBlock uint64 // Block number for which to commit the state to disk @@ -104,19 +103,13 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil) } if basic.snapshotBlock > 0 && basic.snapshotBlock == point { - if basic.legacy { - // Here we commit the snapshot disk root to simulate - // committing the legacy snapshot. - rawdb.WriteSnapshotRoot(db, blocks[point-1].Root()) - } else { - // Flushing the entire snap tree into the disk, the - // relavant (a) snapshot root and (b) snapshot generator - // will be persisted atomically. - chain.snaps.Cap(blocks[point-1].Root(), 0) - diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root() - if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) { - t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot) - } + // Flushing the entire snap tree into the disk, the + // relavant (a) snapshot root and (b) snapshot generator + // will be persisted atomically. + chain.snaps.Cap(blocks[point-1].Root(), 0) + diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root() + if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) { + t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot) } } } @@ -129,12 +122,6 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo basic.db = db basic.gendb = gendb basic.engine = engine - - // Ugly hack, notify the chain to flush the journal in legacy format - // if it's requested. - if basic.legacy { - chain.writeLegacyJournal = true - } return chain, blocks } @@ -484,46 +471,6 @@ func TestRestartWithNewSnapshot(t *testing.T) { // Expected snapshot disk : G test := &snapshotTest{ snapshotTestBasic{ - legacy: false, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 8, - expSnapshotBottom: 0, // Initial disk layer built from genesis - }, - } - test.test(t) - test.teardown() -} - -// Tests a Geth restart with valid but "legacy" snapshot. Before the shutdown, -// all snapshot journal will be persisted correctly. In this case no snapshot -// recovery is required. -func TestRestartWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(0) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : C8 - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &snapshotTest{ - snapshotTestBasic{ - legacy: true, chainBlocks: 8, snapshotBlock: 0, commitBlock: 0, @@ -563,7 +510,6 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // Expected snapshot disk : C4 test := &crashSnapshotTest{ snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 0, @@ -603,7 +549,6 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // Expected snapshot disk : C4 test := &crashSnapshotTest{ snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 2, @@ -643,7 +588,6 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { // Expected snapshot disk : C4 test := &crashSnapshotTest{ snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 6, @@ -658,131 +602,6 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { test.teardown() } -// Tests a Geth was crashed and restarts with a broken and "legacy format" -// snapshot. In this case the entire legacy snapshot should be discared -// and rebuild from the new chain head. The new head here refers to the -// genesis because there is no committed point. -func TestNoCommitCrashWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G, C4 - // - // CRASH - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : G - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &crashSnapshotTest{ - snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) - }, - } - test.test(t) - test.teardown() -} - -// Tests a Geth was crashed and restarts with a broken and "legacy format" -// snapshot. In this case the entire legacy snapshot should be discared -// and rebuild from the new chain head. The new head here refers to the -// block-2 because it's committed into the disk. -func TestLowCommitCrashWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G, C2 - // Snapshot: G, C4 - // - // CRASH - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : C2 - // Expected snapshot disk : C2 - t.Skip("Legacy format testing is not supported") - test := &crashSnapshotTest{ - snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 2, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 2, - expSnapshotBottom: 2, // Rebuilt snapshot from the latest HEAD - }, - } - test.test(t) - test.teardown() -} - -// Tests a Geth was crashed and restarts with a broken and "legacy format" -// snapshot. In this case the entire legacy snapshot should be discared -// and rebuild from the new chain head. -// -// The new head here refers to the the genesis, the reason is: -// - the state of block-6 is committed into the disk -// - the legacy disk layer of block-4 is committed into the disk -// - the head is rewound the genesis in order to find an available -// state lower than disk layer -func TestHighCommitCrashWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G, C6 - // Snapshot: G, C4 - // - // CRASH - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8 - // - // Expected head header : C8 - // Expected head fast block: C8 - // Expected head block : G - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &crashSnapshotTest{ - snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 6, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis) - }, - } - test.test(t) - test.teardown() -} - // Tests a Geth was running with snapshot enabled. Then restarts without // enabling snapshot and after that re-enable the snapshot again. In this // case the snapshot should be rebuilt with latest chain head. @@ -806,47 +625,6 @@ func TestGappedNewSnapshot(t *testing.T) { // Expected snapshot disk : C10 test := &gappedSnapshotTest{ snapshotTestBasic: snapshotTestBasic{ - legacy: false, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD - }, - gapped: 2, - } - test.test(t) - test.teardown() -} - -// Tests a Geth was running with leagcy snapshot enabled. Then restarts -// without enabling snapshot and after that re-enable the snapshot again. -// In this case the snapshot should be rebuilt with latest chain head. -func TestGappedLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(0) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 - // - // Expected head header : C10 - // Expected head fast block: C10 - // Expected head block : C10 - // Expected snapshot disk : C10 - t.Skip("Legacy format testing is not supported") - test := &gappedSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - legacy: true, chainBlocks: 8, snapshotBlock: 0, commitBlock: 0, @@ -885,7 +663,6 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { // Expected snapshot disk : G test := &setHeadSnapshotTest{ snapshotTestBasic: snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 0, commitBlock: 0, @@ -901,88 +678,6 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { test.teardown() } -// Tests the Geth was running with snapshot(legacy-format) enabled and resetHead -// is applied. In this case the head is rewound to the target(with state available). -// After that the chain is restarted and the original disk layer is kept. -func TestSetHeadWithLegacySnapshot(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(4) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4 - // - // Expected head header : C4 - // Expected head fast block: C4 - // Expected head block : C4 - // Expected snapshot disk : G - t.Skip("Legacy format testing is not supported") - test := &setHeadSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 4, - expHeadHeader: 4, - expHeadFastBlock: 4, - expHeadBlock: 4, - expSnapshotBottom: 0, // The initial disk layer is built from the genesis - }, - setHead: 4, - } - test.test(t) - test.teardown() -} - -// Tests the Geth was running with snapshot(legacy-format) enabled and upgrades -// the disk layer journal(journal generator) to latest format. After that the Geth -// is restarted from a crash. In this case Geth will find the new-format disk layer -// journal but with legacy-format diff journal(the new-format is never committed), -// and the invalid diff journal is expected to be dropped. -func TestRecoverSnapshotFromCrashWithLegacyDiffJournal(t *testing.T) { - // Chain: - // G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD) - // - // Commit: G - // Snapshot: G - // - // SetHead(0) - // - // ------------------------------ - // - // Expected in leveldb: - // G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 - // - // Expected head header : C10 - // Expected head fast block: C10 - // Expected head block : C8 - // Expected snapshot disk : C10 - t.Skip("Legacy format testing is not supported") - test := &restartCrashSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - legacy: true, - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 8, // The persisted state in the first running - expSnapshotBottom: 10, // The persisted disk layer in the second running - }, - newBlocks: 2, - } - test.test(t) - test.teardown() -} - // Tests the Geth was running with a complete snapshot and then imports a few // more new blocks on top without enabling the snapshot. After the restart, // crash happens. Check everything is ok after the restart. @@ -1006,7 +701,6 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) { // Expected snapshot disk : C10 test := &wipeCrashSnapshotTest{ snapshotTestBasic: snapshotTestBasic{ - legacy: false, chainBlocks: 8, snapshotBlock: 4, commitBlock: 0, diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index b31e921ca9..f8cec4d4ea 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -66,30 +66,6 @@ type journalStorage struct { Vals [][]byte } -// loadAndParseLegacyJournal tries to parse the snapshot journal in legacy format. -func loadAndParseLegacyJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) { - // Retrieve the journal, for legacy journal it must exist since even for - // 0 layer it stores whether we've already generated the snapshot or are - // in progress only. - journal := rawdb.ReadSnapshotJournal(db) - if len(journal) == 0 { - return nil, journalGenerator{}, errors.New("missing or corrupted snapshot journal") - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - - // Read the snapshot generation progress for the disk layer - var generator journalGenerator - if err := r.Decode(&generator); err != nil { - return nil, journalGenerator{}, fmt.Errorf("failed to load snapshot progress marker: %v", err) - } - // Load all the snapshot diffs from the journal - snapshot, err := loadDiffLayer(base, r) - if err != nil { - return nil, generator, err - } - return snapshot, generator, nil -} - // loadAndParseJournal tries to parse the snapshot journal in latest format. func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) { // Retrieve the disk layer generator. It must exist, no matter the @@ -163,14 +139,9 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, cache: fastcache.New(cache * 1024 * 1024), root: baseRoot, } - var legacy bool snapshot, generator, err := loadAndParseJournal(diskdb, base) if err != nil { log.Warn("Failed to load new-format journal", "error", err) - snapshot, generator, err = loadAndParseLegacyJournal(diskdb, base) - legacy = true - } - if err != nil { return nil, err } // Entire snapshot journal loaded, sanity check the head. If the loaded @@ -185,7 +156,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. - if legacy || !recovery { + if !recovery { return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root) } // It's in snapshot recovery, the assumption is held that @@ -346,94 +317,3 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } - -// LegacyJournal writes the persistent layer generator stats into a buffer -// to be stored in the database as the snapshot journal. -// -// Note it's the legacy version which is only used in testing right now. -func (dl *diskLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) { - // If the snapshot is currently being generated, abort it - var stats *generatorStats - if dl.genAbort != nil { - abort := make(chan *generatorStats) - dl.genAbort <- abort - - if stats = <-abort; stats != nil { - stats.Log("Journalling in-progress snapshot", dl.root, dl.genMarker) - } - } - // Ensure the layer didn't get stale - dl.lock.RLock() - defer dl.lock.RUnlock() - - if dl.stale { - return common.Hash{}, ErrSnapshotStale - } - // Write out the generator marker - entry := journalGenerator{ - Done: dl.genMarker == nil, - Marker: dl.genMarker, - } - if stats != nil { - entry.Accounts = stats.accounts - entry.Slots = stats.slots - entry.Storage = uint64(stats.storage) - } - log.Debug("Legacy journalled disk layer", "root", dl.root) - if err := rlp.Encode(buffer, entry); err != nil { - return common.Hash{}, err - } - return dl.root, nil -} - -// Journal writes the memory layer contents into a buffer to be stored in the -// database as the snapshot journal. -// -// Note it's the legacy version which is only used in testing right now. -func (dl *diffLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) { - // Journal the parent first - base, err := dl.parent.LegacyJournal(buffer) - if err != nil { - return common.Hash{}, err - } - // Ensure the layer didn't get stale - dl.lock.RLock() - defer dl.lock.RUnlock() - - if dl.Stale() { - return common.Hash{}, ErrSnapshotStale - } - // Everything below was journalled, persist this layer too - if err := rlp.Encode(buffer, dl.root); err != nil { - return common.Hash{}, err - } - destructs := make([]journalDestruct, 0, len(dl.destructSet)) - for hash := range dl.destructSet { - destructs = append(destructs, journalDestruct{Hash: hash}) - } - if err := rlp.Encode(buffer, destructs); err != nil { - return common.Hash{}, err - } - accounts := make([]journalAccount, 0, len(dl.accountData)) - for hash, blob := range dl.accountData { - accounts = append(accounts, journalAccount{Hash: hash, Blob: blob}) - } - if err := rlp.Encode(buffer, accounts); err != nil { - return common.Hash{}, err - } - storage := make([]journalStorage, 0, len(dl.storageData)) - for hash, slots := range dl.storageData { - keys := make([]common.Hash, 0, len(slots)) - vals := make([][]byte, 0, len(slots)) - for key, val := range slots { - keys = append(keys, key) - vals = append(vals, val) - } - storage = append(storage, journalStorage{Hash: hash, Keys: keys, Vals: vals}) - } - if err := rlp.Encode(buffer, storage); err != nil { - return common.Hash{}, err - } - log.Debug("Legacy journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) - return base, nil -} diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 710ba4d4c2..9ecbd4a6c8 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -137,10 +137,6 @@ type snapshot interface { // flattening everything down (bad for reorgs). Journal(buffer *bytes.Buffer) (common.Hash, error) - // LegacyJournal is basically identical to Journal. it's the legacy version for - // flushing legacy journal. Now the only purpose of this function is for testing. - LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) - // Stale return whether this layer has become stale (was flattened across) or // if it's still live. Stale() bool @@ -622,29 +618,6 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { return base, nil } -// LegacyJournal is basically identical to Journal. it's the legacy -// version for flushing legacy journal. Now the only purpose of this -// function is for testing. -func (t *Tree) LegacyJournal(root common.Hash) (common.Hash, error) { - // Retrieve the head snapshot to journal from var snap snapshot - snap := t.Snapshot(root) - if snap == nil { - return common.Hash{}, fmt.Errorf("snapshot [%#x] missing", root) - } - // Run the journaling - t.lock.Lock() - defer t.lock.Unlock() - - journal := new(bytes.Buffer) - base, err := snap.(snapshot).LegacyJournal(journal) - if err != nil { - return common.Hash{}, err - } - // Store the journal into the database and return - rawdb.WriteSnapshotJournal(t.diskdb, journal.Bytes()) - return base, nil -} - // Rebuild wipes all available snapshot data from the persistent database and // discard all caches and diff layers. Afterwards, it starts a new snapshot // generator with the given root hash. From d7bfb978ba9a96f638890f1abb6e5bff760832fc Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 20 Apr 2021 10:29:36 +0200 Subject: [PATCH 282/709] ethash: no block reward in catalyst mode (#22697) --- consensus/ethash/consensus.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index e23bd824af..c405d07fc6 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -618,6 +618,10 @@ var ( // reward. The total reward consists of the static block reward and rewards for // included uncles. The coinbase of each uncle block is also rewarded. func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) { + // Skip block reward in catalyst mode + if config.IsCatalyst(header.Number) { + return + } // Select the correct block reward based on chain progression blockReward := FrontierBlockReward if config.IsByzantium(header.Number) { From 581539c6ee795f17b631b0ae6c2ce4948d41c3ba Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 20 Apr 2021 10:42:02 +0200 Subject: [PATCH 283/709] trie: make stacktrie support binary marshal/unmarshal (#22685) --- trie/stacktrie.go | 94 ++++++++++++++++++++++++++++++++++++++++++ trie/stacktrie_test.go | 47 ++++++++++++++++++++- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index c34ac6c78c..f9ff10b62d 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -17,8 +17,12 @@ package trie import ( + "bufio" + "bytes" + "encoding/gob" "errors" "fmt" + "io" "sync" "github.com/ethereum/go-ethereum/common" @@ -66,6 +70,96 @@ func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie { } } +// NewFromBinary initialises a serialized stacktrie with the given db. +func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) { + var st StackTrie + if err := st.UnmarshalBinary(data); err != nil { + return nil, err + } + // If a database is used, we need to recursively add it to every child + if db != nil { + st.setDb(db) + } + return &st, nil +} + +// MarshalBinary implements encoding.BinaryMarshaler +func (st *StackTrie) MarshalBinary() (data []byte, err error) { + var ( + b bytes.Buffer + w = bufio.NewWriter(&b) + ) + if err := gob.NewEncoder(w).Encode(struct { + Nodetype uint8 + Val []byte + Key []byte + KeyOffset uint8 + }{ + st.nodeType, + st.val, + st.key, + uint8(st.keyOffset), + }); err != nil { + return nil, err + } + for _, child := range st.children { + if child == nil { + w.WriteByte(0) + continue + } + w.WriteByte(1) + if childData, err := child.MarshalBinary(); err != nil { + return nil, err + } else { + w.Write(childData) + } + } + w.Flush() + return b.Bytes(), nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler +func (st *StackTrie) UnmarshalBinary(data []byte) error { + r := bytes.NewReader(data) + return st.unmarshalBinary(r) +} + +func (st *StackTrie) unmarshalBinary(r io.Reader) error { + var dec struct { + Nodetype uint8 + Val []byte + Key []byte + KeyOffset uint8 + } + gob.NewDecoder(r).Decode(&dec) + st.nodeType = dec.Nodetype + st.val = dec.Val + st.key = dec.Key + st.keyOffset = int(dec.KeyOffset) + + var hasChild = make([]byte, 1) + for i := range st.children { + if _, err := r.Read(hasChild); err != nil { + return err + } else if hasChild[0] == 0 { + continue + } + var child StackTrie + child.unmarshalBinary(r) + st.children[i] = &child + } + return nil +} + +func (st *StackTrie) setDb(db ethdb.KeyValueWriter) { + st.db = db + for _, child := range st.children { + if child != nil { + child.setDb(db) + } + } +} + func newLeaf(ko int, key, val []byte, db ethdb.KeyValueWriter) *StackTrie { st := stackTrieFromPool(db) st.nodeType = leafNode diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 50a4eda323..ccdf389d52 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -151,7 +151,6 @@ func TestStacktrieNotModifyValues(t *testing.T) { return big.NewInt(int64(i)).Bytes() } } - for i := 0; i < 1000; i++ { key := common.BigToHash(keyB) value := getValue(i) @@ -168,5 +167,51 @@ func TestStacktrieNotModifyValues(t *testing.T) { if !bytes.Equal(have, want) { t.Fatalf("item %d, have %#x want %#x", i, have, want) } + + } +} + +// TestStacktrieSerialization tests that the stacktrie works well if we +// serialize/unserialize it a lot +func TestStacktrieSerialization(t *testing.T) { + var ( + st = NewStackTrie(nil) + nt, _ = New(common.Hash{}, NewDatabase(memorydb.New())) + keyB = big.NewInt(1) + keyDelta = big.NewInt(1) + vals [][]byte + keys [][]byte + ) + getValue := func(i int) []byte { + if i%2 == 0 { // large + return crypto.Keccak256(big.NewInt(int64(i)).Bytes()) + } else { //small + return big.NewInt(int64(i)).Bytes() + } + } + for i := 0; i < 10; i++ { + vals = append(vals, getValue(i)) + keys = append(keys, common.BigToHash(keyB).Bytes()) + keyB = keyB.Add(keyB, keyDelta) + keyDelta.Add(keyDelta, common.Big1) + } + for i, k := range keys { + nt.TryUpdate(k, common.CopyBytes(vals[i])) + } + + for i, k := range keys { + blob, err := st.MarshalBinary() + if err != nil { + t.Fatal(err) + } + newSt, err := NewFromBinary(blob, nil) + if err != nil { + t.Fatal(err) + } + st = newSt + st.TryUpdate(k, common.CopyBytes(vals[i])) + } + if have, want := st.Hash(), nt.Hash(); have != want { + t.Fatalf("have %#x want %#x", have, want) } } From beee6b77a0ca8c26881dc2bb17c58e996c90a0c6 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 20 Apr 2021 10:54:41 +0200 Subject: [PATCH 284/709] go.mod: upgrade gopsutils to v3.21.4 (#22693) This fixes the OpenBSD/arm64 build. --- go.mod | 5 +++-- go.sum | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0b0e8bf373..74c406f78b 100644 --- a/go.mod +++ b/go.mod @@ -45,13 +45,14 @@ require ( github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v1.7.0 - github.com/shirou/gopsutil v2.20.5+incompatible + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 + github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c + golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa golang.org/x/text v0.3.4 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce diff --git a/go.sum b/go.sum index 1b7212e420..0a9444323c 100644 --- a/go.sum +++ b/go.sum @@ -428,6 +428,10 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= +github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -459,6 +463,10 @@ github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKk github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= @@ -583,6 +591,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From cc33398cef4d9faeaec1e68e13029e1a0e1d6e41 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 20 Apr 2021 12:28:20 +0200 Subject: [PATCH 285/709] tests: disable blockchain tests based on general state tests (#22704) --- tests/block_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/block_test.go b/tests/block_test.go index 2649bae85a..4820ba733f 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -28,7 +28,7 @@ func TestBlockchain(t *testing.T) { // For speedier CI-runs, the line below can be uncommented, so those are skipped. // For now, in hardfork-times (Berlin), we run the tests both as StateTests and // as blockchain tests, since the latter also covers things like receipt root - //bt.skipLoad(`^GeneralStateTests/`) + bt.skipLoad(`^GeneralStateTests/`) // Skip random failures due to selfish mining test bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`) From dd9c3225cf06dab0acf783fad671b4f601a4470e Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 21 Apr 2021 15:21:22 +0800 Subject: [PATCH 286/709] eth, internal: extend the TraceCall API (#22245) Adds an an optional parameter `overrides *map[common.Address]account` to the `eth_call` API in order for the caller to can customize the state. --- eth/tracers/api.go | 33 +++++++- eth/tracers/api_test.go | 170 +++++++++++++++++++++++++++++++++++++++- internal/ethapi/api.go | 44 +++++++---- 3 files changed, 226 insertions(+), 21 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 5a28d6889e..1c727f1366 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -161,6 +161,16 @@ type TraceConfig struct { Reexec *uint64 } +// TraceCallConfig is the config for traceCall API. It holds one more +// field to override the state for tracing. +type TraceCallConfig struct { + *vm.LogConfig + Tracer *string + Timeout *string + Reexec *uint64 + StateOverrides *ethapi.StateOverride +} + // StdTraceConfig holds extra parameters to standard-json trace functions. type StdTraceConfig struct { vm.LogConfig @@ -720,7 +730,7 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. // You can provide -2 as a block number to trace on top of the pending block. -func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceConfig) (interface{}, error) { +func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { // Try to retrieve the specified block var ( err error @@ -730,6 +740,8 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHa block, err = api.blockByHash(ctx, hash) } else if number, ok := blockNrOrHash.Number(); ok { block, err = api.blockByNumber(ctx, number) + } else { + return nil, errors.New("invalid arguments; neither block nor hash specified") } if err != nil { return nil, err @@ -743,11 +755,26 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHa if err != nil { return nil, err } + // Apply the customized state rules if required. + if config != nil { + if err := config.StateOverrides.Apply(statedb); err != nil { + return nil, err + } + } // Execute the trace msg := args.ToMessage(api.backend.RPCGasCap()) vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) - return api.traceTx(ctx, msg, new(txTraceContext), vmctx, statedb, config) + var traceConfig *TraceConfig + if config != nil { + traceConfig = &TraceConfig{ + LogConfig: config.LogConfig, + Tracer: config.Tracer, + Timeout: config.Timeout, + Reexec: config.Reexec, + } + } + return api.traceTx(ctx, msg, new(txTraceContext), vmctx, statedb, traceConfig) } // traceTx configures a new tracer according to the provided configuration, and @@ -797,7 +824,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *txTrac result, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) if err != nil { - return nil, fmt.Errorf("tracing failed: %v", err) + return nil, fmt.Errorf("tracing failed: %w", err) } // Depending on the tracer type, format and return the output. diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 7ca90a6608..81a4bb5d05 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "crypto/ecdsa" + "encoding/json" "errors" "fmt" "math/big" @@ -198,7 +199,7 @@ func TestTraceCall(t *testing.T) { var testSuite = []struct { blockNumber rpc.BlockNumber call ethapi.CallArgs - config *TraceConfig + config *TraceCallConfig expectErr error expect interface{} }{ @@ -305,6 +306,147 @@ func TestTraceCall(t *testing.T) { } } +func TestOverridenTraceCall(t *testing.T) { + t.Parallel() + + // Initialize test accounts + accounts := newAccounts(3) + genesis := &core.Genesis{Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }} + genBlocks := 10 + signer := types.HomesteadSigner{} + api := NewAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, big.NewInt(0), nil), signer, accounts[0].key) + b.AddTx(tx) + })) + randomAccounts, tracer := newAccounts(3), "callTracer" + + var testSuite = []struct { + blockNumber rpc.BlockNumber + call ethapi.CallArgs + config *TraceCallConfig + expectErr error + expect *callTrace + }{ + // Succcessful call with state overriding + { + blockNumber: rpc.PendingBlockNumber, + call: ethapi.CallArgs{ + From: &randomAccounts[0].addr, + To: &randomAccounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: &TraceCallConfig{ + Tracer: &tracer, + StateOverrides: ðapi.StateOverride{ + randomAccounts[0].addr: ethapi.OverrideAccount{Balance: newRPCBalance(new(big.Int).Mul(big.NewInt(1), big.NewInt(params.Ether)))}, + }, + }, + expectErr: nil, + expect: &callTrace{ + Type: "CALL", + From: randomAccounts[0].addr, + To: randomAccounts[1].addr, + Gas: newRPCUint64(24979000), + GasUsed: newRPCUint64(0), + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + }, + // Invalid call without state overriding + { + blockNumber: rpc.PendingBlockNumber, + call: ethapi.CallArgs{ + From: &randomAccounts[0].addr, + To: &randomAccounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1000)), + }, + config: &TraceCallConfig{ + Tracer: &tracer, + }, + expectErr: core.ErrInsufficientFundsForTransfer, + expect: nil, + }, + // Sucessful simple contract call + // + // // SPDX-License-Identifier: GPL-3.0 + // + // pragma solidity >=0.7.0 <0.8.0; + // + // /** + // * @title Storage + // * @dev Store & retrieve value in a variable + // */ + // contract Storage { + // uint256 public number; + // constructor() { + // number = block.number; + // } + // } + { + blockNumber: rpc.PendingBlockNumber, + call: ethapi.CallArgs{ + From: &randomAccounts[0].addr, + To: &randomAccounts[2].addr, + Data: newRPCBytes(common.Hex2Bytes("8381f58a")), // call number() + }, + config: &TraceCallConfig{ + Tracer: &tracer, + StateOverrides: ðapi.StateOverride{ + randomAccounts[2].addr: ethapi.OverrideAccount{ + Code: newRPCBytes(common.Hex2Bytes("6080604052348015600f57600080fd5b506004361060285760003560e01c80638381f58a14602d575b600080fd5b60336049565b6040518082815260200191505060405180910390f35b6000548156fea2646970667358221220eab35ffa6ab2adfe380772a48b8ba78e82a1b820a18fcb6f59aa4efb20a5f60064736f6c63430007040033")), + StateDiff: newStates([]common.Hash{{}}, []common.Hash{common.BigToHash(big.NewInt(123))}), + }, + }, + }, + expectErr: nil, + expect: &callTrace{ + Type: "CALL", + From: randomAccounts[0].addr, + To: randomAccounts[2].addr, + Input: hexutil.Bytes(common.Hex2Bytes("8381f58a")), + Output: hexutil.Bytes(common.BigToHash(big.NewInt(123)).Bytes()), + Gas: newRPCUint64(24978936), + GasUsed: newRPCUint64(2283), + Value: (*hexutil.Big)(big.NewInt(0)), + }, + }, + } + for _, testspec := range testSuite { + result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) + if testspec.expectErr != nil { + if err == nil { + t.Errorf("Expect error %v, get nothing", testspec.expectErr) + continue + } + if !errors.Is(err, testspec.expectErr) { + t.Errorf("Error mismatch, want %v, get %v", testspec.expectErr, err) + } + } else { + if err != nil { + t.Errorf("Expect no error, get %v", err) + continue + } + ret := new(callTrace) + if err := json.Unmarshal(result.(json.RawMessage), ret); err != nil { + t.Fatalf("failed to unmarshal trace result: %v", err) + } + if !jsonEqual(ret, testspec.expect) { + // uncomment this for easier debugging + //have, _ := json.MarshalIndent(ret, "", " ") + //want, _ := json.MarshalIndent(testspec.expect, "", " ") + //t.Fatalf("trace mismatch: \nhave %+v\nwant %+v", string(have), string(want)) + t.Fatalf("trace mismatch: \nhave %+v\nwant %+v", ret, testspec.expect) + } + } + } +} + func TestTraceTransaction(t *testing.T) { t.Parallel() @@ -469,3 +611,29 @@ func newAccounts(n int) (accounts Accounts) { sort.Sort(accounts) return accounts } + +func newRPCBalance(balance *big.Int) **hexutil.Big { + rpcBalance := (*hexutil.Big)(balance) + return &rpcBalance +} + +func newRPCUint64(number uint64) *hexutil.Uint64 { + rpcUint64 := hexutil.Uint64(number) + return &rpcUint64 +} + +func newRPCBytes(bytes []byte) *hexutil.Bytes { + rpcBytes := hexutil.Bytes(bytes) + return &rpcBytes +} + +func newStates(keys []common.Hash, vals []common.Hash) *map[common.Hash]common.Hash { + if len(keys) != len(vals) { + panic("invalid input") + } + m := make(map[common.Hash]common.Hash) + for i := 0; i < len(keys); i++ { + m[keys[i]] = vals[i] + } + return &m +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index fe5c3388b5..a657f8a8f9 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -802,13 +803,13 @@ func (args *CallArgs) ToMessage(globalGasCap uint64) types.Message { return msg } -// account indicates the overriding fields of account during the execution of -// a message call. +// OverrideAccount indicates the overriding fields of account during the execution +// of a message call. // Note, state and stateDiff can't be specified at the same time. If state is // set, message execution will only use the data in the given state. Otherwise // if statDiff is set, all diff will be applied first and then execute the call // message. -type account struct { +type OverrideAccount struct { Nonce *hexutil.Uint64 `json:"nonce"` Code *hexutil.Bytes `json:"code"` Balance **hexutil.Big `json:"balance"` @@ -816,15 +817,15 @@ type account struct { StateDiff *map[common.Hash]common.Hash `json:"stateDiff"` } -func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]account, vmCfg vm.Config, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { - defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) +// StateOverride is the collection of overriden accounts. +type StateOverride map[common.Address]OverrideAccount - state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return nil, err +// Apply overrides the fields of specified accounts into the given state. +func (diff *StateOverride) Apply(state *state.StateDB) error { + if diff == nil { + return nil } - // Override the fields of specified contracts before execution. - for addr, account := range overrides { + for addr, account := range *diff { // Override account nonce. if account.Nonce != nil { state.SetNonce(addr, uint64(*account.Nonce)) @@ -838,7 +839,7 @@ func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.Blo state.SetBalance(addr, (*big.Int)(*account.Balance)) } if account.State != nil && account.StateDiff != nil { - return nil, fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex()) + return fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex()) } // Replace entire state if caller requires. if account.State != nil { @@ -851,6 +852,19 @@ func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.Blo } } } + return nil +} + +func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, vmCfg vm.Config, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { + defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) + + state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + if err := overrides.Apply(state); err != nil { + return nil, err + } // Setup context so it may be cancelled the call has completed // or, in case of unmetered gas, setup a context with a timeout. var cancel context.CancelFunc @@ -929,12 +943,8 @@ func (e *revertError) ErrorData() interface{} { // // Note, this function doesn't make and changes in the state/blockchain and is // useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *map[common.Address]account) (hexutil.Bytes, error) { - var accounts map[common.Address]account - if overrides != nil { - accounts = *overrides - } - result, err := DoCall(ctx, s.b, args, blockNrOrHash, accounts, vm.Config{}, 5*time.Second, s.b.RPCGasCap()) +func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { + result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, vm.Config{}, 5*time.Second, s.b.RPCGasCap()) if err != nil { return nil, err } From 96828c90f5ff9b73dc1a85da5999fe409324ee95 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 21 Apr 2021 16:18:27 +0800 Subject: [PATCH 287/709] eth/tracers, internal/ethapi: fix typos causing lint issue (#22711) --- eth/tracers/api_test.go | 2 +- internal/ethapi/api.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 81a4bb5d05..4c0240cd2c 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -372,7 +372,7 @@ func TestOverridenTraceCall(t *testing.T) { expectErr: core.ErrInsufficientFundsForTransfer, expect: nil, }, - // Sucessful simple contract call + // Successful simple contract call // // // SPDX-License-Identifier: GPL-3.0 // diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a657f8a8f9..fe3f80c038 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -817,7 +817,7 @@ type OverrideAccount struct { StateDiff *map[common.Hash]common.Hash `json:"stateDiff"` } -// StateOverride is the collection of overriden accounts. +// StateOverride is the collection of overridden accounts. type StateOverride map[common.Address]OverrideAccount // Apply overrides the fields of specified accounts into the given state. From 3e68d627b1b930a824942204ae3cd0b042cd1dbb Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 21 Apr 2021 10:19:28 +0200 Subject: [PATCH 288/709] les: fix goroutine leaks in tests (#22707) --- les/server.go | 16 ++++++++++++---- les/test_helper.go | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/les/server.go b/les/server.go index d44b1b57d4..c135e65f2d 100644 --- a/les/server.go +++ b/les/server.go @@ -212,17 +212,25 @@ func (s *LesServer) Stop() error { close(s.closeCh) s.clientPool.Stop() - s.serverset.close() + if s.serverset != nil { + s.serverset.close() + } s.peers.close() s.fcManager.Stop() s.costTracker.stop() s.handler.stop() s.servingQueue.stop() - s.vfluxServer.Stop() + if s.vfluxServer != nil { + s.vfluxServer.Stop() + } // Note, bloom trie indexer is closed by parent bloombits indexer. - s.chtIndexer.Close() - s.lesDb.Close() + if s.chtIndexer != nil { + s.chtIndexer.Close() + } + if s.lesDb != nil { + s.lesDb.Close() + } s.wg.Wait() log.Info("Les server stopped") diff --git a/les/test_helper.go b/les/test_helper.go index ee2da2f8eb..fc85ed957f 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -189,7 +189,7 @@ func testIndexers(db ethdb.Database, odr light.OdrBackend, config *light.Indexer return indexers[:] } -func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, indexers []*core.ChainIndexer, db ethdb.Database, peers *serverPeerSet, ulcServers []string, ulcFraction int) *clientHandler { +func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, indexers []*core.ChainIndexer, db ethdb.Database, peers *serverPeerSet, ulcServers []string, ulcFraction int) (*clientHandler, func()) { var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() @@ -245,10 +245,12 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index client.oracle.Start(backend) } client.handler.start() - return client.handler + return client.handler, func() { + client.handler.stop() + } } -func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Database, clock mclock.Clock) (*serverHandler, *backends.SimulatedBackend) { +func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Database, clock mclock.Clock) (*serverHandler, *backends.SimulatedBackend, func()) { var ( gspec = core.Genesis{ Config: params.AllEthashProtocolChanges, @@ -314,7 +316,8 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da } server.servingQueue.setThreads(4) server.handler.start() - return server.handler, simulation + closer := func() { server.Stop() } + return server.handler, simulation, closer } func alwaysTrueFn() bool { @@ -600,8 +603,8 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC ccIndexer, cbIndexer, cbtIndexer := cIndexers[0], cIndexers[1], cIndexers[2] odr.SetIndexers(ccIndexer, cbIndexer, cbtIndexer) - server, b := newTestServerHandler(config.blocks, sindexers, sdb, clock) - client := newTestClientHandler(b, odr, cIndexers, cdb, speers, config.ulcServers, config.ulcFraction) + server, b, serverClose := newTestServerHandler(config.blocks, sindexers, sdb, clock) + client, clientClose := newTestClientHandler(b, odr, cIndexers, cdb, speers, config.ulcServers, config.ulcFraction) scIndexer.Start(server.blockchain) sbIndexer.Start(server.blockchain) @@ -658,7 +661,10 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC cbIndexer.Close() scIndexer.Close() sbIndexer.Close() + dist.close() + serverClose() b.Close() + clientClose() } return s, c, teardown } From 4b783c0064661be55fd35b765c2a90d1f9b9abcb Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 21 Apr 2021 12:25:26 +0200 Subject: [PATCH 289/709] trie: improve the node iterator seek operation (#22470) This change improves the efficiency of the nodeIterator seek operation. Previously, seek essentially ran the iterator forward until it found the matching node. With this change, it skips over fullnode children and avoids resolving them from the database. --- trie/iterator.go | 145 +++++++++++++++++++++++++++++++++++------- trie/iterator_test.go | 81 +++++++++++++++++++++++ 2 files changed, 202 insertions(+), 24 deletions(-) diff --git a/trie/iterator.go b/trie/iterator.go index 76d437c403..4f72258a1d 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -243,7 +243,7 @@ func (it *nodeIterator) seek(prefix []byte) error { key = key[:len(key)-1] // Move forward until we're just before the closest match to key. for { - state, parentIndex, path, err := it.peek(bytes.HasPrefix(key, it.path)) + state, parentIndex, path, err := it.peekSeek(key) if err == errIteratorEnd { return errIteratorEnd } else if err != nil { @@ -255,16 +255,21 @@ func (it *nodeIterator) seek(prefix []byte) error { } } +// init initializes the the iterator. +func (it *nodeIterator) init() (*nodeIteratorState, error) { + root := it.trie.Hash() + state := &nodeIteratorState{node: it.trie.root, index: -1} + if root != emptyRoot { + state.hash = root + } + return state, state.resolve(it.trie, nil) +} + // peek creates the next state of the iterator. func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, error) { + // Initialize the iterator if we've just started. if len(it.stack) == 0 { - // Initialize the iterator if we've just started. - root := it.trie.Hash() - state := &nodeIteratorState{node: it.trie.root, index: -1} - if root != emptyRoot { - state.hash = root - } - err := state.resolve(it.trie, nil) + state, err := it.init() return state, nil, nil, err } if !descend { @@ -292,6 +297,39 @@ func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, er return nil, nil, nil, errIteratorEnd } +// peekSeek is like peek, but it also tries to skip resolving hashes by skipping +// over the siblings that do not lead towards the desired seek position. +func (it *nodeIterator) peekSeek(seekKey []byte) (*nodeIteratorState, *int, []byte, error) { + // Initialize the iterator if we've just started. + if len(it.stack) == 0 { + state, err := it.init() + return state, nil, nil, err + } + if !bytes.HasPrefix(seekKey, it.path) { + // If we're skipping children, pop the current node first + it.pop() + } + + // Continue iteration to the next child + for len(it.stack) > 0 { + parent := it.stack[len(it.stack)-1] + ancestor := parent.hash + if (ancestor == common.Hash{}) { + ancestor = parent.parent + } + state, path, ok := it.nextChildAt(parent, ancestor, seekKey) + if ok { + if err := state.resolve(it.trie, path); err != nil { + return parent, &parent.index, path, err + } + return state, &parent.index, path, nil + } + // No more child nodes, move back up. + it.pop() + } + return nil, nil, nil, errIteratorEnd +} + func (st *nodeIteratorState) resolve(tr *Trie, path []byte) error { if hash, ok := st.node.(hashNode); ok { resolved, err := tr.resolveHash(hash, path) @@ -304,25 +342,38 @@ func (st *nodeIteratorState) resolve(tr *Trie, path []byte) error { return nil } +func findChild(n *fullNode, index int, path []byte, ancestor common.Hash) (node, *nodeIteratorState, []byte, int) { + var ( + child node + state *nodeIteratorState + childPath []byte + ) + for ; index < len(n.Children); index++ { + if n.Children[index] != nil { + child = n.Children[index] + hash, _ := child.cache() + state = &nodeIteratorState{ + hash: common.BytesToHash(hash), + node: child, + parent: ancestor, + index: -1, + pathlen: len(path), + } + childPath = append(childPath, path...) + childPath = append(childPath, byte(index)) + return child, state, childPath, index + } + } + return nil, nil, nil, 0 +} + func (it *nodeIterator) nextChild(parent *nodeIteratorState, ancestor common.Hash) (*nodeIteratorState, []byte, bool) { switch node := parent.node.(type) { case *fullNode: - // Full node, move to the first non-nil child. - for i := parent.index + 1; i < len(node.Children); i++ { - child := node.Children[i] - if child != nil { - hash, _ := child.cache() - state := &nodeIteratorState{ - hash: common.BytesToHash(hash), - node: child, - parent: ancestor, - index: -1, - pathlen: len(it.path), - } - path := append(it.path, byte(i)) - parent.index = i - 1 - return state, path, true - } + //Full node, move to the first non-nil child. + if child, state, path, index := findChild(node, parent.index+1, it.path, ancestor); child != nil { + parent.index = index - 1 + return state, path, true } case *shortNode: // Short node, return the pointer singleton child @@ -342,6 +393,52 @@ func (it *nodeIterator) nextChild(parent *nodeIteratorState, ancestor common.Has return parent, it.path, false } +// nextChildAt is similar to nextChild, except that it targets a child as close to the +// target key as possible, thus skipping siblings. +func (it *nodeIterator) nextChildAt(parent *nodeIteratorState, ancestor common.Hash, key []byte) (*nodeIteratorState, []byte, bool) { + switch n := parent.node.(type) { + case *fullNode: + // Full node, move to the first non-nil child before the desired key position + child, state, path, index := findChild(n, parent.index+1, it.path, ancestor) + if child == nil { + // No more children in this fullnode + return parent, it.path, false + } + // If the child we found is already past the seek position, just return it. + if bytes.Compare(path, key) >= 0 { + parent.index = index - 1 + return state, path, true + } + // The child is before the seek position. Try advancing + for { + nextChild, nextState, nextPath, nextIndex := findChild(n, index+1, it.path, ancestor) + // If we run out of children, or skipped past the target, return the + // previous one + if nextChild == nil || bytes.Compare(nextPath, key) >= 0 { + parent.index = index - 1 + return state, path, true + } + // We found a better child closer to the target + state, path, index = nextState, nextPath, nextIndex + } + case *shortNode: + // Short node, return the pointer singleton child + if parent.index < 0 { + hash, _ := n.Val.cache() + state := &nodeIteratorState{ + hash: common.BytesToHash(hash), + node: n.Val, + parent: ancestor, + index: -1, + pathlen: len(it.path), + } + path := append(it.path, n.Key...) + return state, path, true + } + } + return parent, it.path, false +} + func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path []byte) { it.path = path it.stack = append(it.stack, state) diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 75a0a99e51..2518f7bac8 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -18,11 +18,14 @@ package trie import ( "bytes" + "encoding/binary" "fmt" "math/rand" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" ) @@ -440,3 +443,81 @@ func checkIteratorNoDups(t *testing.T, it NodeIterator, seen map[string]bool) in } return len(seen) } + +type loggingDb struct { + getCount uint64 + backend ethdb.KeyValueStore +} + +func (l *loggingDb) Has(key []byte) (bool, error) { + return l.backend.Has(key) +} + +func (l *loggingDb) Get(key []byte) ([]byte, error) { + l.getCount++ + return l.backend.Get(key) +} + +func (l *loggingDb) Put(key []byte, value []byte) error { + return l.backend.Put(key, value) +} + +func (l *loggingDb) Delete(key []byte) error { + return l.backend.Delete(key) +} + +func (l *loggingDb) NewBatch() ethdb.Batch { + return l.backend.NewBatch() +} + +func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { + fmt.Printf("NewIterator\n") + return l.backend.NewIterator(prefix, start) +} +func (l *loggingDb) Stat(property string) (string, error) { + return l.backend.Stat(property) +} + +func (l *loggingDb) Compact(start []byte, limit []byte) error { + return l.backend.Compact(start, limit) +} + +func (l *loggingDb) Close() error { + return l.backend.Close() +} + +// makeLargeTestTrie create a sample test trie +func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { + // Create an empty trie + logDb := &loggingDb{0, memorydb.New()} + triedb := NewDatabase(logDb) + trie, _ := NewSecure(common.Hash{}, triedb) + + // Fill it with some arbitrary data + for i := 0; i < 10000; i++ { + key := make([]byte, 32) + val := make([]byte, 32) + binary.BigEndian.PutUint64(key, uint64(i)) + binary.BigEndian.PutUint64(val, uint64(i)) + key = crypto.Keccak256(key) + val = crypto.Keccak256(val) + trie.Update(key, val) + } + trie.Commit(nil) + // Return the generated trie + return triedb, trie, logDb +} + +// Tests that the node iterator indeed walks over the entire database contents. +func TestNodeIteratorLargeTrie(t *testing.T) { + // Create some arbitrary test trie to iterate + db, trie, logDb := makeLargeTestTrie() + db.Cap(0) // flush everything + // Do a seek operation + trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885")) + // master: 24 get operations + // this pr: 5 get operations + if have, want := logDb.getCount, uint64(5); have != want { + t.Fatalf("Too many lookups during seek, have %d want %d", have, want) + } +} From 67da83aca51d30df9ba2d00b9ec694422626a3ad Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 21 Apr 2021 13:03:33 +0200 Subject: [PATCH 290/709] accounts/external, signer/core: add support for EIP-2930 transactions (#22585) This adds support for signing EIP-2930 with clef. --- accounts/external/backend.go | 14 ++++++++++++++ signer/core/api.go | 8 ++++++++ signer/core/cliui.go | 12 ++++++++++++ signer/core/types.go | 34 +++++++++++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/accounts/external/backend.go b/accounts/external/backend.go index 17a747db0e..de241385c2 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -212,6 +212,20 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio To: to, From: common.NewMixedcaseAddress(account.Address), } + // We should request the default chain id that we're operating with + // (the chain we're executing on) + if chainID != nil { + args.ChainID = (*hexutil.Big)(chainID) + } + // However, if the user asked for a particular chain id, then we should + // use that instead. + if tx.Type() != types.LegacyTxType && tx.ChainId() != nil { + args.ChainID = (*hexutil.Big)(tx.ChainId()) + } + if tx.Type() == types.AccessListTxType { + accessList := tx.AccessList() + args.AccessList = &accessList + } var res signTransactionResult if err := api.client.Call(&res, "account_signTransaction", args); err != nil { return nil, err diff --git a/signer/core/api.go b/signer/core/api.go index 968dcfb2ed..3811162f8f 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -534,6 +534,14 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth return nil, err } } + if args.ChainID != nil { + requestedChainId := (*big.Int)(args.ChainID) + if api.chainID.Cmp(requestedChainId) != 0 { + log.Error("Signing request with wrong chain id", "requested", requestedChainId, "configured", api.chainID) + return nil, fmt.Errorf("requested chainid %d does not match the configuration of the signer", + requestedChainId) + } + } req := SignTxRequest{ Transaction: args, Meta: MetadataFromContext(ctx), diff --git a/signer/core/cliui.go b/signer/core/cliui.go index cbfb56c9df..e0375483c3 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -118,6 +118,18 @@ func (ui *CommandlineUI) ApproveTx(request *SignTxRequest) (SignTxResponse, erro fmt.Printf("gas: %v (%v)\n", request.Transaction.Gas, uint64(request.Transaction.Gas)) fmt.Printf("gasprice: %v wei\n", request.Transaction.GasPrice.ToInt()) fmt.Printf("nonce: %v (%v)\n", request.Transaction.Nonce, uint64(request.Transaction.Nonce)) + if chainId := request.Transaction.ChainID; chainId != nil { + fmt.Printf("chainid: %v\n", chainId) + } + if list := request.Transaction.AccessList; list != nil { + fmt.Printf("Accesslist\n") + for i, el := range *list { + fmt.Printf(" %d. %v\n", i, el.Address) + for j, slot := range el.StorageKeys { + fmt.Printf(" %d. %v\n", j, slot) + } + } + } if request.Transaction.Data != nil { d := *request.Transaction.Data if len(d) > 0 { diff --git a/signer/core/types.go b/signer/core/types.go index 58b377c8d8..e952a21209 100644 --- a/signer/core/types.go +++ b/signer/core/types.go @@ -76,6 +76,10 @@ type SendTxArgs struct { // We accept "data" and "input" for backwards-compatibility reasons. Data *hexutil.Bytes `json:"data"` Input *hexutil.Bytes `json:"input,omitempty"` + + // For non-legacy transactions + AccessList *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` } func (args SendTxArgs) String() string { @@ -93,8 +97,32 @@ func (args *SendTxArgs) toTransaction() *types.Transaction { } else if args.Input != nil { input = *args.Input } - if args.To == nil { - return types.NewContractCreation(uint64(args.Nonce), (*big.Int)(&args.Value), uint64(args.Gas), (*big.Int)(&args.GasPrice), input) + var to *common.Address + if args.To != nil { + _to := args.To.Address() + to = &_to + } + var data types.TxData + if args.AccessList == nil { + data = &types.LegacyTx{ + To: to, + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasPrice: (*big.Int)(&args.GasPrice), + Value: (*big.Int)(&args.Value), + Data: input, + } + } else { + data = &types.AccessListTx{ + To: to, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasPrice: (*big.Int)(&args.GasPrice), + Value: (*big.Int)(&args.Value), + Data: input, + AccessList: *args.AccessList, + } } - return types.NewTransaction(uint64(args.Nonce), args.To.Address(), (*big.Int)(&args.Value), (uint64)(args.Gas), (*big.Int)(&args.GasPrice), input) + return types.NewTx(data) } From 9357280fce5c5d57111d690a336cca5f89e34da6 Mon Sep 17 00:00:00 2001 From: ryanc414 Date: Wed, 21 Apr 2021 14:51:30 +0100 Subject: [PATCH 291/709] rpc: add HTTPError type for HTTP error responses (#22677) The new error type is returned by client operations contains details of the response error code and response body. Co-authored-by: Felix Lange --- rpc/errors.go | 29 +++++++++++++++++++++++++++++ rpc/http.go | 24 +++++++++++++----------- rpc/http_test.go | 39 +++++++++++++++++++++++++++++++++++++++ rpc/types.go | 12 ------------ 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/rpc/errors.go b/rpc/errors.go index dbfde8b196..4c06a745fb 100644 --- a/rpc/errors.go +++ b/rpc/errors.go @@ -18,6 +18,35 @@ package rpc import "fmt" +// HTTPError is returned by client operations when the HTTP status code of the +// response is not a 2xx status. +type HTTPError struct { + StatusCode int + Status string + Body []byte +} + +func (err HTTPError) Error() string { + if len(err.Body) == 0 { + return err.Status + } + return fmt.Sprintf("%v: %s", err.Status, err.Body) +} + +// Error wraps RPC errors, which contain an error code in addition to the message. +type Error interface { + Error() string // returns the message + ErrorCode() int // returns the code +} + +// A DataError contains some data in addition to the error message. +type DataError interface { + Error() string // returns the message + ErrorData() interface{} // returns the error data +} + +// Error types defined below are the built-in JSON-RPC errors. + var ( _ Error = new(methodNotFoundError) _ Error = new(subscriptionNotFoundError) diff --git a/rpc/http.go b/rpc/http.go index 87a96e49ea..32f4e7d90a 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -134,19 +134,11 @@ func DialHTTP(endpoint string) (*Client, error) { func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error { hc := c.writeConn.(*httpConn) respBody, err := hc.doRequest(ctx, msg) - if respBody != nil { - defer respBody.Close() - } - if err != nil { - if respBody != nil { - buf := new(bytes.Buffer) - if _, err2 := buf.ReadFrom(respBody); err2 == nil { - return fmt.Errorf("%v: %v", err, buf.String()) - } - } return err } + defer respBody.Close() + var respmsg jsonrpcMessage if err := json.NewDecoder(respBody).Decode(&respmsg); err != nil { return err @@ -194,7 +186,17 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return resp.Body, errors.New(resp.Status) + var buf bytes.Buffer + var body []byte + if _, err := buf.ReadFrom(resp.Body); err == nil { + body = buf.Bytes() + } + + return nil, HTTPError{ + Status: resp.Status, + StatusCode: resp.StatusCode, + Body: body, + } } return resp.Body, nil } diff --git a/rpc/http_test.go b/rpc/http_test.go index b75af67c52..97f8d44c39 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -123,3 +123,42 @@ func TestHTTPRespBodyUnlimited(t *testing.T) { t.Fatalf("response has wrong length %d, want %d", len(r), respLength) } } + +// Tests that an HTTP error results in an HTTPError instance +// being returned with the expected attributes. +func TestHTTPErrorResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "error has occurred!", http.StatusTeapot) + })) + defer ts.Close() + + c, err := DialHTTP(ts.URL) + if err != nil { + t.Fatal(err) + } + + var r string + err = c.Call(&r, "test_method") + if err == nil { + t.Fatal("error was expected") + } + + httpErr, ok := err.(HTTPError) + if !ok { + t.Fatalf("unexpected error type %T", err) + } + + if httpErr.StatusCode != http.StatusTeapot { + t.Error("unexpected status code", httpErr.StatusCode) + } + if httpErr.Status != "418 I'm a teapot" { + t.Error("unexpected status text", httpErr.Status) + } + if body := string(httpErr.Body); body != "error has occurred!\n" { + t.Error("unexpected body", body) + } + + if errMsg := httpErr.Error(); errMsg != "418 I'm a teapot: error has occurred!\n" { + t.Error("unexpected error message", errMsg) + } +} diff --git a/rpc/types.go b/rpc/types.go index bab1b3957b..d1b878c785 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -35,18 +35,6 @@ type API struct { Public bool // indication if the methods must be considered safe for public use } -// Error wraps RPC errors, which contain an error code in addition to the message. -type Error interface { - Error() string // returns the message - ErrorCode() int // returns the code -} - -// A DataError contains some data in addition to the error message. -type DataError interface { - Error() string // returns the message - ErrorData() interface{} // returns the error data -} - // ServerCodec implements reading, parsing and writing RPC messages for the server side of // a RPC session. Implementations must be go-routine safe since the codec can be called in // multiple go-routines concurrently. From 1fb9a6dd32b581c912d672634882d7e2eb2775cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 22 Apr 2021 11:42:46 +0300 Subject: [PATCH 292/709] eth/protocols, prp/tracker: add support for req/rep rtt tracking (#22608) * eth/protocols, prp/tracker: add support for req/rep rtt tracking * p2p/tracker: sanity cap the number of pending requests * pap/tracker: linter <3 * p2p/tracker: disable entire tracker if no metrics are enabled --- eth/protocols/eth/handler.go | 2 +- eth/protocols/eth/handlers.go | 10 ++ eth/protocols/eth/peer.go | 35 ++++-- eth/protocols/eth/tracker.go | 26 +++++ eth/protocols/snap/handler.go | 8 ++ eth/protocols/snap/peer.go | 7 ++ eth/protocols/snap/tracker.go | 26 +++++ p2p/metrics.go | 3 - p2p/tracker/tracker.go | 203 ++++++++++++++++++++++++++++++++++ 9 files changed, 309 insertions(+), 11 deletions(-) create mode 100644 eth/protocols/eth/tracker.go create mode 100644 eth/protocols/snap/tracker.go create mode 100644 p2p/tracker/tracker.go diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 52dcf94011..6bbaa2f555 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -223,7 +223,7 @@ func handleMessage(backend Backend, peer *Peer) error { if peer.Version() >= ETH66 { handlers = eth66 } - // Track the emount of time it takes to serve the request and run the handler + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) defer func(start time.Time) { diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 8433fa343a..d0dec7b0b2 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -327,6 +327,8 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + requestTracker.Fulfil(peer.id, peer.version, BlockHeadersMsg, res.RequestId) + return backend.Handle(peer, &res.BlockHeadersPacket) } @@ -345,6 +347,8 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + requestTracker.Fulfil(peer.id, peer.version, BlockBodiesMsg, res.RequestId) + return backend.Handle(peer, &res.BlockBodiesPacket) } @@ -363,6 +367,8 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + requestTracker.Fulfil(peer.id, peer.version, NodeDataMsg, res.RequestId) + return backend.Handle(peer, &res.NodeDataPacket) } @@ -381,6 +387,8 @@ func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + requestTracker.Fulfil(peer.id, peer.version, ReceiptsMsg, res.RequestId) + return backend.Handle(peer, &res.ReceiptsPacket) } @@ -506,5 +514,7 @@ func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error } peer.markTransaction(tx.Hash()) } + requestTracker.Fulfil(peer.id, peer.version, PooledTransactionsMsg, txs.RequestId) + return backend.Handle(peer, &txs.PooledTransactionsPacket) } diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 709fca8655..e619c183ba 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -413,8 +413,11 @@ func (p *Peer) RequestOneHeader(hash common.Hash) error { Reverse: false, } if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetBlockHeadersPacket: &query, }) } @@ -432,8 +435,11 @@ func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, re Reverse: reverse, } if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetBlockHeadersPacket: &query, }) } @@ -451,8 +457,11 @@ func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, rever Reverse: reverse, } if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetBlockHeadersPacket: &query, }) } @@ -476,8 +485,11 @@ func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetBlockBodiesPacket: hashes, }) } @@ -489,8 +501,11 @@ func (p *Peer) RequestBodies(hashes []common.Hash) error { func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetNodeDataPacket: hashes, }) } @@ -501,8 +516,11 @@ func (p *Peer) RequestNodeData(hashes []common.Hash) error { func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetReceiptsPacket: hashes, }) } @@ -513,8 +531,11 @@ func (p *Peer) RequestReceipts(hashes []common.Hash) error { func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: rand.Uint64(), + RequestId: id, GetPooledTransactionsPacket: hashes, }) } diff --git a/eth/protocols/eth/tracker.go b/eth/protocols/eth/tracker.go new file mode 100644 index 0000000000..324fd22839 --- /dev/null +++ b/eth/protocols/eth/tracker.go @@ -0,0 +1,26 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "time" + + "github.com/ethereum/go-ethereum/p2p/tracker" +) + +// requestTracker is a singleton tracker for eth/66 and newer request times. +var requestTracker = tracker.New(ProtocolName, 5*time.Minute) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index f7939964f3..4c12adfa81 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -227,6 +227,8 @@ func handleMessage(backend Backend, peer *Peer) error { return fmt.Errorf("accounts not monotonically increasing: #%d [%x] vs #%d [%x]", i-1, res.Accounts[i-1].Hash[:], i, res.Accounts[i].Hash[:]) } } + requestTracker.Fulfil(peer.id, peer.version, AccountRangeMsg, res.ID) + return backend.Handle(peer, res) case msg.Code == GetStorageRangesMsg: @@ -360,6 +362,8 @@ func handleMessage(backend Backend, peer *Peer) error { } } } + requestTracker.Fulfil(peer.id, peer.version, StorageRangesMsg, res.ID) + return backend.Handle(peer, res) case msg.Code == GetByteCodesMsg: @@ -404,6 +408,8 @@ func handleMessage(backend Backend, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + requestTracker.Fulfil(peer.id, peer.version, ByteCodesMsg, res.ID) + return backend.Handle(peer, res) case msg.Code == GetTrieNodesMsg: @@ -497,6 +503,8 @@ func handleMessage(backend Backend, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + requestTracker.Fulfil(peer.id, peer.version, TrieNodesMsg, res.ID) + return backend.Handle(peer, res) default: diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 4f3d550f1f..cf0ce65bd7 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -65,6 +65,8 @@ func (p *Peer) Log() log.Logger { // trie, starting with the origin. func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes uint64) error { p.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) + + requestTracker.Track(p.id, p.version, GetAccountRangeMsg, AccountRangeMsg, id) return p2p.Send(p.rw, GetAccountRangeMsg, &GetAccountRangePacket{ ID: id, Root: root, @@ -83,6 +85,7 @@ func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []comm } else { p.logger.Trace("Fetching ranges of small storage slots", "reqid", id, "root", root, "accounts", len(accounts), "first", accounts[0], "bytes", common.StorageSize(bytes)) } + requestTracker.Track(p.id, p.version, GetStorageRangesMsg, StorageRangesMsg, id) return p2p.Send(p.rw, GetStorageRangesMsg, &GetStorageRangesPacket{ ID: id, Root: root, @@ -96,6 +99,8 @@ func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []comm // RequestByteCodes fetches a batch of bytecodes by hash. func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { p.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) + + requestTracker.Track(p.id, p.version, GetByteCodesMsg, ByteCodesMsg, id) return p2p.Send(p.rw, GetByteCodesMsg, &GetByteCodesPacket{ ID: id, Hashes: hashes, @@ -107,6 +112,8 @@ func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) e // a specificstate trie. func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) + + requestTracker.Track(p.id, p.version, GetTrieNodesMsg, TrieNodesMsg, id) return p2p.Send(p.rw, GetTrieNodesMsg, &GetTrieNodesPacket{ ID: id, Root: root, diff --git a/eth/protocols/snap/tracker.go b/eth/protocols/snap/tracker.go new file mode 100644 index 0000000000..2cf59cc23a --- /dev/null +++ b/eth/protocols/snap/tracker.go @@ -0,0 +1,26 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "time" + + "github.com/ethereum/go-ethereum/p2p/tracker" +) + +// requestTracker is a singleton tracker for request times. +var requestTracker = tracker.New(ProtocolName, time.Minute) diff --git a/p2p/metrics.go b/p2p/metrics.go index be0d2f495e..1bb505cdfb 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -33,9 +33,6 @@ const ( // HandleHistName is the prefix of the per-packet serving time histograms. HandleHistName = "p2p/handle" - - // WaitHistName is the prefix of the per-packet (req only) waiting time histograms. - WaitHistName = "p2p/wait" ) var ( diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go new file mode 100644 index 0000000000..b50a952f62 --- /dev/null +++ b/p2p/tracker/tracker.go @@ -0,0 +1,203 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracker + +import ( + "container/list" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + // trackedGaugeName is the prefix of the per-packet request tracking. + trackedGaugeName = "p2p/tracked" + + // lostMeterName is the prefix of the per-packet request expirations. + lostMeterName = "p2p/lost" + + // staleMeterName is the prefix of the per-packet stale responses. + staleMeterName = "p2p/stale" + + // waitHistName is the prefix of the per-packet (req only) waiting time histograms. + waitHistName = "p2p/wait" + + // maxTrackedPackets is a huge number to act as a failsafe on the number of + // pending requests the node will track. It should never be hit unless an + // attacker figures out a way to spin requests. + maxTrackedPackets = 100000 +) + +// request tracks sent network requests which have not yet received a response. +type request struct { + peer string + version uint // Protocol version + + reqCode uint64 // Protocol message code of the request + resCode uint64 // Protocol message code of the expected response + + time time.Time // Timestamp when the request was made + expire *list.Element // Expiration marker to untrack it +} + +// Tracker is a pending network request tracker to measure how much time it takes +// a remote peer to respond. +type Tracker struct { + protocol string // Protocol capability identifier for the metrics + timeout time.Duration // Global timeout after which to drop a tracked packet + + pending map[uint64]*request // Currently pending requests + expire *list.List // Linked list tracking the expiration order + wake *time.Timer // Timer tracking the expiration of the next item + + lock sync.Mutex // Lock protecting from concurrent updates +} + +// New creates a new network request tracker to monitor how much time it takes to +// fill certain requests and how individual peers perform. +func New(protocol string, timeout time.Duration) *Tracker { + return &Tracker{ + protocol: protocol, + timeout: timeout, + pending: make(map[uint64]*request), + expire: list.New(), + } +} + +// Track adds a network request to the tracker to wait for a response to arrive +// or until the request it cancelled or times out. +func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint64, id uint64) { + if !metrics.Enabled { + return + } + t.lock.Lock() + defer t.lock.Unlock() + + // If there's a duplicate request, we've just random-collided (or more probably, + // we have a bug), report it. We could also add a metric, but we're not really + // expecting ourselves to be buggy, so a noisy warning should be enough. + if _, ok := t.pending[id]; ok { + log.Error("Network request id collision", "protocol", t.protocol, "version", version, "code", reqCode, "id", id) + return + } + // If we have too many pending requests, bail out instead of leaking memory + if pending := len(t.pending); pending >= maxTrackedPackets { + log.Error("Request tracker exceeded allowance", "pending", pending, "peer", peer, "protocol", t.protocol, "version", version, "code", reqCode) + return + } + // Id doesn't exist yet, start tracking it + t.pending[id] = &request{ + peer: peer, + version: version, + reqCode: reqCode, + resCode: resCode, + time: time.Now(), + expire: t.expire.PushBack(id), + } + g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, version, reqCode) + metrics.GetOrRegisterGauge(g, nil).Inc(1) + + // If we've just inserted the first item, start the expiration timer + if t.wake == nil { + t.wake = time.AfterFunc(t.timeout, t.clean) + } +} + +// clean is called automatically when a preset time passes without a response +// being dleivered for the first network request. +func (t *Tracker) clean() { + t.lock.Lock() + defer t.lock.Unlock() + + // Expire anything within a certain threshold (might be no items at all if + // we raced with the delivery) + for t.expire.Len() > 0 { + // Stop iterating if the next pending request is still alive + var ( + head = t.expire.Front() + id = head.Value.(uint64) + req = t.pending[id] + ) + if time.Since(req.time) < t.timeout+5*time.Millisecond { + break + } + // Nope, dead, drop it + t.expire.Remove(head) + delete(t.pending, id) + + g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) + metrics.GetOrRegisterGauge(g, nil).Dec(1) + + m := fmt.Sprintf("%s/%s/%d/%#02x", lostMeterName, t.protocol, req.version, req.reqCode) + metrics.GetOrRegisterMeter(m, nil).Mark(1) + } + t.schedule() +} + +// schedule starts a timer to trigger on the expiration of the first network +// packet. +func (t *Tracker) schedule() { + if t.expire.Len() == 0 { + t.wake = nil + return + } + t.wake = time.AfterFunc(time.Until(t.pending[t.expire.Front().Value.(uint64)].time.Add(t.timeout)), t.clean) +} + +// Fulfil fills a pending request, if any is available, reporting on various metrics. +func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) { + if !metrics.Enabled { + return + } + t.lock.Lock() + defer t.lock.Unlock() + + // If it's a non existing request, track as stale response + req, ok := t.pending[id] + if !ok { + m := fmt.Sprintf("%s/%s/%d/%#02x", staleMeterName, t.protocol, version, code) + metrics.GetOrRegisterMeter(m, nil).Mark(1) + return + } + // If the response is funky, it might be some active attack + if req.peer != peer || req.version != version || req.resCode != code { + log.Warn("Network response id collision", + "have", fmt.Sprintf("%s:%s/%d:%d", peer, t.protocol, version, code), + "want", fmt.Sprintf("%s:%s/%d:%d", peer, t.protocol, req.version, req.resCode), + ) + return + } + // Everything matches, mark the request serviced and meter it + t.expire.Remove(req.expire) + if req.expire.Prev() == nil { + t.wake.Stop() + t.schedule() + } + g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) + metrics.GetOrRegisterGauge(g, nil).Dec(1) + + h := fmt.Sprintf("%s/%s/%d/%#02x", waitHistName, t.protocol, req.version, req.reqCode) + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) + } + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(req.time).Microseconds()) +} From ea54c58d4f37d8cd4563332989caf943f4964333 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 23 Apr 2021 11:15:42 +0200 Subject: [PATCH 293/709] cmd/devp2p/internal/ethtest: run test suite as Go unit test (#22698) This change adds a Go unit test that runs the protocol test suite against the go-ethereum implementation of the eth protocol. --- cmd/devp2p/internal/ethtest/chain.go | 29 ++++- cmd/devp2p/internal/ethtest/eth66_suite.go | 46 ++++++-- .../internal/ethtest/eth66_suiteHelpers.go | 6 +- cmd/devp2p/internal/ethtest/suite.go | 104 ++++++++++++------ cmd/devp2p/internal/ethtest/suite_test.go | 99 +++++++++++++++++ cmd/devp2p/internal/ethtest/transaction.go | 56 ++++++---- cmd/devp2p/internal/ethtest/types.go | 3 +- 7 files changed, 269 insertions(+), 74 deletions(-) create mode 100644 cmd/devp2p/internal/ethtest/suite_test.go diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index d67387e80b..83c55181ad 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -34,6 +34,7 @@ import ( ) type Chain struct { + genesis core.Genesis blocks []*types.Block chainConfig *params.ChainConfig } @@ -124,16 +125,34 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { // loadChain takes the given chain.rlp file, and decodes and returns // the blocks from the file. func loadChain(chainfile string, genesis string) (*Chain, error) { - chainConfig, err := ioutil.ReadFile(genesis) + gen, err := loadGenesis(genesis) if err != nil { return nil, err } + gblock := gen.ToBlock(nil) + + blocks, err := blocksFromFile(chainfile, gblock) + if err != nil { + return nil, err + } + + c := &Chain{genesis: gen, blocks: blocks, chainConfig: gen.Config} + return c, nil +} + +func loadGenesis(genesisFile string) (core.Genesis, error) { + chainConfig, err := ioutil.ReadFile(genesisFile) + if err != nil { + return core.Genesis{}, err + } var gen core.Genesis if err := json.Unmarshal(chainConfig, &gen); err != nil { - return nil, err + return core.Genesis{}, err } - gblock := gen.ToBlock(nil) + return gen, nil +} +func blocksFromFile(chainfile string, gblock *types.Block) ([]*types.Block, error) { // Load chain.rlp. fh, err := os.Open(chainfile) if err != nil { @@ -161,7 +180,5 @@ func loadChain(chainfile string, genesis string) (*Chain, error) { } blocks = append(blocks, &b) } - - c := &Chain{blocks: blocks, chainConfig: gen.Config} - return c, nil + return blocks, nil } diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 0995dcb3e4..41a4246f30 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -41,6 +41,7 @@ func (s *Suite) Is_66(t *utesting.T) { // make sure the chain head is correct. func (s *Suite) TestStatus_66(t *utesting.T) { conn := s.dial66(t) + defer conn.Close() // get protoHandshake conn.handshake(t) // get status @@ -60,6 +61,7 @@ func (s *Suite) TestStatus_66(t *utesting.T) { // an eth66 `GetBlockHeaders` request and that the response is accurate. func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) { conn := s.setupConnection66(t) + defer conn.Close() // get block headers req := ð.GetBlockHeadersPacket66{ RequestId: 3, @@ -84,6 +86,8 @@ func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) { func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { // create two connections conn1, conn2 := s.setupConnection66(t), s.setupConnection66(t) + defer conn1.Close() + defer conn2.Close() // create two requests req1 := ð.GetBlockHeadersPacket66{ RequestId: 111, @@ -122,6 +126,9 @@ func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { // propagated to the given node's peer(s) on the eth66 protocol. func (s *Suite) TestBroadcast_66(t *utesting.T) { sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) + defer sendConn.Close() + defer receiveConn.Close() + nextBlock := len(s.chain.blocks) blockAnnouncement := &NewBlock{ Block: s.fullChain.blocks[nextBlock], @@ -141,6 +148,7 @@ func (s *Suite) TestBroadcast_66(t *utesting.T) { // the eth66 protocol. func (s *Suite) TestGetBlockBodies_66(t *utesting.T) { conn := s.setupConnection66(t) + defer conn.Close() // create block bodies request id := uint64(55) req := ð.GetBlockBodiesPacket66{ @@ -195,17 +203,20 @@ func (s *Suite) TestLargeAnnounce_66(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) { + switch msg := sendConn.ReadAndServe(s.chain, time.Second*8).(type) { case *Disconnect: case *Error: break default: t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) } + sendConn.Close() } // Test the last block as a valid block - sendConn := s.setupConnection66(t) - receiveConn := s.setupConnection66(t) + sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) + defer sendConn.Close() + defer receiveConn.Close() + s.testAnnounce66(t, sendConn, receiveConn, blocks[3]) // update test suite chain s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) @@ -216,12 +227,17 @@ func (s *Suite) TestLargeAnnounce_66(t *utesting.T) { } func (s *Suite) TestOldAnnounce_66(t *utesting.T) { - s.oldAnnounce(t, s.setupConnection66(t), s.setupConnection66(t)) + sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t) + defer sendConn.Close() + defer recvConn.Close() + + s.oldAnnounce(t, sendConn, recvConn) } // TestMaliciousHandshake_66 tries to send malicious data during the handshake. func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) { conn := s.dial66(t) + defer conn.Close() // write hello to client pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] handshakes := []*Hello{ @@ -295,6 +311,7 @@ func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) { // TestMaliciousStatus_66 sends a status package with a large total difficulty. func (s *Suite) TestMaliciousStatus_66(t *utesting.T) { conn := s.dial66(t) + defer conn.Close() // get protoHandshake conn.handshake(t) status := &Status{ @@ -334,23 +351,37 @@ func (s *Suite) TestTransaction_66(t *utesting.T) { } func (s *Suite) TestMaliciousTx_66(t *utesting.T) { - tests := []*types.Transaction{ + badTxs := []*types.Transaction{ getOldTxFromChain(t, s), invalidNonceTx(t, s), hugeAmount(t, s), hugeGasPrice(t, s), hugeData(t, s), } - for i, tx := range tests { + sendConn := s.setupConnection66(t) + defer sendConn.Close() + // set up receiving connection before sending txs to make sure + // no announcements are missed + recvConn := s.setupConnection66(t) + defer recvConn.Close() + + for i, tx := range badTxs { t.Logf("Testing malicious tx propagation: %v\n", i) - sendFailingTx66(t, s, tx) + if err := sendConn.Write(&Transactions{tx}); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + } + // check to make sure bad txs aren't propagated + waitForTxPropagation(t, s, badTxs, recvConn) } // TestZeroRequestID_66 checks that a request ID of zero is still handled // by the node. func (s *Suite) TestZeroRequestID_66(t *utesting.T) { conn := s.setupConnection66(t) + defer conn.Close() + req := ð.GetBlockHeadersPacket66{ RequestId: 0, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ @@ -367,6 +398,7 @@ func (s *Suite) TestZeroRequestID_66(t *utesting.T) { // concurrently to a single node. func (s *Suite) TestSameRequestID_66(t *utesting.T) { conn := s.setupConnection66(t) + defer conn.Close() // create two separate requests with same ID reqID := uint64(1234) req1 := ð.GetBlockHeadersPacket66{ diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go index 4ef349740f..40427fcd30 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -238,14 +238,10 @@ func (c *Conn) waitForBlock66(block *types.Block) error { func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) { sendConn := s.setupConnection66(t) + defer sendConn.Close() sendSuccessfulTxWithConn(t, s, tx, sendConn) } -func sendFailingTx66(t *utesting.T, s *Suite, tx *types.Transaction) { - sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t) - sendFailingTxWithConns(t, s, tx, sendConn, recvConn) -} - func (s *Suite) getBlockHeaders66(t *utesting.T, conn *Conn, req eth.Packet, expectedID uint64) BlockHeaders { if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil { t.Fatalf("could not write to connection: %v", err) diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 66fb8026a0..1cae16b7d5 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -69,20 +69,20 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e func (s *Suite) AllEthTests() []utesting.Test { return []utesting.Test{ // status - {Name: "Status", Fn: s.TestStatus}, - {Name: "Status_66", Fn: s.TestStatus_66}, + {Name: "TestStatus", Fn: s.TestStatus}, + {Name: "TestStatus_66", Fn: s.TestStatus_66}, // get block headers - {Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders}, - {Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, + {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "TestGetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, {Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66}, {Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66}, {Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66}, // get block bodies - {Name: "GetBlockBodies", Fn: s.TestGetBlockBodies}, - {Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, + {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, + {Name: "TestGetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, // broadcast - {Name: "Broadcast", Fn: s.TestBroadcast}, - {Name: "Broadcast_66", Fn: s.TestBroadcast_66}, + {Name: "TestBroadcast", Fn: s.TestBroadcast}, + {Name: "TestBroadcast_66", Fn: s.TestBroadcast_66}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, @@ -91,44 +91,44 @@ func (s *Suite) AllEthTests() []utesting.Test { {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, {Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66}, - {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus}, + {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66}, // test transactions - {Name: "TestTransactions", Fn: s.TestTransaction}, - {Name: "TestTransactions_66", Fn: s.TestTransaction_66}, - {Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx}, - {Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66}, + {Name: "TestTransaction", Fn: s.TestTransaction}, + {Name: "TestTransaction_66", Fn: s.TestTransaction_66}, + {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, + {Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66}, } } func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ - {Name: "Status", Fn: s.TestStatus}, - {Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders}, - {Name: "GetBlockBodies", Fn: s.TestGetBlockBodies}, - {Name: "Broadcast", Fn: s.TestBroadcast}, + {Name: "TestStatus", Fn: s.TestStatus}, + {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, + {Name: "TestBroadcast", Fn: s.TestBroadcast}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, - {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus}, - {Name: "TestTransactions", Fn: s.TestTransaction}, - {Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx}, + {Name: "TestTransaction", Fn: s.TestTransaction}, + {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, } } func (s *Suite) Eth66Tests() []utesting.Test { return []utesting.Test{ // only proceed with eth66 test suite if node supports eth 66 protocol - {Name: "Status_66", Fn: s.TestStatus_66}, - {Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, + {Name: "TestStatus_66", Fn: s.TestStatus_66}, + {Name: "TestGetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, {Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66}, {Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66}, {Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66}, - {Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, - {Name: "Broadcast_66", Fn: s.TestBroadcast_66}, + {Name: "TestGetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, + {Name: "TestBroadcast_66", Fn: s.TestBroadcast_66}, {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, {Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66}, - {Name: "TestTransactions_66", Fn: s.TestTransaction_66}, - {Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66}, + {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66}, + {Name: "TestTransaction_66", Fn: s.TestTransaction_66}, + {Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66}, } } @@ -140,6 +140,7 @@ func (s *Suite) TestStatus(t *utesting.T) { if err != nil { t.Fatalf("could not dial: %v", err) } + defer conn.Close() // get protoHandshake conn.handshake(t) // get status @@ -157,6 +158,7 @@ func (s *Suite) TestMaliciousStatus(t *utesting.T) { if err != nil { t.Fatalf("could not dial: %v", err) } + defer conn.Close() // get protoHandshake conn.handshake(t) status := &Status{ @@ -191,6 +193,7 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { if err != nil { t.Fatalf("could not dial: %v", err) } + defer conn.Close() conn.handshake(t) conn.statusExchange(t, s.chain, nil) @@ -229,6 +232,7 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { if err != nil { t.Fatalf("could not dial: %v", err) } + defer conn.Close() conn.handshake(t) conn.statusExchange(t, s.chain, nil) @@ -253,6 +257,9 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { // propagated to the given node's peer(s). func (s *Suite) TestBroadcast(t *utesting.T) { sendConn, receiveConn := s.setupConnection(t), s.setupConnection(t) + defer sendConn.Close() + defer receiveConn.Close() + nextBlock := len(s.chain.blocks) blockAnnouncement := &NewBlock{ Block: s.fullChain.blocks[nextBlock], @@ -273,6 +280,7 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) { if err != nil { t.Fatalf("could not dial: %v", err) } + defer conn.Close() // write hello to client pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] handshakes := []*Hello{ @@ -372,17 +380,21 @@ func (s *Suite) TestLargeAnnounce(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) { + switch msg := sendConn.ReadAndServe(s.chain, time.Second*8).(type) { case *Disconnect: case *Error: break default: t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) } + sendConn.Close() } // Test the last block as a valid block sendConn := s.setupConnection(t) receiveConn := s.setupConnection(t) + defer sendConn.Close() + defer receiveConn.Close() + s.testAnnounce(t, sendConn, receiveConn, blocks[3]) // update test suite chain s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) @@ -393,7 +405,11 @@ func (s *Suite) TestLargeAnnounce(t *utesting.T) { } func (s *Suite) TestOldAnnounce(t *utesting.T) { - s.oldAnnounce(t, s.setupConnection(t), s.setupConnection(t)) + sendConn, recvConn := s.setupConnection(t), s.setupConnection(t) + defer sendConn.Close() + defer recvConn.Close() + + s.oldAnnounce(t, sendConn, recvConn) } func (s *Suite) oldAnnounce(t *utesting.T, sendConn, receiveConn *Conn) { @@ -406,11 +422,19 @@ func (s *Suite) oldAnnounce(t *utesting.T, sendConn, receiveConn *Conn) { t.Fatalf("could not write to connection: %v", err) } - switch msg := receiveConn.ReadAndServe(s.chain, timeout*2).(type) { + switch msg := receiveConn.ReadAndServe(s.chain, time.Second*8).(type) { case *NewBlock: - t.Fatalf("unexpected: block propagated: %s", pretty.Sdump(msg)) + block := *msg + if block.Block.Hash() == oldBlockAnnounce.Block.Hash() { + t.Fatalf("unexpected: block propagated: %s", pretty.Sdump(msg)) + } case *NewBlockHashes: - t.Fatalf("unexpected: block announced: %s", pretty.Sdump(msg)) + hashes := *msg + for _, hash := range hashes { + if hash.Hash == oldBlockAnnounce.Block.Hash() { + t.Fatalf("unexpected: block announced: %s", pretty.Sdump(msg)) + } + } case *Error: errMsg := *msg // check to make sure error is timeout (propagation didn't come through == test successful) @@ -502,15 +526,27 @@ func (s *Suite) TestTransaction(t *utesting.T) { } func (s *Suite) TestMaliciousTx(t *utesting.T) { - tests := []*types.Transaction{ + badTxs := []*types.Transaction{ getOldTxFromChain(t, s), invalidNonceTx(t, s), hugeAmount(t, s), hugeGasPrice(t, s), hugeData(t, s), } - for i, tx := range tests { + sendConn := s.setupConnection(t) + defer sendConn.Close() + // set up receiving connection before sending txs to make sure + // no announcements are missed + recvConn := s.setupConnection(t) + defer recvConn.Close() + + for i, tx := range badTxs { t.Logf("Testing malicious tx propagation: %v\n", i) - sendFailingTx(t, s, tx) + if err := sendConn.Write(&Transactions{tx}); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + } + // check to make sure bad txs aren't propagated + waitForTxPropagation(t, s, badTxs, recvConn) } diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go new file mode 100644 index 0000000000..2c628757bc --- /dev/null +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -0,0 +1,99 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethtest + +import ( + "os" + "testing" + + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" +) + +var ( + genesisFile = "./testdata/genesis.json" + halfchainFile = "./testdata/halfchain.rlp" + fullchainFile = "./testdata/chain.rlp" +) + +func TestEthSuite(t *testing.T) { + geth, err := runGeth() + if err != nil { + t.Fatalf("could not run geth: %v", err) + } + defer geth.Close() + + suite, err := NewSuite(geth.Server().Self(), fullchainFile, genesisFile) + if err != nil { + t.Fatalf("could not create new test suite: %v", err) + } + for _, test := range suite.AllEthTests() { + t.Run(test.Name, func(t *testing.T) { + result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) + if result[0].Failed { + t.Fatal() + } + }) + } +} + +// runGeth creates and starts a geth node +func runGeth() (*node.Node, error) { + stack, err := node.New(&node.Config{ + P2P: p2p.Config{ + ListenAddr: "127.0.0.1:0", + NoDiscovery: true, + MaxPeers: 10, // in case a test requires multiple connections, can be changed in the future + NoDial: true, + }, + }) + if err != nil { + return nil, err + } + + err = setupGeth(stack) + if err != nil { + stack.Close() + return nil, err + } + if err = stack.Start(); err != nil { + stack.Close() + return nil, err + } + return stack, nil +} + +func setupGeth(stack *node.Node) error { + chain, err := loadChain(halfchainFile, genesisFile) + if err != nil { + return err + } + + backend, err := eth.New(stack, ðconfig.Config{ + Genesis: &chain.genesis, + NetworkId: chain.genesis.Config.ChainID.Uint64(), // 19763 + }) + if err != nil { + return err + } + + _, err = backend.BlockChain().InsertChain(chain.blocks[1:]) + return err +} diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 21aa221e8b..f8b0a9da81 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -30,6 +30,7 @@ var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c666 func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) { sendConn := s.setupConnection(t) + defer sendConn.Close() sendSuccessfulTxWithConn(t, s, tx, sendConn) } @@ -65,29 +66,30 @@ func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, se } } -func sendFailingTx(t *utesting.T, s *Suite, tx *types.Transaction) { - sendConn, recvConn := s.setupConnection(t), s.setupConnection(t) - sendFailingTxWithConns(t, s, tx, sendConn, recvConn) -} - -func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, sendConn, recvConn *Conn) { - // Wait for a transaction announcement - switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { - case *NewPooledTransactionHashes: - break - default: - t.Logf("unexpected message, logging: %v", pretty.Sdump(msg)) - } - // Send the transaction - if err := sendConn.Write(&Transactions{tx}); err != nil { - t.Fatal(err) - } +func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, recvConn *Conn) { // Wait for another transaction announcement - switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { + switch msg := recvConn.ReadAndServe(s.chain, time.Second*8).(type) { case *Transactions: - t.Fatalf("Received unexpected transaction announcement: %v", msg) + // check to see if any of the failing txs were in the announcement + recvTxs := make([]common.Hash, len(*msg)) + for i, recvTx := range *msg { + recvTxs[i] = recvTx.Hash() + } + badTxs := containsTxs(recvTxs, txs) + if len(badTxs) > 0 { + for _, tx := range badTxs { + t.Logf("received bad tx: %v", tx) + } + t.Fatalf("received %d bad txs", len(badTxs)) + } case *NewPooledTransactionHashes: - t.Fatalf("Received unexpected pooledTx announcement: %v", msg) + badTxs := containsTxs(*msg, txs) + if len(badTxs) > 0 { + for _, tx := range badTxs { + t.Logf("received bad tx: %v", tx) + } + t.Fatalf("received %d bad txs", len(badTxs)) + } case *Error: // Transaction should not be announced -> wait for timeout return @@ -96,6 +98,20 @@ func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, send } } +// containsTxs checks whether the hashes of the received transactions are present in +// the given set of txs +func containsTxs(recvTxs []common.Hash, txs []*types.Transaction) []common.Hash { + containedTxs := make([]common.Hash, 0) + for _, recvTx := range recvTxs { + for _, tx := range txs { + if recvTx == tx.Hash() { + containedTxs = append(containedTxs, recvTx) + } + } + } + return containedTxs +} + func unknownTx(t *utesting.T, s *Suite) *types.Transaction { tx := getNextTxFromChain(t, s) var to common.Address diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index 1e2ae77965..50108f2dc3 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -178,8 +178,7 @@ func (c *Conn) Read() Message { func (c *Conn) ReadAndServe(chain *Chain, timeout time.Duration) Message { start := time.Now() for time.Since(start) < timeout { - timeout := time.Now().Add(10 * time.Second) - c.SetReadDeadline(timeout) + c.SetReadDeadline(time.Now().Add(5 * time.Second)) switch msg := c.Read().(type) { case *Ping: c.Write(&Pong{}) From 49281ab84fa5d2dd5704ce35ad984d415b529c4f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 23 Apr 2021 13:39:18 +0200 Subject: [PATCH 294/709] core/state/snapshot, true: reuse dirty data instead of hitting disk when generating (#22667) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * core/state/snapshot: reuse memory data instead of hitting disk when generating * trie: minor nitpicks wrt the resolver optimization * core/state/snapshot, trie: use key/value store for resolver * trie: fix linter Co-authored-by: Péter Szilágyi --- core/state/snapshot/generate.go | 20 ++++++++++++- trie/iterator.go | 50 +++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index ed431fcb3d..13b34f4d69 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" @@ -434,6 +435,20 @@ func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, } meter.Mark(1) } + + // We use the snap data to build up a cache which can be used by the + // main account trie as a primary lookup when resolving hashes + var snapNodeCache ethdb.KeyValueStore + if len(result.keys) > 0 { + snapNodeCache = memorydb.New() + snapTrieDb := trie.NewDatabase(snapNodeCache) + snapTrie, _ := trie.New(common.Hash{}, snapTrieDb) + for i, key := range result.keys { + snapTrie.Update(key, result.vals[i]) + } + root, _ := snapTrie.Commit(nil) + snapTrieDb.Commit(root, false, nil) + } tr := result.tr if tr == nil { tr, err = trie.New(root, dl.triedb) @@ -442,9 +457,11 @@ func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, return false, nil, errMissingTrie } } + var ( trieMore bool - iter = trie.NewIterator(tr.NodeIterator(origin)) + nodeIt = tr.NodeIterator(origin) + iter = trie.NewIterator(nodeIt) kvkeys, kvvals = result.keys, result.vals // counters @@ -458,6 +475,7 @@ func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, start = time.Now() internal time.Duration ) + nodeIt.AddResolver(snapNodeCache) for iter.Next() { if last != nil && bytes.Compare(iter.Key, last) > 0 { trieMore = true diff --git a/trie/iterator.go b/trie/iterator.go index 4f72258a1d..406f216c22 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" ) @@ -102,6 +103,19 @@ type NodeIterator interface { // iterator is not positioned at a leaf. Callers must not retain references // to the value after calling Next. LeafProof() [][]byte + + // AddResolver sets an intermediate database to use for looking up trie nodes + // before reaching into the real persistent layer. + // + // This is not required for normal operation, rather is an optimization for + // cases where trie nodes can be recovered from some external mechanism without + // reading from disk. In those cases, this resolver allows short circuiting + // accesses and returning them from memory. + // + // Before adding a similar mechanism to any other place in Geth, consider + // making trie.Database an interface and wrapping at that level. It's a huge + // refactor, but it could be worth it if another occurrence arises. + AddResolver(ethdb.KeyValueStore) } // nodeIteratorState represents the iteration state at one particular node of the @@ -119,6 +133,8 @@ type nodeIterator struct { stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state path []byte // Path to the current node err error // Failure set in case of an internal error in the iterator + + resolver ethdb.KeyValueStore // Optional intermediate resolver above the disk layer } // errIteratorEnd is stored in nodeIterator.err when iteration is done. @@ -143,6 +159,10 @@ func newNodeIterator(trie *Trie, start []byte) NodeIterator { return it } +func (it *nodeIterator) AddResolver(resolver ethdb.KeyValueStore) { + it.resolver = resolver +} + func (it *nodeIterator) Hash() common.Hash { if len(it.stack) == 0 { return common.Hash{} @@ -262,7 +282,7 @@ func (it *nodeIterator) init() (*nodeIteratorState, error) { if root != emptyRoot { state.hash = root } - return state, state.resolve(it.trie, nil) + return state, state.resolve(it, nil) } // peek creates the next state of the iterator. @@ -286,7 +306,7 @@ func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, er } state, path, ok := it.nextChild(parent, ancestor) if ok { - if err := state.resolve(it.trie, path); err != nil { + if err := state.resolve(it, path); err != nil { return parent, &parent.index, path, err } return state, &parent.index, path, nil @@ -319,7 +339,7 @@ func (it *nodeIterator) peekSeek(seekKey []byte) (*nodeIteratorState, *int, []by } state, path, ok := it.nextChildAt(parent, ancestor, seekKey) if ok { - if err := state.resolve(it.trie, path); err != nil { + if err := state.resolve(it, path); err != nil { return parent, &parent.index, path, err } return state, &parent.index, path, nil @@ -330,9 +350,21 @@ func (it *nodeIterator) peekSeek(seekKey []byte) (*nodeIteratorState, *int, []by return nil, nil, nil, errIteratorEnd } -func (st *nodeIteratorState) resolve(tr *Trie, path []byte) error { +func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { + if it.resolver != nil { + if blob, err := it.resolver.Get(hash); err == nil && len(blob) > 0 { + if resolved, err := decodeNode(hash, blob); err == nil { + return resolved, nil + } + } + } + resolved, err := it.trie.resolveHash(hash, path) + return resolved, err +} + +func (st *nodeIteratorState) resolve(it *nodeIterator, path []byte) error { if hash, ok := st.node.(hashNode); ok { - resolved, err := tr.resolveHash(hash, path) + resolved, err := it.resolveHash(hash, path) if err != nil { return err } @@ -517,6 +549,10 @@ func (it *differenceIterator) Path() []byte { return it.b.Path() } +func (it *differenceIterator) AddResolver(resolver ethdb.KeyValueStore) { + panic("not implemented") +} + func (it *differenceIterator) Next(bool) bool { // Invariants: // - We always advance at least one element in b. @@ -624,6 +660,10 @@ func (it *unionIterator) Path() []byte { return (*it.items)[0].Path() } +func (it *unionIterator) AddResolver(resolver ethdb.KeyValueStore) { + panic("not implemented") +} + // Next returns the next node in the union of tries being iterated over. // // It does this by maintaining a heap of iterators, sorted by the iteration From cac1b21d392370f8768a9ee45a9a10c0b5ddcc9b Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 23 Apr 2021 18:14:39 +0200 Subject: [PATCH 295/709] cmd/devp2p/internal/ethtest: add more tx propagation tests (#22630) This adds a test for large tx announcement messages, as well as a test to check that announced tx hashes are requested by the node. --- cmd/devp2p/internal/ethtest/eth66_suite.go | 97 ++++++++++--- .../internal/ethtest/eth66_suiteHelpers.go | 90 +++++++++--- cmd/devp2p/internal/ethtest/suite.go | 5 +- cmd/devp2p/internal/ethtest/transaction.go | 132 +++++++++++++++--- cmd/devp2p/internal/ethtest/types.go | 12 ++ 5 files changed, 278 insertions(+), 58 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 41a4246f30..176d8bf33c 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -19,6 +19,7 @@ package ethtest import ( "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -125,22 +126,7 @@ func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { // TestBroadcast_66 tests whether a block announcement is correctly // propagated to the given node's peer(s) on the eth66 protocol. func (s *Suite) TestBroadcast_66(t *utesting.T) { - sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) - defer sendConn.Close() - defer receiveConn.Close() - - nextBlock := len(s.chain.blocks) - blockAnnouncement := &NewBlock{ - Block: s.fullChain.blocks[nextBlock], - TD: s.fullChain.TD(nextBlock + 1), - } - s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement) - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) - // wait for client to update its chain - if err := receiveConn.waitForBlock66(s.chain.Head()); err != nil { - t.Fatal(err) - } + s.sendNextBlock66(t) } // TestGetBlockBodies_66 tests whether the given node can respond to @@ -426,3 +412,82 @@ func (s *Suite) TestSameRequestID_66(t *utesting.T) { // check response from first request headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req1, reqID)) } + +// TestLargeTxRequest_66 tests whether a node can fulfill a large GetPooledTransactions +// request. +func (s *Suite) TestLargeTxRequest_66(t *utesting.T) { + // send the next block to ensure the node is no longer syncing and is able to accept + // txs + s.sendNextBlock66(t) + // send 2000 transactions to the node + hashMap, txs := generateTxs(t, s, 2000) + sendConn := s.setupConnection66(t) + defer sendConn.Close() + + sendMultipleSuccessfulTxs(t, s, sendConn, txs) + // set up connection to receive to ensure node is peered with the receiving connection + // before tx request is sent + recvConn := s.setupConnection66(t) + defer recvConn.Close() + // create and send pooled tx request + hashes := make([]common.Hash, 0) + for _, hash := range hashMap { + hashes = append(hashes, hash) + } + getTxReq := ð.GetPooledTransactionsPacket66{ + RequestId: 1234, + GetPooledTransactionsPacket: hashes, + } + if err := recvConn.write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { + t.Fatalf("could not write to conn: %v", err) + } + // check that all received transactions match those that were sent to node + switch msg := recvConn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { + case PooledTransactions: + for _, gotTx := range msg { + if _, exists := hashMap[gotTx.Hash()]; !exists { + t.Fatalf("unexpected tx received: %v", gotTx.Hash()) + } + } + default: + t.Fatalf("unexpected %s", pretty.Sdump(msg)) + } +} + +// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions +// request upon receiving a NewPooledTransactionHashes announcement. +func (s *Suite) TestNewPooledTxs_66(t *utesting.T) { + // send the next block to ensure the node is no longer syncing and is able to accept + // txs + s.sendNextBlock66(t) + // generate 50 txs + hashMap, _ := generateTxs(t, s, 50) + // create new pooled tx hashes announcement + hashes := make([]common.Hash, 0) + for _, hash := range hashMap { + hashes = append(hashes, hash) + } + announce := NewPooledTransactionHashes(hashes) + // send announcement + conn := s.setupConnection66(t) + defer conn.Close() + if err := conn.Write(announce); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + // wait for GetPooledTxs request + for { + _, msg := conn.readAndServe66(s.chain, timeout) + switch msg := msg.(type) { + case GetPooledTransactions: + if len(msg) != len(hashes) { + t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) + } + return + case *NewPooledTransactionHashes: + // ignore propagated txs from old tests + continue + default: + t.Fatalf("unexpected %s", pretty.Sdump(msg)) + } + } +} diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go index 40427fcd30..3af8295c61 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -111,6 +111,18 @@ func (c *Conn) read66() (uint64, Message) { msg = new(Transactions) case (NewPooledTransactionHashes{}).Code(): msg = new(NewPooledTransactionHashes) + case (GetPooledTransactions{}.Code()): + ethMsg := new(eth.GetPooledTransactionsPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) + case (PooledTransactions{}.Code()): + ethMsg := new(eth.PooledTransactionsPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) default: msg = errorf("invalid message code: %d", code) } @@ -124,6 +136,15 @@ func (c *Conn) read66() (uint64, Message) { return 0, errorf("invalid message: %s", string(rawData)) } +func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { + for { + id, msg := c.readAndServe66(chain, timeout) + if id == requestID { + return msg + } + } +} + // ReadAndServe serves GetBlockHeaders requests while waiting // on another message from the node. func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { @@ -173,27 +194,33 @@ func (s *Suite) testAnnounce66(t *utesting.T, sendConn, receiveConn *Conn, block } func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) { - timeout := 20 * time.Second - _, msg := conn.readAndServe66(s.chain, timeout) - switch msg := msg.(type) { - case *NewBlock: - t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block)) - assert.Equal(t, - blockAnnouncement.Block.Header(), msg.Block.Header(), - "wrong block header in announcement", - ) - assert.Equal(t, - blockAnnouncement.TD, msg.TD, - "wrong TD in announcement", - ) - case *NewBlockHashes: - blockHashes := *msg - t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes)) - assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash, - "wrong block hash in announcement", - ) - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + for { + _, msg := conn.readAndServe66(s.chain, timeout) + switch msg := msg.(type) { + case *NewBlock: + t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block)) + assert.Equal(t, + blockAnnouncement.Block.Header(), msg.Block.Header(), + "wrong block header in announcement", + ) + assert.Equal(t, + blockAnnouncement.TD, msg.TD, + "wrong TD in announcement", + ) + return + case *NewBlockHashes: + blockHashes := *msg + t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes)) + assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash, + "wrong block hash in announcement", + ) + return + case *NewPooledTransactionHashes: + // ignore old txs being propagated + continue + default: + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } } } @@ -268,3 +295,24 @@ func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) { assert.Equal(t, chain.blocks[int(num)].Header(), header) } } + +func (s *Suite) sendNextBlock66(t *utesting.T) { + sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) + defer sendConn.Close() + defer receiveConn.Close() + + // create new block announcement + nextBlock := len(s.chain.blocks) + blockAnnouncement := &NewBlock{ + Block: s.fullChain.blocks[nextBlock], + TD: s.fullChain.TD(nextBlock + 1), + } + // send announcement and wait for node to request the header + s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement) + // update test suite chain + s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) + // wait for client to update its chain + if err := receiveConn.waitForBlock66(s.chain.Head()); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 1cae16b7d5..2fa31ad31d 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -97,6 +97,8 @@ func (s *Suite) AllEthTests() []utesting.Test { {Name: "TestTransaction_66", Fn: s.TestTransaction_66}, {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, {Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66}, + {Name: "TestLargeTxRequest_66", Fn: s.TestLargeTxRequest_66}, + {Name: "TestNewPooledTxs_66", Fn: s.TestNewPooledTxs_66}, } } @@ -129,6 +131,8 @@ func (s *Suite) Eth66Tests() []utesting.Test { {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66}, {Name: "TestTransaction_66", Fn: s.TestTransaction_66}, {Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66}, + {Name: "TestLargeTxRequest_66", Fn: s.TestLargeTxRequest_66}, + {Name: "TestNewPooledTxs_66", Fn: s.TestNewPooledTxs_66}, } } @@ -455,7 +459,6 @@ func (s *Suite) testAnnounce(t *utesting.T, sendConn, receiveConn *Conn, blockAn } func (s *Suite) waitAnnounce(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) { - timeout := 20 * time.Second switch msg := conn.ReadAndServe(s.chain, timeout).(type) { case *NewBlock: t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block)) diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index f8b0a9da81..a6166bd2e3 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -17,12 +17,15 @@ package ethtest import ( + "math/big" + "strings" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/params" ) //var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") @@ -40,7 +43,9 @@ func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, se if err := sendConn.Write(&Transactions{tx}); err != nil { t.Fatal(err) } - time.Sleep(100 * time.Millisecond) + // update last nonce seen + nonce = tx.Nonce() + recvConn := s.setupConnection(t) // Wait for the transaction announcement switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { @@ -66,6 +71,60 @@ func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, se } } +var nonce = uint64(99) + +func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*types.Transaction) { + txMsg := Transactions(txs) + t.Logf("sending %d txs\n", len(txs)) + + recvConn := s.setupConnection(t) + defer recvConn.Close() + + // Send the transactions + if err := sendConn.Write(&txMsg); err != nil { + t.Fatal(err) + } + // update nonce + nonce = txs[len(txs)-1].Nonce() + // Wait for the transaction announcement(s) and make sure all sent txs are being propagated + recvHashes := make([]common.Hash, 0) + // all txs should be announced within 3 announcements + for i := 0; i < 3; i++ { + switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { + case *Transactions: + for _, tx := range *msg { + recvHashes = append(recvHashes, tx.Hash()) + } + case *NewPooledTransactionHashes: + recvHashes = append(recvHashes, *msg...) + default: + if !strings.Contains(pretty.Sdump(msg), "i/o timeout") { + t.Fatalf("unexpected message while waiting to receive txs: %s", pretty.Sdump(msg)) + } + } + // break once all 2000 txs have been received + if len(recvHashes) == 2000 { + break + } + if len(recvHashes) > 0 { + _, missingTxs := compareReceivedTxs(recvHashes, txs) + if len(missingTxs) > 0 { + continue + } else { + t.Logf("successfully received all %d txs", len(txs)) + return + } + } + } + _, missingTxs := compareReceivedTxs(recvHashes, txs) + if len(missingTxs) > 0 { + for _, missing := range missingTxs { + t.Logf("missing tx: %v", missing.Hash()) + } + t.Fatalf("missing %d txs", len(missingTxs)) + } +} + func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, recvConn *Conn) { // Wait for another transaction announcement switch msg := recvConn.ReadAndServe(s.chain, time.Second*8).(type) { @@ -75,7 +134,7 @@ func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, rec for i, recvTx := range *msg { recvTxs[i] = recvTx.Hash() } - badTxs := containsTxs(recvTxs, txs) + badTxs, _ := compareReceivedTxs(recvTxs, txs) if len(badTxs) > 0 { for _, tx := range badTxs { t.Logf("received bad tx: %v", tx) @@ -83,7 +142,7 @@ func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, rec t.Fatalf("received %d bad txs", len(badTxs)) } case *NewPooledTransactionHashes: - badTxs := containsTxs(*msg, txs) + badTxs, _ := compareReceivedTxs(*msg, txs) if len(badTxs) > 0 { for _, tx := range badTxs { t.Logf("received bad tx: %v", tx) @@ -98,18 +157,27 @@ func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, rec } } -// containsTxs checks whether the hashes of the received transactions are present in -// the given set of txs -func containsTxs(recvTxs []common.Hash, txs []*types.Transaction) []common.Hash { - containedTxs := make([]common.Hash, 0) - for _, recvTx := range recvTxs { - for _, tx := range txs { - if recvTx == tx.Hash() { - containedTxs = append(containedTxs, recvTx) - } +// compareReceivedTxs compares the received set of txs against the given set of txs, +// returning both the set received txs that were present within the given txs, and +// the set of txs that were missing from the set of received txs +func compareReceivedTxs(recvTxs []common.Hash, txs []*types.Transaction) (present []*types.Transaction, missing []*types.Transaction) { + // create a map of the hashes received from node + recvHashes := make(map[common.Hash]common.Hash) + for _, hash := range recvTxs { + recvHashes[hash] = hash + } + + // collect present txs and missing txs separately + present = make([]*types.Transaction, 0) + missing = make([]*types.Transaction, 0) + for _, tx := range txs { + if _, exists := recvHashes[tx.Hash()]; exists { + present = append(present, tx) + } else { + missing = append(missing, tx) } } - return containedTxs + return present, missing } func unknownTx(t *utesting.T, s *Suite) *types.Transaction { @@ -119,7 +187,7 @@ func unknownTx(t *utesting.T, s *Suite) *types.Transaction { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce()+1, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data()) - return signWithFaucet(t, txNew) + return signWithFaucet(t, s.chain.chainConfig, txNew) } func getNextTxFromChain(t *utesting.T, s *Suite) *types.Transaction { @@ -138,6 +206,30 @@ func getNextTxFromChain(t *utesting.T, s *Suite) *types.Transaction { return tx } +func generateTxs(t *utesting.T, s *Suite, numTxs int) (map[common.Hash]common.Hash, []*types.Transaction) { + txHashMap := make(map[common.Hash]common.Hash, numTxs) + txs := make([]*types.Transaction, numTxs) + + nextTx := getNextTxFromChain(t, s) + gas := nextTx.Gas() + + nonce = nonce + 1 + // generate txs + for i := 0; i < numTxs; i++ { + tx := generateTx(t, s.chain.chainConfig, nonce, gas) + txHashMap[tx.Hash()] = tx.Hash() + txs[i] = tx + nonce = nonce + 1 + } + return txHashMap, txs +} + +func generateTx(t *utesting.T, chainConfig *params.ChainConfig, nonce uint64, gas uint64) *types.Transaction { + var to common.Address + tx := types.NewTransaction(nonce, to, big.NewInt(1), gas, big.NewInt(1), []byte{}) + return signWithFaucet(t, chainConfig, tx) +} + func getOldTxFromChain(t *utesting.T, s *Suite) *types.Transaction { var tx *types.Transaction for _, blocks := range s.fullChain.blocks[:s.chain.Len()-1] { @@ -160,7 +252,7 @@ func invalidNonceTx(t *utesting.T, s *Suite) *types.Transaction { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce()-2, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data()) - return signWithFaucet(t, txNew) + return signWithFaucet(t, s.chain.chainConfig, txNew) } func hugeAmount(t *utesting.T, s *Suite) *types.Transaction { @@ -171,7 +263,7 @@ func hugeAmount(t *utesting.T, s *Suite) *types.Transaction { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce(), to, amount, tx.Gas(), tx.GasPrice(), tx.Data()) - return signWithFaucet(t, txNew) + return signWithFaucet(t, s.chain.chainConfig, txNew) } func hugeGasPrice(t *utesting.T, s *Suite) *types.Transaction { @@ -182,7 +274,7 @@ func hugeGasPrice(t *utesting.T, s *Suite) *types.Transaction { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), gasPrice, tx.Data()) - return signWithFaucet(t, txNew) + return signWithFaucet(t, s.chain.chainConfig, txNew) } func hugeData(t *utesting.T, s *Suite) *types.Transaction { @@ -192,11 +284,11 @@ func hugeData(t *utesting.T, s *Suite) *types.Transaction { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), tx.GasPrice(), largeBuffer(2)) - return signWithFaucet(t, txNew) + return signWithFaucet(t, s.chain.chainConfig, txNew) } -func signWithFaucet(t *utesting.T, tx *types.Transaction) *types.Transaction { - signer := types.HomesteadSigner{} +func signWithFaucet(t *utesting.T, chainConfig *params.ChainConfig, tx *types.Transaction) *types.Transaction { + signer := types.LatestSigner(chainConfig) signedTx, err := types.SignTx(tx, signer, faucetKey) if err != nil { t.Fatalf("could not sign tx: %v\n", err) diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index 50108f2dc3..55adb75f85 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -120,6 +120,14 @@ type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket func (nb NewPooledTransactionHashes) Code() int { return 24 } +type GetPooledTransactions eth.GetPooledTransactionsPacket + +func (gpt GetPooledTransactions) Code() int { return 25 } + +type PooledTransactions eth.PooledTransactionsPacket + +func (pt PooledTransactions) Code() int { return 26 } + // Conn represents an individual connection with a peer type Conn struct { *rlpx.Conn @@ -163,6 +171,10 @@ func (c *Conn) Read() Message { msg = new(Transactions) case (NewPooledTransactionHashes{}).Code(): msg = new(NewPooledTransactionHashes) + case (GetPooledTransactions{}.Code()): + msg = new(GetPooledTransactions) + case (PooledTransactions{}.Code()): + msg = new(PooledTransactions) default: return errorf("invalid message code: %d", code) } From 34f3c9539b49396b17524ee72ea20498f96600b9 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Sat, 24 Apr 2021 00:18:10 +0800 Subject: [PATCH 296/709] p2p/discover: improve discv5 handling of IPv4-in-IPv6 addresses (#22703) When receiving PING from an IPv4 address over IPv6, the implementation sent back a IPv4-in-IPv6 address. This change makes it reflect the IPv4 address. --- p2p/discover/v5_udp.go | 9 ++++++++- p2p/discover/v5_udp_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index eb01d95e93..71a39ea5a5 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -763,9 +763,16 @@ func (t *UDPv5) matchWithCall(fromID enode.ID, nonce v5wire.Nonce) (*callV5, err // handlePing sends a PONG response. func (t *UDPv5) handlePing(p *v5wire.Ping, fromID enode.ID, fromAddr *net.UDPAddr) { + remoteIP := fromAddr.IP + // Handle IPv4 mapped IPv6 addresses in the + // event the local node is binded to an + // ipv6 interface. + if remoteIP.To4() != nil { + remoteIP = remoteIP.To4() + } t.sendResponse(fromID, fromAddr, &v5wire.Pong{ ReqID: p.ReqID, - ToIP: fromAddr.IP, + ToIP: remoteIP, ToPort: uint16(fromAddr.Port), ENRSeq: t.localNode.Node().Seq(), }) diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 292785bd51..f061f5ab41 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -597,6 +597,38 @@ func TestUDPv5_LocalNode(t *testing.T) { } } +func TestUDPv5_PingWithIPV4MappedAddress(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + rawIP := net.IPv4(0xFF, 0x12, 0x33, 0xE5) + test.remoteaddr = &net.UDPAddr{ + IP: rawIP.To16(), + Port: 0, + } + remote := test.getNode(test.remotekey, test.remoteaddr).Node() + done := make(chan struct{}, 1) + + // This handler will truncate the ipv4-mapped in ipv6 address. + go func() { + test.udp.handlePing(&v5wire.Ping{ENRSeq: 1}, remote.ID(), test.remoteaddr) + done <- struct{}{} + }() + test.waitPacketOut(func(p *v5wire.Pong, addr *net.UDPAddr, _ v5wire.Nonce) { + if len(p.ToIP) == net.IPv6len { + t.Error("Received untruncated ip address") + } + if len(p.ToIP) != net.IPv4len { + t.Errorf("Received ip address with incorrect length: %d", len(p.ToIP)) + } + if !p.ToIP.Equal(rawIP) { + t.Errorf("Received incorrect ip address: wanted %s but received %s", rawIP.String(), p.ToIP.String()) + } + }) + <-done +} + // udpV5Test is the framework for all tests above. // It runs the UDPv5 transport on a virtual socket and allows testing outgoing packets. type udpV5Test struct { From 83375b08731d95fdc91b6e1576db596c21384a3d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 26 Apr 2021 14:27:56 +0200 Subject: [PATCH 297/709] core: remove old conversion to shuffle leveldb blocks into ancients --- core/blockchain.go | 63 ++++++---------------------------------------- 1 file changed, 7 insertions(+), 56 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 49aa1c3e86..2e3c5cf908 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1207,63 +1207,14 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if !bc.HasHeader(block.Hash(), block.NumberU64()) { return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4]) } - var ( - start = time.Now() - logged = time.Now() - count int - ) - // Migrate all ancient blocks. This can happen if someone upgrades from Geth - // 1.8.x to 1.9.x mid-fast-sync. Perhaps we can get rid of this path in the - // long term. - for { - // We can ignore the error here since light client won't hit this code path. - frozen, _ := bc.db.Ancients() - if frozen >= block.NumberU64() { - break - } - h := rawdb.ReadCanonicalHash(bc.db, frozen) - b := rawdb.ReadBlock(bc.db, h, frozen) - size += rawdb.WriteAncientBlock(bc.db, b, rawdb.ReadReceipts(bc.db, h, frozen, bc.chainConfig), rawdb.ReadTd(bc.db, h, frozen)) - count += 1 - - // Always keep genesis block in active database. - if b.NumberU64() != 0 { - deleted = append(deleted, &numberHash{b.NumberU64(), b.Hash()}) + if block.NumberU64() == 1 { + // Make sure to write the genesis into the freezer + if frozen, _ := bc.db.Ancients(); frozen == 0 { + h := rawdb.ReadCanonicalHash(bc.db, 0) + b := rawdb.ReadBlock(bc.db, h, 0) + size += rawdb.WriteAncientBlock(bc.db, b, rawdb.ReadReceipts(bc.db, h, 0, bc.chainConfig), rawdb.ReadTd(bc.db, h, 0)) + log.Info("Wrote genesis to ancients") } - if time.Since(logged) > 8*time.Second { - log.Info("Migrating ancient blocks", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) - logged = time.Now() - } - // Don't collect too much in-memory, write it out every 100K blocks - if len(deleted) > 100000 { - // Sync the ancient store explicitly to ensure all data has been flushed to disk. - if err := bc.db.Sync(); err != nil { - return 0, err - } - // Wipe out canonical block data. - for _, nh := range deleted { - rawdb.DeleteBlockWithoutNumber(batch, nh.hash, nh.number) - rawdb.DeleteCanonicalHash(batch, nh.number) - } - if err := batch.Write(); err != nil { - return 0, err - } - batch.Reset() - // Wipe out side chain too. - for _, nh := range deleted { - for _, hash := range rawdb.ReadAllHashes(bc.db, nh.number) { - rawdb.DeleteBlock(batch, hash, nh.number) - } - } - if err := batch.Write(); err != nil { - return 0, err - } - batch.Reset() - deleted = deleted[0:] - } - } - if count > 0 { - log.Info("Migrated ancient blocks", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) } // Flush data into ancient database. size += rawdb.WriteAncientBlock(bc.db, block, receiptChain[i], bc.GetTd(block.Hash(), block.NumberU64())) From 9b99e3dfe04baabdb51917673bc046e33731caca Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 26 Apr 2021 18:19:07 +0200 Subject: [PATCH 298/709] core/rawdb: fix datarace in freezer (#22728) The Append / truncate operations were racy. When a datafile reaches 2Gb, a new file is needed. For this operation, we require a writelock, which is not needed in the 99.99% of all cases where the data does fit in the current head-file. This transition from readlock to writelock was incorrect, and as the readlock was released, a truncate operation could slip in between, and truncate the data. This would have been fine, however, the Append operation continued writing as if no truncation had occurred, e.g writing item 5 where item 0 should reside. This PR changes the behaviour, so that if when we run into the situation that a new file is needed, it aborts, and retries, this time with a writelock. The outcome of the situation described above, running on this PR, would instead be that the Append operation exits with a failure. --- core/rawdb/freezer_table.go | 92 ++++++++++++++++++++------------ core/rawdb/freezer_table_test.go | 58 ++++++++++++++++++-- 2 files changed, 113 insertions(+), 37 deletions(-) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index b614c10d37..d7bfe18e02 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -465,35 +465,59 @@ func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) { // Note, this method will *not* flush any data to disk so be sure to explicitly // fsync before irreversibly deleting data from the database. func (t *freezerTable) Append(item uint64, blob []byte) error { + // Encode the blob before the lock portion + if !t.noCompression { + blob = snappy.Encode(nil, blob) + } // Read lock prevents competition with truncate - t.lock.RLock() + retry, err := t.append(item, blob, false) + if err != nil { + return err + } + if retry { + // Read lock was insufficient, retry with a writelock + _, err = t.append(item, blob, true) + } + return err +} + +// append injects a binary blob at the end of the freezer table. +// Normally, inserts do not require holding the write-lock, so it should be invoked with 'wlock' set to +// false. +// However, if the data will grown the current file out of bounds, then this +// method will return 'true, nil', indicating that the caller should retry, this time +// with 'wlock' set to true. +func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool, error) { + if wlock { + t.lock.Lock() + defer t.lock.Unlock() + } else { + t.lock.RLock() + defer t.lock.RUnlock() + } // Ensure the table is still accessible if t.index == nil || t.head == nil { - t.lock.RUnlock() - return errClosed + return false, errClosed } // Ensure only the next item can be written, nothing else if atomic.LoadUint64(&t.items) != item { - t.lock.RUnlock() - return fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item) - } - // Encode the blob and write it into the data file - if !t.noCompression { - blob = snappy.Encode(nil, blob) + return false, fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item) } - bLen := uint32(len(blob)) + bLen := uint32(len(encodedBlob)) if t.headBytes+bLen < bLen || t.headBytes+bLen > t.maxFileSize { - // we need a new file, writing would overflow - t.lock.RUnlock() - t.lock.Lock() + // Writing would overflow, so we need to open a new data file. + // If we don't already hold the writelock, abort and let the caller + // invoke this method a second time. + if !wlock { + return true, nil + } nextID := atomic.LoadUint32(&t.headId) + 1 // We open the next file in truncated mode -- if this file already // exists, we need to start over from scratch on it newHead, err := t.openFile(nextID, openFreezerFileTruncated) if err != nil { - t.lock.Unlock() - return err + return false, err } // Close old file, and reopen in RDONLY mode t.releaseFile(t.headId) @@ -503,13 +527,9 @@ func (t *freezerTable) Append(item uint64, blob []byte) error { t.head = newHead atomic.StoreUint32(&t.headBytes, 0) atomic.StoreUint32(&t.headId, nextID) - t.lock.Unlock() - t.lock.RLock() } - - defer t.lock.RUnlock() - if _, err := t.head.Write(blob); err != nil { - return err + if _, err := t.head.Write(encodedBlob); err != nil { + return false, err } newOffset := atomic.AddUint32(&t.headBytes, bLen) idx := indexEntry{ @@ -523,7 +543,7 @@ func (t *freezerTable) Append(item uint64, blob []byte) error { t.sizeGauge.Inc(int64(bLen + indexEntrySize)) atomic.AddUint64(&t.items, 1) - return nil + return false, nil } // getBounds returns the indexes for the item @@ -562,44 +582,48 @@ func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) { // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { + blob, err := t.retrieve(item) + if err != nil { + return nil, err + } + if t.noCompression { + return blob, nil + } + return snappy.Decode(nil, blob) +} + +// retrieve looks up the data offset of an item with the given number and retrieves +// the raw binary blob from the data file. OBS! This method does not decode +// compressed data. +func (t *freezerTable) retrieve(item uint64) ([]byte, error) { t.lock.RLock() + defer t.lock.RUnlock() // Ensure the table and the item is accessible if t.index == nil || t.head == nil { - t.lock.RUnlock() return nil, errClosed } if atomic.LoadUint64(&t.items) <= item { - t.lock.RUnlock() return nil, errOutOfBounds } // Ensure the item was not deleted from the tail either if uint64(t.itemOffset) > item { - t.lock.RUnlock() return nil, errOutOfBounds } startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset)) if err != nil { - t.lock.RUnlock() return nil, err } dataFile, exist := t.files[filenum] if !exist { - t.lock.RUnlock() return nil, fmt.Errorf("missing data file %d", filenum) } // Retrieve the data itself, decompress and return blob := make([]byte, endOffset-startOffset) if _, err := dataFile.ReadAt(blob, int64(startOffset)); err != nil { - t.lock.RUnlock() return nil, err } - t.lock.RUnlock() t.readMeter.Mark(int64(len(blob) + 2*indexEntrySize)) - - if t.noCompression { - return blob, nil - } - return snappy.Decode(nil, blob) + return blob, nil } // has returns an indicator whether the specified number data diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index b8d3170c62..0df28f236d 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -18,10 +18,13 @@ package rawdb import ( "bytes" + "encoding/binary" "fmt" + "io/ioutil" "math/rand" "os" "path/filepath" + "sync" "testing" "time" @@ -637,6 +640,55 @@ func TestOffset(t *testing.T) { // 1. have data files d0, d1, d2, d3 // 2. remove d2,d3 // -// However, all 'normal' failure modes arising due to failing to sync() or save a file should be -// handled already, and the case described above can only (?) happen if an external process/user -// deletes files from the filesystem. +// However, all 'normal' failure modes arising due to failing to sync() or save a file +// should be handled already, and the case described above can only (?) happen if an +// external process/user deletes files from the filesystem. + +// TestAppendTruncateParallel is a test to check if the Append/truncate operations are +// racy. +// +// The reason why it's not a regular fuzzer, within tests/fuzzers, is that it is dependent +// on timing rather than 'clever' input -- there's no determinism. +func TestAppendTruncateParallel(t *testing.T) { + dir, err := ioutil.TempDir("", "freezer") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + f, err := newCustomTable(dir, "tmp", metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, 8, true) + if err != nil { + t.Fatal(err) + } + + fill := func(mark uint64) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, mark) + return data + } + + for i := 0; i < 5000; i++ { + f.truncate(0) + data0 := fill(0) + f.Append(0, data0) + data1 := fill(1) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + f.truncate(0) + wg.Done() + }() + go func() { + f.Append(1, data1) + wg.Done() + }() + wg.Wait() + + if have, err := f.Retrieve(0); err == nil { + if !bytes.Equal(have, data0) { + t.Fatalf("have %x want %x", have, data0) + } + } + } +} From 854f068ed60918c624857594a703a6724b8efb3a Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 27 Apr 2021 15:44:59 +0800 Subject: [PATCH 299/709] les: polish code (#22625) * les: polish code * les/vflus/server: fixes * les: fix lint --- les/metrics.go | 1 - les/peer.go | 1 - les/server_handler.go | 21 +++--- les/vflux/server/prioritypool.go | 123 +++++++++++++++++++------------ 4 files changed, 83 insertions(+), 63 deletions(-) diff --git a/les/metrics.go b/les/metrics.go index d356326b76..07d3133c95 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -71,7 +71,6 @@ var ( connectionTimer = metrics.NewRegisteredTimer("les/connection/duration", nil) serverConnectionGauge = metrics.NewRegisteredGauge("les/connection/server", nil) - clientConnectionGauge = metrics.NewRegisteredGauge("les/connection/client", nil) totalCapacityGauge = metrics.NewRegisteredGauge("les/server/totalCapacity", nil) totalRechargeGauge = metrics.NewRegisteredGauge("les/server/totalRecharge", nil) diff --git a/les/peer.go b/les/peer.go index f6cc94dfad..c6c672942b 100644 --- a/les/peer.go +++ b/les/peer.go @@ -1099,7 +1099,6 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge // set default announceType on server side p.announceType = announceTypeSimple } - p.fcClient = flowcontrol.NewClientNode(server.fcManager, p.fcParams) } return nil }) diff --git a/les/server_handler.go b/les/server_handler.go index 0a683c1b41..80fcf1c44e 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -122,26 +123,27 @@ func (h *serverHandler) handle(p *clientPeer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } - + // Connected to another server, no messages expected, just wait for disconnection if p.server { if err := h.server.serverset.register(p); err != nil { return err } - // connected to another server, no messages expected, just wait for disconnection _, err := p.rw.ReadMsg() h.server.serverset.unregister(p) return err } - defer p.fcClient.Disconnect() // set by handshake if it's not another server + // Setup flow control mechanism for the peer + p.fcClient = flowcontrol.NewClientNode(h.server.fcManager, p.fcParams) + defer p.fcClient.Disconnect() - // Reject light clients if server is not synced. - // - // Put this checking here, so that "non-synced" les-server peers are still allowed - // to keep the connection. + // Reject light clients if server is not synced. Put this checking here, so + // that "non-synced" les-server peers are still allowed to keep the connection. if !h.synced() { p.Log().Debug("Light server not synced, rejecting peer") return p2p.DiscRequested } + + // Register the peer into the peerset and clientpool if err := h.server.peers.register(p); err != nil { return err } @@ -150,19 +152,14 @@ func (h *serverHandler) handle(p *clientPeer) error { p.Log().Debug("Client pool already closed") return p2p.DiscRequested } - activeCount, _ := h.server.clientPool.Active() - clientConnectionGauge.Update(int64(activeCount)) p.connectedAt = mclock.Now() var wg sync.WaitGroup // Wait group used to track all in-flight task routines. - defer func() { wg.Wait() // Ensure all background task routines have exited. h.server.clientPool.Unregister(p) h.server.peers.unregister(p.ID()) p.balance = nil - activeCount, _ := h.server.clientPool.Active() - clientConnectionGauge.Update(int64(activeCount)) connectionTimer.Update(time.Duration(mclock.Now() - p.connectedAt)) }() diff --git a/les/vflux/server/prioritypool.go b/les/vflux/server/prioritypool.go index 573a3570a4..480f77e6af 100644 --- a/les/vflux/server/prioritypool.go +++ b/les/vflux/server/prioritypool.go @@ -63,20 +63,22 @@ type priorityPool struct { ns *nodestate.NodeStateMachine clock mclock.Clock lock sync.Mutex - inactiveQueue *prque.Prque maxCount, maxCap uint64 minCap uint64 activeBias time.Duration capacityStepDiv, fineStepDiv uint64 + // The snapshot of priority pool for query. cachedCurve *capacityCurve ccUpdatedAt mclock.AbsTime ccUpdateForced bool - tempState []*ppNodeInfo // nodes currently in temporary state - // the following fields represent the temporary state if tempState is not empty + // Runtime status of prioritypool, represents the + // temporary state if tempState is not empty + tempState []*ppNodeInfo activeCount, activeCap uint64 activeQueue *prque.LazyQueue + inactiveQueue *prque.Prque } // ppNodeInfo is the internal node descriptor of priorityPool @@ -89,8 +91,9 @@ type ppNodeInfo struct { tempState bool // should only be true while the priorityPool lock is held tempCapacity uint64 // equals capacity when tempState is false + // the following fields only affect the temporary state and they are set to their - // default value when entering the temp state + // default value when leaving the temp state minTarget, stepDiv uint64 bias time.Duration } @@ -157,11 +160,6 @@ func newPriorityPool(ns *nodestate.NodeStateMachine, setup *serverSetup, clock m func (pp *priorityPool) requestCapacity(node *enode.Node, minTarget, maxTarget uint64, bias time.Duration) uint64 { pp.lock.Lock() pp.activeQueue.Refresh() - var updates []capUpdate - defer func() { - pp.lock.Unlock() - pp.updateFlags(updates) - }() if minTarget < pp.minCap { minTarget = pp.minCap @@ -175,12 +173,13 @@ func (pp *priorityPool) requestCapacity(node *enode.Node, minTarget, maxTarget u c, _ := pp.ns.GetField(node, pp.setup.queueField).(*ppNodeInfo) if c == nil { log.Error("requestCapacity called for unknown node", "id", node.ID()) + pp.lock.Unlock() return 0 } pp.setTempState(c) if maxTarget > c.capacity { - c.bias = bias - c.stepDiv = pp.fineStepDiv + pp.setTempStepDiv(c, pp.fineStepDiv) + pp.setTempBias(c, bias) } pp.setTempCapacity(c, maxTarget) c.minTarget = minTarget @@ -188,7 +187,9 @@ func (pp *priorityPool) requestCapacity(node *enode.Node, minTarget, maxTarget u pp.inactiveQueue.Remove(c.inactiveIndex) pp.activeQueue.Push(c) pp.enforceLimits() - updates = pp.finalizeChanges(c.tempCapacity >= minTarget && c.tempCapacity <= maxTarget && c.tempCapacity != c.capacity) + updates := pp.finalizeChanges(c.tempCapacity >= minTarget && c.tempCapacity <= maxTarget && c.tempCapacity != c.capacity) + pp.lock.Unlock() + pp.updateFlags(updates) return c.capacity } @@ -196,15 +197,11 @@ func (pp *priorityPool) requestCapacity(node *enode.Node, minTarget, maxTarget u func (pp *priorityPool) SetLimits(maxCount, maxCap uint64) { pp.lock.Lock() pp.activeQueue.Refresh() - var updates []capUpdate - defer func() { - pp.lock.Unlock() - pp.ns.Operation(func() { pp.updateFlags(updates) }) - }() - inc := (maxCount > pp.maxCount) || (maxCap > pp.maxCap) dec := (maxCount < pp.maxCount) || (maxCap < pp.maxCap) pp.maxCount, pp.maxCap = maxCount, maxCap + + var updates []capUpdate if dec { pp.enforceLimits() updates = pp.finalizeChanges(true) @@ -212,6 +209,8 @@ func (pp *priorityPool) SetLimits(maxCount, maxCap uint64) { if inc { updates = append(updates, pp.tryActivate(false)...) } + pp.lock.Unlock() + pp.ns.Operation(func() { pp.updateFlags(updates) }) } // setActiveBias sets the bias applied when trying to activate inactive nodes @@ -291,18 +290,15 @@ func (pp *priorityPool) inactivePriority(p *ppNodeInfo) int64 { func (pp *priorityPool) connectedNode(c *ppNodeInfo) { pp.lock.Lock() pp.activeQueue.Refresh() - var updates []capUpdate - defer func() { - pp.lock.Unlock() - pp.updateFlags(updates) - }() - if c.connected { + pp.lock.Unlock() return } c.connected = true pp.inactiveQueue.Push(c, pp.inactivePriority(c)) - updates = pp.tryActivate(false) + updates := pp.tryActivate(false) + pp.lock.Unlock() + pp.updateFlags(updates) } // disconnectedNode is called when a node has been removed from the pool (both inactiveFlag @@ -311,23 +307,22 @@ func (pp *priorityPool) connectedNode(c *ppNodeInfo) { func (pp *priorityPool) disconnectedNode(c *ppNodeInfo) { pp.lock.Lock() pp.activeQueue.Refresh() - var updates []capUpdate - defer func() { - pp.lock.Unlock() - pp.updateFlags(updates) - }() - if !c.connected { + pp.lock.Unlock() return } c.connected = false pp.activeQueue.Remove(c.activeIndex) pp.inactiveQueue.Remove(c.inactiveIndex) + + var updates []capUpdate if c.capacity != 0 { pp.setTempState(c) pp.setTempCapacity(c, 0) updates = pp.tryActivate(true) } + pp.lock.Unlock() + pp.updateFlags(updates) } // setTempState internally puts a node in a temporary state that can either be reverted @@ -342,27 +337,62 @@ func (pp *priorityPool) setTempState(c *ppNodeInfo) { if c.tempCapacity != c.capacity { // should never happen log.Error("tempCapacity != capacity when entering tempState") } + // Assign all the defaults to the temp state. c.minTarget = pp.minCap c.stepDiv = pp.capacityStepDiv + c.bias = 0 pp.tempState = append(pp.tempState, c) } +// unsetTempState revokes the temp status of the node and reset all internal +// fields to the default value. +func (pp *priorityPool) unsetTempState(c *ppNodeInfo) { + if !c.tempState { + return + } + c.tempState = false + if c.tempCapacity != c.capacity { // should never happen + log.Error("tempCapacity != capacity when leaving tempState") + } + c.minTarget = pp.minCap + c.stepDiv = pp.capacityStepDiv + c.bias = 0 +} + // setTempCapacity changes the capacity of a node in the temporary state and adjusts // activeCap and activeCount accordingly. Since this change is performed in the temporary // state it should be called after setTempState and before finalizeChanges. -func (pp *priorityPool) setTempCapacity(n *ppNodeInfo, cap uint64) { - if !n.tempState { // should never happen +func (pp *priorityPool) setTempCapacity(c *ppNodeInfo, cap uint64) { + if !c.tempState { // should never happen log.Error("Node is not in temporary state") return } - pp.activeCap += cap - n.tempCapacity - if n.tempCapacity == 0 { + pp.activeCap += cap - c.tempCapacity + if c.tempCapacity == 0 { pp.activeCount++ } if cap == 0 { pp.activeCount-- } - n.tempCapacity = cap + c.tempCapacity = cap +} + +// setTempBias changes the connection bias of a node in the temporary state. +func (pp *priorityPool) setTempBias(c *ppNodeInfo, bias time.Duration) { + if !c.tempState { // should never happen + log.Error("Node is not in temporary state") + return + } + c.bias = bias +} + +// setTempStepDiv changes the capacity divisor of a node in the temporary state. +func (pp *priorityPool) setTempStepDiv(c *ppNodeInfo, stepDiv uint64) { + if !c.tempState { // should never happen + log.Error("Node is not in temporary state") + return + } + c.stepDiv = stepDiv } // enforceLimits enforces active node count and total capacity limits. It returns the @@ -412,10 +442,8 @@ func (pp *priorityPool) finalizeChanges(commit bool) (updates []capUpdate) { } else { pp.setTempCapacity(c, c.capacity) // revert activeCount/activeCap } - c.tempState = false - c.bias = 0 - c.stepDiv = pp.capacityStepDiv - c.minTarget = pp.minCap + pp.unsetTempState(c) + if c.connected { if c.capacity != 0 { pp.activeQueue.Push(c) @@ -462,13 +490,13 @@ func (pp *priorityPool) tryActivate(commit bool) []capUpdate { for pp.inactiveQueue.Size() > 0 { c := pp.inactiveQueue.PopItem().(*ppNodeInfo) pp.setTempState(c) + pp.setTempBias(c, pp.activeBias) pp.setTempCapacity(c, pp.minCap) - c.bias = pp.activeBias pp.activeQueue.Push(c) pp.enforceLimits() if c.tempCapacity > 0 { commit = true - c.bias = 0 + pp.setTempBias(c, 0) } else { break } @@ -483,14 +511,9 @@ func (pp *priorityPool) tryActivate(commit bool) []capUpdate { func (pp *priorityPool) updatePriority(node *enode.Node) { pp.lock.Lock() pp.activeQueue.Refresh() - var updates []capUpdate - defer func() { - pp.lock.Unlock() - pp.updateFlags(updates) - }() - c, _ := pp.ns.GetField(node, pp.setup.queueField).(*ppNodeInfo) if c == nil || !c.connected { + pp.lock.Unlock() return } pp.activeQueue.Remove(c.activeIndex) @@ -500,7 +523,9 @@ func (pp *priorityPool) updatePriority(node *enode.Node) { } else { pp.inactiveQueue.Push(c, pp.inactivePriority(c)) } - updates = pp.tryActivate(false) + updates := pp.tryActivate(false) + pp.lock.Unlock() + pp.updateFlags(updates) } // capacityCurve is a snapshot of the priority pool contents in a format that can efficiently From a3f0da1ac42cf78769921ebf6974e4e4c6a197ae Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 27 Apr 2021 10:49:06 +0200 Subject: [PATCH 300/709] build: upgrade to golangci-lint v1.39.0 (#22696) * build: upgrade to golangci-lint v1.39.0 * consensus/ethash: fix go vet warning regarding reflect.SliceHeader * eth/catalyst: fix lint issue * consensus/ethash: fix bug in memoryMapFile --- build/checksums.txt | 33 +++++++++++++++++---------------- build/ci.go | 2 +- consensus/ethash/ethash.go | 13 +++++++------ eth/catalyst/api_test.go | 18 +++++++++++++++--- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index d5bd4d0cd3..7c8d3fd5b1 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -13,19 +13,20 @@ d7d6c70b05a7c2f68b48aab5ab8cb5116b8444c9ddad131673b152e7cff7c726 go1.16.freebsd 27a1aaa988e930b7932ce459c8a63ad5b3333b3a06b016d87ff289f2a11aacd6 go1.16.linux-ppc64le.tar.gz be4c9e4e2cf058efc4e3eb013a760cb989ddc4362f111950c990d1c63b27ccbe go1.16.linux-s390x.tar.gz -d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35 golangci-lint-1.27.0-linux-armv7.tar.gz -bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint.exe-1.27.0-windows-amd64.zip -bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint-1.27.0-windows-amd64.zip -0e2a57d6ba709440d3ed018ef1037465fa010ed02595829092860e5cf863042e golangci-lint-1.27.0-freebsd-386.tar.gz -90205fc42ab5ed0096413e790d88ac9b4ed60f4c47e576d13dc0660f7ed4b013 golangci-lint-1.27.0-linux-arm64.tar.gz -8d345e4e88520e21c113d81978e89ad77fc5b13bfdf20e5bca86b83fc4261272 golangci-lint-1.27.0-linux-amd64.tar.gz -cc619634a77f18dc73df2a0725be13116d64328dc35131ca1737a850d6f76a59 golangci-lint-1.27.0-freebsd-armv7.tar.gz -fe683583cfc9eeec83e498c0d6159d87b5e1919dbe4b6c3b3913089642906069 golangci-lint-1.27.0-linux-s390x.tar.gz -058f5579bee75bdaacbaf75b75e1369f7ad877fd8b3b145aed17a17545de913e golangci-lint-1.27.0-freebsd-armv6.tar.gz -38e1e3dadbe3f56ab62b4de82ee0b88e8fad966d8dfd740a26ef94c2edef9818 golangci-lint-1.27.0-linux-armv6.tar.gz -071b34af5516f4e1ddcaea6011e18208f4f043e1af8ba21eeccad4585cb3d095 golangci-lint.exe-1.27.0-windows-386.zip -071b34af5516f4e1ddcaea6011e18208f4f043e1af8ba21eeccad4585cb3d095 golangci-lint-1.27.0-windows-386.zip -5f37e2b33914ecddb7cad38186ef4ec61d88172fc04f930fa0267c91151ff306 golangci-lint-1.27.0-linux-386.tar.gz -4d94cfb51fdebeb205f1d5a349ac2b683c30591c5150708073c1c329e15965f0 golangci-lint-1.27.0-freebsd-amd64.tar.gz -52572ba8ff07d5169c2365d3de3fec26dc55a97522094d13d1596199580fa281 golangci-lint-1.27.0-linux-ppc64le.tar.gz -3fb1a1683a29c6c0a8cd76135f62b606fbdd538d5a7aeab94af1af70ffdc2fd4 golangci-lint-1.27.0-darwin-amd64.tar.gz +7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c golangci-lint-1.39.0-darwin-amd64.tar.gz +574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064 golangci-lint-1.39.0-darwin-arm64.tar.gz +6225f7014987324ab78e9b511f294e3f25be013728283c33918c67c8576d543e golangci-lint-1.39.0-freebsd-386.tar.gz +6b3e76e1e5eaf0159411c8e2727f8d533989d3bb19f10e9caa6e0b9619ee267d golangci-lint-1.39.0-freebsd-amd64.tar.gz +a301cacfff87ed9b00313d95278533c25a4527a06b040a17d969b4b7e1b8a90d golangci-lint-1.39.0-freebsd-armv7.tar.gz +25bfd96a29c3112f508d5e4fc860dbad7afce657233c343acfa20715717d51e7 golangci-lint-1.39.0-freebsd-armv6.tar.gz +9687e4ff15545cfc722b0e46107a94195166a505023b48a316579af25ad09505 golangci-lint-1.39.0-linux-armv7.tar.gz +a7fa7ab2bfc99cbe5e5bcbf5684f5a997f920afbbe2f253d2feb1001d5e3c8b3 golangci-lint-1.39.0-linux-armv6.tar.gz +c8f9634115beddb4ed9129c1f7ecd4c97c99d07aeef33e3707234097eeb51b7b golangci-lint-1.39.0-linux-mips64le.tar.gz +d1234c213b74751f1af413302dde0e9a6d4d29aecef034af7abb07dc1b6e887f golangci-lint-1.39.0-linux-arm64.tar.gz +df25d9267168323b163147acb823ab0215a8a3bb6898a4a9320afdfedde66817 golangci-lint-1.39.0-linux-386.tar.gz +1767e75fba357b7651b1a796d38453558f371c60af805505ec99e166908c04b5 golangci-lint-1.39.0-linux-ppc64le.tar.gz +25fd75bf3186b3d930ecae10185689968fd18fd8fa6f9f555d6beb04348c20f6 golangci-lint-1.39.0-linux-s390x.tar.gz +3a73aa7468087caa62673c8adea99b4e4dff846dc72707222db85f8679b40cbf golangci-lint-1.39.0-linux-amd64.tar.gz +578caceccf81739bda67dbfec52816709d03608c6878888ecdc0e186a094a41b golangci-lint-1.39.0-linux-mips64.tar.gz +494b66ba0e32c8ddf6c4f6b1d05729b110900f6017eda943057e43598c17d7a8 golangci-lint-1.39.0-windows-386.zip +52ec2e13a3cbb47147244dff8cfc35103563deb76e0459133058086fc35fb2c7 golangci-lint-1.39.0-windows-amd64.zip diff --git a/build/ci.go b/build/ci.go index fb06c0f420..c94b96821e 100644 --- a/build/ci.go +++ b/build/ci.go @@ -379,7 +379,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.27.0" + const version = "1.39.0" csdb := build.MustLoadChecksums("build/checksums.txt") base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d922be7773..ec06d02a54 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -112,12 +112,13 @@ func memoryMapFile(file *os.File, write bool) (mmap.MMap, []uint32, error) { if err != nil { return nil, nil, err } - // Yay, we managed to memory map the file, here be dragons - header := *(*reflect.SliceHeader)(unsafe.Pointer(&mem)) - header.Len /= 4 - header.Cap /= 4 - - return mem, *(*[]uint32)(unsafe.Pointer(&header)), nil + // The file is now memory-mapped. Create a []uint32 view of the file. + var view []uint32 + header := (*reflect.SliceHeader)(unsafe.Pointer(&view)) + header.Data = (*reflect.SliceHeader)(unsafe.Pointer(&mem)).Data + header.Cap = len(mem) / 4 + header.Len = header.Cap + return mem, view, nil } // memoryMapAndGenerate tries to memory map a temporary file of uint32s for write diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 456b6867bd..b8a6e43fcd 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -20,7 +20,6 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -67,8 +66,21 @@ func generateTestChainWithFork(n int, fork int) (*core.Genesis, []*types.Block, fork = n - 1 } db := rawdb.NewMemoryDatabase() - //nolint:composites - config := ¶ms.ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, big.NewInt(0), new(params.EthashConfig), nil} + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(1337), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + CatalystBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), + } genesis := &core.Genesis{ Config: config, Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, From ad983b300b138b2300402187c2e50f9afdc4b2a3 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 27 Apr 2021 03:36:57 -0600 Subject: [PATCH 301/709] cmd/puppeth: add support for authentication via ssh agent (#22634) --- cmd/puppeth/ssh.go | 62 ++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index da2862db2f..039cb6cb45 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" "golang.org/x/crypto/ssh/terminal" ) @@ -43,6 +44,8 @@ type sshClient struct { logger log.Logger } +const EnvSSHAuthSock = "SSH_AUTH_SOCK" + // dial establishes an SSH connection to a remote node using the current user and // the user's configured private RSA key. If that fails, password authentication // is fallen back to. server can be a string like user:identity@server:port. @@ -79,38 +82,49 @@ func dial(server string, pubkey []byte) (*sshClient, error) { if username == "" { username = user.Username } - // Configure the supported authentication methods (private key and password) - var auths []ssh.AuthMethod - path := filepath.Join(user.HomeDir, ".ssh", identity) - if buf, err := ioutil.ReadFile(path); err != nil { - log.Warn("No SSH key, falling back to passwords", "path", path, "err", err) + // Configure the supported authentication methods (ssh agent, private key and password) + var ( + auths []ssh.AuthMethod + conn net.Conn + ) + if conn, err = net.Dial("unix", os.Getenv(EnvSSHAuthSock)); err != nil { + log.Warn("Unable to dial SSH agent, falling back to private keys", "err", err) } else { - key, err := ssh.ParsePrivateKey(buf) - if err != nil { - fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - if err != nil { - log.Warn("Couldn't read password", "err", err) - } - key, err := ssh.ParsePrivateKeyWithPassphrase(buf, blob) + client := agent.NewClient(conn) + auths = append(auths, ssh.PublicKeysCallback(client.Signers)) + } + if err != nil { + path := filepath.Join(user.HomeDir, ".ssh", identity) + if buf, err := ioutil.ReadFile(path); err != nil { + log.Warn("No SSH key, falling back to passwords", "path", path, "err", err) + } else { + key, err := ssh.ParsePrivateKey(buf) if err != nil { - log.Warn("Failed to decrypt SSH key, falling back to passwords", "path", path, "err", err) + fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path) + blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + fmt.Println() + if err != nil { + log.Warn("Couldn't read password", "err", err) + } + key, err := ssh.ParsePrivateKeyWithPassphrase(buf, blob) + if err != nil { + log.Warn("Failed to decrypt SSH key, falling back to passwords", "path", path, "err", err) + } else { + auths = append(auths, ssh.PublicKeys(key)) + } } else { auths = append(auths, ssh.PublicKeys(key)) } - } else { - auths = append(auths, ssh.PublicKeys(key)) } - } - auths = append(auths, ssh.PasswordCallback(func() (string, error) { - fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + auths = append(auths, ssh.PasswordCallback(func() (string, error) { + fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) + blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - return string(blob), err - })) + fmt.Println() + return string(blob), err + })) + } // Resolve the IP address of the remote server addr, err := net.LookupHost(hostname) if err != nil { From a0a99e610df1485b685138ddd9efd12bddb8a945 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 27 Apr 2021 12:43:47 +0200 Subject: [PATCH 302/709] build: upgrade -dlgo version to Go 1.16.3 (#22746) --- build/checksums.txt | 25 +++++++++++++------------ build/ci.go | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 7c8d3fd5b1..0e33e07f2c 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,17 +1,18 @@ # This file contains sha256 checksums of optional build dependencies. -7688063d55656105898f323d90a79a39c378d86fe89ae192eb3b7fc46347c95a go1.16.src.tar.gz -6000a9522975d116bf76044967d7e69e04e982e9625330d9a539a8b45395f9a8 go1.16.darwin-amd64.tar.gz -ea435a1ac6d497b03e367fdfb74b33e961d813883468080f6e239b3b03bea6aa go1.16.linux-386.tar.gz -013a489ebb3e24ef3d915abe5b94c3286c070dfe0818d5bca8108f1d6e8440d2 go1.16.linux-amd64.tar.gz -3770f7eb22d05e25fbee8fb53c2a4e897da043eb83c69b9a14f8d98562cd8098 go1.16.linux-arm64.tar.gz -d1d9404b1dbd77afa2bdc70934e10fbfcf7d785c372efc29462bb7d83d0a32fd go1.16.linux-armv6l.tar.gz -481492a17d42193d471b93b7a06da3555331bd833b76336afc87be820c48933f go1.16.windows-386.zip -5cc88fa506b3d5c453c54c3ea218fc8dd05d7362ae1de15bb67986b72089ce93 go1.16.windows-amd64.zip -d7d6c70b05a7c2f68b48aab5ab8cb5116b8444c9ddad131673b152e7cff7c726 go1.16.freebsd-386.tar.gz -40b03216f6945fb6883a50604fc7f409a83f62171607229a9c598e701e684f8a go1.16.freebsd-amd64.tar.gz -27a1aaa988e930b7932ce459c8a63ad5b3333b3a06b016d87ff289f2a11aacd6 go1.16.linux-ppc64le.tar.gz -be4c9e4e2cf058efc4e3eb013a760cb989ddc4362f111950c990d1c63b27ccbe go1.16.linux-s390x.tar.gz +b298d29de9236ca47a023e382313bcc2d2eed31dfa706b60a04103ce83a71a25 go1.16.3.src.tar.gz +6bb1cf421f8abc2a9a4e39140b7397cdae6aca3e8d36dcff39a1a77f4f1170ac go1.16.3.darwin-amd64.tar.gz +f4e96bbcd5d2d1942f5b55d9e4ab19564da4fad192012f6d7b0b9b055ba4208f go1.16.3.darwin-arm64.tar.gz +48b2d1481db756c88c18b1f064dbfc3e265ce4a775a23177ca17e25d13a24c5d go1.16.3.linux-386.tar.gz +951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2 go1.16.3.linux-amd64.tar.gz +566b1d6f17d2bc4ad5f81486f0df44f3088c3ed47a3bec4099d8ed9939e90d5d go1.16.3.linux-arm64.tar.gz +0dae30385e3564a557dac7f12a63eedc73543e6da0f6017990e214ce8cc8797c go1.16.3.linux-armv6l.tar.gz +a3c16e1531bf9726f47911c4a9ed7cb665a6207a51c44f10ebad4db63b4bcc5a go1.16.3.windows-386.zip +a4400345135b36cb7942e52bbaf978b66814738b855eeff8de879a09fd99de7f go1.16.3.windows-amd64.zip +31ecd11d497684fa8b0f01ba784590c4c760943665fdc4fe0adaa1405c71736c go1.16.3.freebsd-386.tar.gz +ffbd920b309e62e807457b11d80e8c17fefe3ef6de423aaba4b1e270b2ca4c3d go1.16.3.freebsd-amd64.tar.gz +5eb046bbbbc7fe2591846a4303884cb5a01abb903e3e61e33459affe7874e811 go1.16.3.linux-ppc64le.tar.gz +3e8bd7bde533a73fd6fa75b5288678ef397e76c198cfb26b8ae086035383b1cf go1.16.3.linux-s390x.tar.gz 7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c golangci-lint-1.39.0-darwin-amd64.tar.gz 574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064 golangci-lint-1.39.0-darwin-arm64.tar.gz diff --git a/build/ci.go b/build/ci.go index c94b96821e..b73435c5e6 100644 --- a/build/ci.go +++ b/build/ci.go @@ -152,7 +152,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.16" + dlgoVersion = "1.16.3" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From 65a1c2d829ca11ba2a9b08aad977807731b6f009 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 27 Apr 2021 13:21:41 +0200 Subject: [PATCH 303/709] core/vm: make gas cost reporting to tracers correct (#22702) Previously, the makeCallVariantGasCallEIP2929 charged the cold account access cost directly, leading to an incorrect gas cost passed to the tracer from the main execution loop. This change still temporarily charges the cost (to allow for an accurate calculation of the available gas for the call), but then afterwards refunds it and instead returns the correct total gas cost to be then properly charged in the main loop. --- core/vm/operations_acl.go | 22 +++++++-- core/vm/runtime/runtime_test.go | 80 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index 191953ce5e..45b51d80cd 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -177,10 +177,15 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { addr := common.Address(stack.Back(1).Bytes20()) // Check slot presence in the access list - if !evm.StateDB.AddressInAccessList(addr) { + warmAccess := evm.StateDB.AddressInAccessList(addr) + // The WarmStorageReadCostEIP2929 (100) is already deducted in the form of a constant cost, so + // the cost to charge for cold access, if any, is Cold - Warm + coldCost := ColdAccountAccessCostEIP2929 - WarmStorageReadCostEIP2929 + if !warmAccess { evm.StateDB.AddAddressToAccessList(addr) - // The WarmStorageReadCostEIP2929 (100) is already deducted in the form of a constant cost - if !contract.UseGas(ColdAccountAccessCostEIP2929 - WarmStorageReadCostEIP2929) { + // Charge the remaining difference here already, to correctly calculate available + // gas for call + if !contract.UseGas(coldCost) { return 0, ErrOutOfGas } } @@ -189,7 +194,16 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { // - transfer value // - memory expansion // - 63/64ths rule - return oldCalculator(evm, contract, stack, mem, memorySize) + gas, err := oldCalculator(evm, contract, stack, mem, memorySize) + if warmAccess || err != nil { + return gas, err + } + // In case of a cold access, we temporarily add the cold charge back, and also + // add it to the returned gas. By adding it to the return, it will be charged + // outside of this function, as part of the dynamic gas, and that will make it + // also become correctly reported to tracers. + contract.Gas += coldCost + return gas + coldCost, nil } } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 2692755324..dcf2d0d447 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -608,3 +608,83 @@ func TestEip2929Cases(t *testing.T) { "account (cheap)", code) } } + +// TestColdAccountAccessCost test that the cold account access cost is reported +// correctly +// see: https://github.com/ethereum/go-ethereum/issues/22649 +func TestColdAccountAccessCost(t *testing.T) { + for i, tc := range []struct { + code []byte + step int + want uint64 + }{ + { // EXTCODEHASH(0xff) + code: []byte{byte(vm.PUSH1), 0xFF, byte(vm.EXTCODEHASH), byte(vm.POP)}, + step: 1, + want: 2600, + }, + { // BALANCE(0xff) + code: []byte{byte(vm.PUSH1), 0xFF, byte(vm.BALANCE), byte(vm.POP)}, + step: 1, + want: 2600, + }, + { // CALL(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.CALL), byte(vm.POP), + }, + step: 7, + want: 2855, + }, + { // CALLCODE(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.CALLCODE), byte(vm.POP), + }, + step: 7, + want: 2855, + }, + { // DELEGATECALL(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.DELEGATECALL), byte(vm.POP), + }, + step: 6, + want: 2855, + }, + { // STATICCALL(0xff) + code: []byte{ + byte(vm.PUSH1), 0x0, + byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xff, byte(vm.DUP1), byte(vm.STATICCALL), byte(vm.POP), + }, + step: 6, + want: 2855, + }, + { // SELFDESTRUCT(0xff) + code: []byte{ + byte(vm.PUSH1), 0xff, byte(vm.SELFDESTRUCT), + }, + step: 1, + want: 7600, + }, + } { + tracer := vm.NewStructLogger(nil) + Execute(tc.code, nil, &Config{ + EVMConfig: vm.Config{ + Debug: true, + Tracer: tracer, + }, + }) + have := tracer.StructLogs()[tc.step].GasCost + if want := tc.want; have != want { + for ii, op := range tracer.StructLogs() { + t.Logf("%d: %v %d", ii, op.OpName(), op.GasCost) + } + t.Fatalf("tescase %d, gas report wrong, step %d, have %d want %d", i, tc.step, have, want) + } + } +} From caea6c466147752ed726a79f69e71215131065dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Apr 2021 17:19:59 +0300 Subject: [PATCH 304/709] eth/protocols/snap: generate storage trie from full dirty snap data (#22668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/protocols/snap: generate storage trie from full dirty snap data * eth/protocols/snap: get rid of some more dead code * eth/protocols/snap: less frequent logs, also log during trie generation * eth/protocols/snap: implement dirty account range stack-hashing * eth/protocols/snap: don't loop on account trie generation * eth/protocols/snap: fix account format in trie * core, eth, ethdb: glue snap packets together, but not chunks * eth/protocols/snap: print completion log for snap phase * eth/protocols/snap: extended tests * eth/protocols/snap: make testcase pass * eth/protocols/snap: fix account stacktrie commit without defer * ethdb: fix key counts on reset * eth/protocols: fix typos * eth/protocols/snap: make better use of delivered data (#44) * eth/protocols/snap: make better use of delivered data * squashme * eth/protocols/snap: reduce chunking * squashme * eth/protocols/snap: reduce chunking further * eth/protocols/snap: break out hash range calculations * eth/protocols/snap: use sort.Search instead of looping * eth/protocols/snap: prevent crash on storage response with no keys * eth/protocols/snap: nitpicks all around * eth/protocols/snap: clear heal need on 1-chunk storage completion * eth/protocols/snap: fix range chunker, add tests Co-authored-by: Péter Szilágyi * trie: fix test API error * eth/protocols/snap: fix some further liter issues * eth/protocols/snap: fix accidental batch reuse Co-authored-by: Martin Holst Swende --- core/rawdb/database_test.go | 17 ++ core/rawdb/table.go | 5 + eth/protocols/snap/handler.go | 2 +- eth/protocols/snap/range.go | 80 +++++ eth/protocols/snap/range_test.go | 143 +++++++++ eth/protocols/snap/sync.go | 408 ++++++++++++++----------- eth/protocols/snap/sync_test.go | 113 ++++++- ethdb/batch.go | 3 + ethdb/leveldb/leveldb.go | 9 +- ethdb/memorydb/memorydb.go | 9 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 1 + trie/trie_test.go | 1 + 12 files changed, 603 insertions(+), 188 deletions(-) create mode 100644 core/rawdb/database_test.go create mode 100644 eth/protocols/snap/range.go create mode 100644 eth/protocols/snap/range_test.go diff --git a/core/rawdb/database_test.go b/core/rawdb/database_test.go new file mode 100644 index 0000000000..8bf06f97d8 --- /dev/null +++ b/core/rawdb/database_test.go @@ -0,0 +1,17 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 323ef6293c..4daa6b5349 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -176,6 +176,11 @@ func (b *tableBatch) Delete(key []byte) error { return b.batch.Delete(append([]byte(b.prefix), key...)) } +// KeyCount retrieves the number of keys queued up for writing. +func (b *tableBatch) KeyCount() int { + return b.batch.KeyCount() +} + // ValueSize retrieves the amount of data queued up for writing. func (b *tableBatch) ValueSize() int { return b.batch.ValueSize() diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 4c12adfa81..9bfac6f03f 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -354,7 +354,7 @@ func handleMessage(backend Backend, peer *Peer) error { if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - // Ensure the ranges ae monotonically increasing + // Ensure the ranges are monotonically increasing for i, slots := range res.Slots { for j := 1; j < len(slots); j++ { if bytes.Compare(slots[j-1].Hash[:], slots[j].Hash[:]) >= 0 { diff --git a/eth/protocols/snap/range.go b/eth/protocols/snap/range.go new file mode 100644 index 0000000000..dd380ff471 --- /dev/null +++ b/eth/protocols/snap/range.go @@ -0,0 +1,80 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/holiman/uint256" +) + +// hashRange is a utility to handle ranges of hashes, Split up the +// hash-space into sections, and 'walk' over the sections +type hashRange struct { + current *uint256.Int + step *uint256.Int +} + +// newHashRange creates a new hashRange, initiated at the start position, +// and with the step set to fill the desired 'num' chunks +func newHashRange(start common.Hash, num uint64) *hashRange { + left := new(big.Int).Sub(hashSpace, start.Big()) + step := new(big.Int).Div( + new(big.Int).Add(left, new(big.Int).SetUint64(num-1)), + new(big.Int).SetUint64(num), + ) + step256 := new(uint256.Int) + step256.SetFromBig(step) + + return &hashRange{ + current: uint256.NewInt().SetBytes32(start[:]), + step: step256, + } +} + +// Next pushes the hash range to the next interval. +func (r *hashRange) Next() bool { + next := new(uint256.Int) + if overflow := next.AddOverflow(r.current, r.step); overflow { + return false + } + r.current = next + return true +} + +// Start returns the first hash in the current interval. +func (r *hashRange) Start() common.Hash { + return r.current.Bytes32() +} + +// End returns the last hash in the current interval. +func (r *hashRange) End() common.Hash { + // If the end overflows (non divisible range), return a shorter interval + next := new(uint256.Int) + if overflow := next.AddOverflow(r.current, r.step); overflow { + return common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + } + return new(uint256.Int).Sub(next, uint256.NewInt().SetOne()).Bytes32() +} + +// incHash returns the next hash, in lexicographical order (a.k.a plus one) +func incHash(h common.Hash) common.Hash { + a := uint256.NewInt().SetBytes32(h[:]) + a.Add(a, uint256.NewInt().SetOne()) + return common.Hash(a.Bytes32()) +} diff --git a/eth/protocols/snap/range_test.go b/eth/protocols/snap/range_test.go new file mode 100644 index 0000000000..23273e50bf --- /dev/null +++ b/eth/protocols/snap/range_test.go @@ -0,0 +1,143 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +// Tests that given a starting hash and a density, the hash ranger can correctly +// split up the remaining hash space into a fixed number of chunks. +func TestHashRanges(t *testing.T) { + tests := []struct { + head common.Hash + chunks uint64 + starts []common.Hash + ends []common.Hash + }{ + // Simple test case to split the entire hash range into 4 chunks + { + head: common.Hash{}, + chunks: 4, + starts: []common.Hash{ + {}, + common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0xc000000000000000000000000000000000000000000000000000000000000000"), + }, + ends: []common.Hash{ + common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.HexToHash("0xbfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, + }, + // Split a divisible part of the hash range up into 2 chunks + { + head: common.HexToHash("0x2000000000000000000000000000000000000000000000000000000000000000"), + chunks: 2, + starts: []common.Hash{ + common.Hash{}, + common.HexToHash("0x9000000000000000000000000000000000000000000000000000000000000000"), + }, + ends: []common.Hash{ + common.HexToHash("0x8fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, + }, + // Split the entire hash range into a non divisible 3 chunks + { + head: common.Hash{}, + chunks: 3, + starts: []common.Hash{ + {}, + common.HexToHash("0x5555555555555555555555555555555555555555555555555555555555555556"), + common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"), + }, + ends: []common.Hash{ + common.HexToHash("0x5555555555555555555555555555555555555555555555555555555555555555"), + common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"), + common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, + }, + // Split a part of hash range into a non divisible 3 chunks + { + head: common.HexToHash("0x2000000000000000000000000000000000000000000000000000000000000000"), + chunks: 3, + starts: []common.Hash{ + {}, + common.HexToHash("0x6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"), + common.HexToHash("0xb555555555555555555555555555555555555555555555555555555555555556"), + }, + ends: []common.Hash{ + common.HexToHash("0x6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + common.HexToHash("0xb555555555555555555555555555555555555555555555555555555555555555"), + common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, + }, + // Split a part of hash range into a non divisible 3 chunks, but with a + // meaningful space size for manual verification. + // - The head being 0xff...f0, we have 14 hashes left in the space + // - Chunking up 14 into 3 pieces is 4.(6), but we need the ceil of 5 to avoid a micro-last-chunk + // - Since the range is not divisible, the last interval will be shrter, capped at 0xff...f + // - The chunk ranges thus needs to be [..0, ..5], [..6, ..b], [..c, ..f] + { + head: common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0"), + chunks: 3, + starts: []common.Hash{ + {}, + common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6"), + common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc"), + }, + ends: []common.Hash{ + common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5"), + common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb"), + common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, + }, + } + for i, tt := range tests { + r := newHashRange(tt.head, tt.chunks) + + var ( + starts = []common.Hash{{}} + ends = []common.Hash{r.End()} + ) + for r.Next() { + starts = append(starts, r.Start()) + ends = append(ends, r.End()) + } + if len(starts) != len(tt.starts) { + t.Errorf("test %d: starts count mismatch: have %d, want %d", i, len(starts), len(tt.starts)) + } + for j := 0; j < len(starts) && j < len(tt.starts); j++ { + if starts[j] != tt.starts[j] { + t.Errorf("test %d, start %d: hash mismatch: have %x, want %x", i, j, starts[j], tt.starts[j]) + } + } + if len(ends) != len(tt.ends) { + t.Errorf("test %d: ends count mismatch: have %d, want %d", i, len(ends), len(tt.ends)) + } + for j := 0; j < len(ends) && j < len(tt.ends); j++ { + if ends[j] != tt.ends[j] { + t.Errorf("test %d, end %d: hash mismatch: have %x, want %x", i, j, ends[j], tt.ends[j]) + } + } + } +} diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index cff1a77e6c..3ce4c8735f 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -23,10 +23,12 @@ import ( "fmt" "math/big" "math/rand" + "sort" "sync" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -73,7 +75,9 @@ const ( // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. maxTrieRequestCount = 512 +) +var ( // accountConcurrency is the number of chunks to split the account trie into // to allow concurrent retrievals. accountConcurrency = 16 @@ -81,9 +85,7 @@ const ( // storageConcurrency is the number of chunks to split the a large contract // storage trie into to allow concurrent retrievals. storageConcurrency = 16 -) -var ( // requestTimeout is the maximum time a peer is allowed to spend on serving // a single network request. requestTimeout = 15 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? @@ -127,12 +129,6 @@ type accountResponse struct { hashes []common.Hash // Account hashes in the returned range accounts []*state.Account // Expanded accounts in the returned range - nodes ethdb.KeyValueStore // Database containing the reconstructed trie nodes - trie *trie.Trie // Reconstructed trie to reject incomplete account paths - - bounds map[common.Hash]struct{} // Boundary nodes to avoid persisting incomplete accounts - overflow *light.NodeSet // Overflow nodes to avoid persisting across chunk boundaries - cont bool // Whether the account range has a continuation } @@ -209,12 +205,8 @@ type storageResponse struct { hashes [][]common.Hash // Storage slot hashes in the returned range slots [][][]byte // Storage slot values in the returned range nodes []ethdb.KeyValueStore // Database containing the reconstructed trie nodes - tries []*trie.Trie // Reconstructed tries to reject overflown slots - // Fields relevant for the last account only - bounds map[common.Hash]struct{} // Boundary nodes to avoid persisting (incomplete) - overflow *light.NodeSet // Overflow nodes to avoid persisting across chunk boundaries - cont bool // Whether the last storage range has a continuation + cont bool // Whether the last storage range has a continuation } // trienodeHealRequest tracks a pending state trie request to ensure responses @@ -301,6 +293,9 @@ type accountTask struct { codeTasks map[common.Hash]struct{} // Code hashes that need retrieval stateTasks map[common.Hash]common.Hash // Account hashes->roots that need full state retrieval + genBatch ethdb.Batch // Batch used by the node generator + genTrie *trie.StackTrie // Node generator from storage slots + done bool // Flag whether the task can be removed } @@ -312,7 +307,11 @@ type storageTask struct { // These fields are internals used during runtime root common.Hash // Storage root hash for this instance req *storageRequest // Pending request to fill this task - done bool // Flag whether the task can be removed + + genBatch ethdb.Batch // Batch used by the node generator + genTrie *trie.StackTrie // Node generator from storage slots + + done bool // Flag whether the task can be removed } // healTask represents the sync task for healing the snap-synced chunk boundaries. @@ -359,7 +358,7 @@ type SyncPeer interface { // trie, starting with the origin. RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error - // RequestStorageRange fetches a batch of storage slots belonging to one or + // RequestStorageRanges fetches a batch of storage slots belonging to one or // more accounts. If slots from only one accout is requested, an origin marker // may also be used to retrieve from there. RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error @@ -680,6 +679,17 @@ func (s *Syncer) loadSyncStatus() { log.Debug("Scheduled account sync task", "from", task.Next, "last", task.Last) } s.tasks = progress.Tasks + for _, task := range s.tasks { + task.genBatch = s.db.NewBatch() + task.genTrie = trie.NewStackTrie(task.genBatch) + + for _, subtasks := range task.SubTasks { + for _, subtask := range subtasks { + subtask.genBatch = s.db.NewBatch() + subtask.genTrie = trie.NewStackTrie(task.genBatch) + } + } + } s.snapped = len(s.tasks) == 0 s.accountSynced = progress.AccountSynced @@ -710,7 +720,7 @@ func (s *Syncer) loadSyncStatus() { step := new(big.Int).Sub( new(big.Int).Div( new(big.Int).Exp(common.Big2, common.Big256, nil), - big.NewInt(accountConcurrency), + big.NewInt(int64(accountConcurrency)), ), common.Big1, ) for i := 0; i < accountConcurrency; i++ { @@ -719,10 +729,13 @@ func (s *Syncer) loadSyncStatus() { // Make sure we don't overflow if the step is not a proper divisor last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") } + batch := s.db.NewBatch() s.tasks = append(s.tasks, &accountTask{ Next: next, Last: last, SubTasks: make(map[common.Hash][]*storageTask), + genBatch: batch, + genTrie: trie.NewStackTrie(batch), }) log.Debug("Created account sync task", "from", next, "last", last) next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) @@ -731,6 +744,25 @@ func (s *Syncer) loadSyncStatus() { // saveSyncStatus marshals the remaining sync tasks into leveldb. func (s *Syncer) saveSyncStatus() { + // Serialize any partial progress to disk before spinning down + for _, task := range s.tasks { + keys, bytes := task.genBatch.KeyCount(), task.genBatch.ValueSize() + if err := task.genBatch.Write(); err != nil { + log.Error("Failed to persist account slots", "err", err) + } + s.accountBytes += common.StorageSize(keys*common.HashLength + bytes) + + for _, subtasks := range task.SubTasks { + for _, subtask := range subtasks { + keys, bytes := subtask.genBatch.KeyCount(), subtask.genBatch.ValueSize() + if err := subtask.genBatch.Write(); err != nil { + log.Error("Failed to persist storage slots", "err", err) + } + s.accountBytes += common.StorageSize(keys*common.HashLength + bytes) + } + } + } + // Store the actual progress markers progress := &syncProgress{ Tasks: s.tasks, AccountSynced: s.accountSynced, @@ -754,16 +786,25 @@ func (s *Syncer) saveSyncStatus() { // cleanAccountTasks removes account range retrieval tasks that have already been // completed. func (s *Syncer) cleanAccountTasks() { + // If the sync was already done before, don't even bother + if len(s.tasks) == 0 { + return + } + // Sync wasn't finished previously, check for any task that can be finalized for i := 0; i < len(s.tasks); i++ { if s.tasks[i].done { s.tasks = append(s.tasks[:i], s.tasks[i+1:]...) i-- } } + // If everything was just finalized just, generate the account trie and start heal if len(s.tasks) == 0 { s.lock.Lock() s.snapped = true s.lock.Unlock() + + // Push the final sync report + s.reportSyncProgress(true) } } @@ -1600,12 +1641,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { continue } if cmp > 0 { - // Chunk overflown, cut off excess, but also update the boundary nodes - for j := i; j < len(res.hashes); j++ { - if err := res.trie.Prove(res.hashes[j][:], 0, res.overflow); err != nil { - panic(err) // Account range was already proven, what happened - } - } + // Chunk overflown, cut off excess res.hashes = res.hashes[:i] res.accounts = res.accounts[:i] res.cont = false // Mark range completed @@ -1681,7 +1717,6 @@ func (s *Syncer) processBytecodeResponse(res *bytecodeResponse) { var ( codes uint64 - bytes common.StorageSize ) for i, hash := range res.hashes { code := res.codes[i] @@ -1699,17 +1734,16 @@ func (s *Syncer) processBytecodeResponse(res *bytecodeResponse) { } } // Push the bytecode into a database batch - s.bytecodeSynced++ - s.bytecodeBytes += common.StorageSize(len(code)) - codes++ - bytes += common.StorageSize(len(code)) - rawdb.WriteCode(batch, hash, code) } + bytes := common.StorageSize(batch.ValueSize()) if err := batch.Write(); err != nil { log.Crit("Failed to persist bytecodes", "err", err) } + s.bytecodeSynced += codes + s.bytecodeBytes += bytes + log.Debug("Persisted set of bytecodes", "count", codes, "bytes", bytes) // If this delivery completed the last pending task, forward the account task @@ -1732,10 +1766,9 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { batch := s.db.NewBatch() var ( - slots int - nodes int - skipped int - bytes common.StorageSize + slots int + nodes int + bytes common.StorageSize ) // Iterate over all the accounts and reconstruct their storage tries from the // delivered slots @@ -1772,27 +1805,50 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // the subtasks for it within the main account task if tasks, ok := res.mainTask.SubTasks[account]; !ok { var ( - next common.Hash - ) - step := new(big.Int).Sub( - new(big.Int).Div( - new(big.Int).Exp(common.Big2, common.Big256, nil), - big.NewInt(storageConcurrency), - ), common.Big1, + keys = res.hashes[i] + chunks = uint64(storageConcurrency) + lastKey common.Hash ) - for k := 0; k < storageConcurrency; k++ { - last := common.BigToHash(new(big.Int).Add(next.Big(), step)) - if k == storageConcurrency-1 { - // Make sure we don't overflow if the step is not a proper divisor - last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + if len(keys) > 0 { + lastKey = keys[len(keys)-1] + } + // If the number of slots remaining is low, decrease the + // number of chunks. Somewhere on the order of 10-15K slots + // fit into a packet of 500KB. A key/slot pair is maximum 64 + // bytes, so pessimistically maxRequestSize/64 = 8K. + // + // Chunk so that at least 2 packets are needed to fill a task. + if estimate, err := estimateRemainingSlots(len(keys), lastKey); err == nil { + if n := estimate / (2 * (maxRequestSize / 64)); n+1 < chunks { + chunks = n + 1 } + log.Debug("Chunked large contract", "initiators", len(keys), "tail", lastKey, "remaining", estimate, "chunks", chunks) + } else { + log.Debug("Chunked large contract", "initiators", len(keys), "tail", lastKey, "chunks", chunks) + } + r := newHashRange(lastKey, chunks) + + // Our first task is the one that was just filled by this response. + batch := s.db.NewBatch() + tasks = append(tasks, &storageTask{ + Next: common.Hash{}, + Last: r.End(), + root: acc.Root, + genBatch: batch, + genTrie: trie.NewStackTrie(batch), + }) + for r.Next() { + batch := s.db.NewBatch() tasks = append(tasks, &storageTask{ - Next: next, - Last: last, - root: acc.Root, + Next: r.Start(), + Last: r.End(), + root: acc.Root, + genBatch: batch, + genTrie: trie.NewStackTrie(batch), }) - log.Debug("Created storage sync task", "account", account, "root", acc.Root, "from", next, "last", last) - next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) + } + for _, task := range tasks { + log.Debug("Created storage sync task", "account", account, "root", acc.Root, "from", task.Next, "last", task.Last) } res.mainTask.SubTasks[account] = tasks @@ -1805,74 +1861,90 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { if res.subTask != nil { // Ensure the response doesn't overflow into the subsequent task last := res.subTask.Last.Big() - for k, hash := range res.hashes[i] { - // Mark the range complete if the last is already included. - // Keep iteration to delete the extra states if exists. - cmp := hash.Big().Cmp(last) - if cmp == 0 { + // Find the first overflowing key. While at it, mark res as complete + // if we find the range to include or pass the 'last' + index := sort.Search(len(res.hashes[i]), func(k int) bool { + cmp := res.hashes[i][k].Big().Cmp(last) + if cmp >= 0 { res.cont = false - continue - } - if cmp > 0 { - // Chunk overflown, cut off excess, but also update the boundary - for l := k; l < len(res.hashes[i]); l++ { - if err := res.tries[i].Prove(res.hashes[i][l][:], 0, res.overflow); err != nil { - panic(err) // Account range was already proven, what happened - } - } - res.hashes[i] = res.hashes[i][:k] - res.slots[i] = res.slots[i][:k] - res.cont = false // Mark range completed - break } + return cmp > 0 + }) + if index >= 0 { + // cut off excess + res.hashes[i] = res.hashes[i][:index] + res.slots[i] = res.slots[i][:index] } // Forward the relevant storage chunk (even if created just now) if res.cont { - res.subTask.Next = common.BigToHash(new(big.Int).Add(res.hashes[i][len(res.hashes[i])-1].Big(), big.NewInt(1))) + res.subTask.Next = incHash(res.hashes[i][len(res.hashes[i])-1]) } else { res.subTask.done = true } } } // Iterate over all the reconstructed trie nodes and push them to disk + // if the contract is fully delivered. If it's chunked, the trie nodes + // will be reconstructed later. slots += len(res.hashes[i]) - it := res.nodes[i].NewIterator(nil, nil) - for it.Next() { - // Boundary nodes are not written for the last result, since they are incomplete - if i == len(res.hashes)-1 && res.subTask != nil { - if _, ok := res.bounds[common.BytesToHash(it.Key())]; ok { - skipped++ - continue - } - if _, err := res.overflow.Get(it.Key()); err == nil { - skipped++ - continue - } - } - // Node is not a boundary, persist to disk - batch.Put(it.Key(), it.Value()) + if i < len(res.hashes)-1 || res.subTask == nil { + it := res.nodes[i].NewIterator(nil, nil) + for it.Next() { + batch.Put(it.Key(), it.Value()) - bytes += common.StorageSize(common.HashLength + len(it.Value())) - nodes++ + bytes += common.StorageSize(common.HashLength + len(it.Value())) + nodes++ + } + it.Release() } - it.Release() - // Persist the received storage segements. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. for j := 0; j < len(res.hashes[i]); j++ { rawdb.WriteStorageSnapshot(batch, account, res.hashes[i][j], res.slots[i][j]) bytes += common.StorageSize(1 + 2*common.HashLength + len(res.slots[i][j])) + + // If we're storing large contracts, generate the trie nodes + // on the fly to not trash the gluing points + if i == len(res.hashes)-1 && res.subTask != nil { + res.subTask.genTrie.Update(res.hashes[i][j][:], res.slots[i][j]) + } } } + // Large contracts could have generated new trie nodes, flush them to disk + if res.subTask != nil { + if res.subTask.done { + if root, err := res.subTask.genTrie.Commit(); err != nil { + log.Error("Failed to commit stack slots", "err", err) + } else if root == res.subTask.root { + // If the chunk's root is an overflown but full delivery, clear the heal request + for i, account := range res.mainTask.res.hashes { + if account == res.accounts[len(res.accounts)-1] { + res.mainTask.needHeal[i] = false + } + } + } + } + if data := res.subTask.genBatch.ValueSize(); data > ethdb.IdealBatchSize || res.subTask.done { + keys := res.subTask.genBatch.KeyCount() + if err := res.subTask.genBatch.Write(); err != nil { + log.Error("Failed to persist stack slots", "err", err) + } + res.subTask.genBatch.Reset() + + bytes += common.StorageSize(keys*common.HashLength + data) + nodes += keys + } + } + // Flush anything written just now and update the stats if err := batch.Write(); err != nil { log.Crit("Failed to persist storage slots", "err", err) } s.storageSynced += uint64(slots) s.storageBytes += bytes - log.Debug("Persisted set of storage slots", "accounts", len(res.hashes), "slots", slots, "nodes", nodes, "skipped", skipped, "bytes", bytes) + log.Debug("Persisted set of storage slots", "accounts", len(res.hashes), "slots", slots, "nodes", nodes, "bytes", bytes) // If this delivery completed the last pending task, forward the account task // to the next chunk @@ -1967,87 +2039,69 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } task.res = nil - // Iterate over all the accounts and gather all the incomplete trie nodes. A - // node is incomplete if we haven't yet filled it (sync was interrupted), or - // if we filled it in multiple chunks (storage trie), in which case the few - // nodes on the chunk boundaries are missing. - incompletes := light.NewNodeSet() - for i := range res.accounts { - // If the filling was interrupted, mark everything after as incomplete + // Persist the received account segements. These flat state maybe + // outdated during the sync, but it can be fixed later during the + // snapshot generation. + var ( + nodes int + bytes common.StorageSize + ) + batch := s.db.NewBatch() + for i, hash := range res.hashes { if task.needCode[i] || task.needState[i] { - for j := i; j < len(res.accounts); j++ { - if err := res.trie.Prove(res.hashes[j][:], 0, incompletes); err != nil { - panic(err) // Account range was already proven, what happened - } - } break } - // Filling not interrupted until this point, mark incomplete if needs healing - if task.needHeal[i] { - if err := res.trie.Prove(res.hashes[i][:], 0, incompletes); err != nil { - panic(err) // Account range was already proven, what happened - } - } - } - // Persist every finalized trie node that's not on the boundary - batch := s.db.NewBatch() + slim := snapshot.SlimAccountRLP(res.accounts[i].Nonce, res.accounts[i].Balance, res.accounts[i].Root, res.accounts[i].CodeHash) + rawdb.WriteAccountSnapshot(batch, hash, slim) + bytes += common.StorageSize(1 + common.HashLength + len(slim)) - var ( - nodes int - skipped int - bytes common.StorageSize - ) - it := res.nodes.NewIterator(nil, nil) - for it.Next() { - // Boundary nodes are not written, since they are incomplete - if _, ok := res.bounds[common.BytesToHash(it.Key())]; ok { - skipped++ - continue - } - // Overflow nodes are not written, since they mess with another task - if _, err := res.overflow.Get(it.Key()); err == nil { - skipped++ - continue - } - // Accounts with split storage requests are incomplete - if _, err := incompletes.Get(it.Key()); err == nil { - skipped++ - continue + // If the task is complete, drop it into the stack trie to generate + // account trie nodes for it + if !task.needHeal[i] { + full, err := snapshot.FullAccountRLP(slim) // TODO(karalabe): Slim parsing can be omitted + if err != nil { + panic(err) // Really shouldn't ever happen + } + task.genTrie.Update(hash[:], full) } - // Node is neither a boundary, not an incomplete account, persist to disk - batch.Put(it.Key(), it.Value()) - - bytes += common.StorageSize(common.HashLength + len(it.Value())) - nodes++ - } - it.Release() - - // Persist the received account segements. These flat state maybe - // outdated during the sync, but it can be fixed later during the - // snapshot generation. - for i, hash := range res.hashes { - blob := snapshot.SlimAccountRLP(res.accounts[i].Nonce, res.accounts[i].Balance, res.accounts[i].Root, res.accounts[i].CodeHash) - rawdb.WriteAccountSnapshot(batch, hash, blob) - bytes += common.StorageSize(1 + common.HashLength + len(blob)) } + // Flush anything written just now and update the stats if err := batch.Write(); err != nil { log.Crit("Failed to persist accounts", "err", err) } s.accountBytes += bytes s.accountSynced += uint64(len(res.accounts)) - log.Debug("Persisted range of accounts", "accounts", len(res.accounts), "nodes", nodes, "skipped", skipped, "bytes", bytes) - // Task filling persisted, push it the chunk marker forward to the first // account still missing data. for i, hash := range res.hashes { if task.needCode[i] || task.needState[i] { return } - task.Next = common.BigToHash(new(big.Int).Add(hash.Big(), big.NewInt(1))) + task.Next = incHash(hash) } // All accounts marked as complete, track if the entire task is done task.done = !res.cont + + // Stack trie could have generated trie nodes, push them to disk (we need to + // flush after finalizing task.done. It's fine even if we crash and lose this + // write as it will only cause more data to be downloaded during heal. + if task.done { + if _, err := task.genTrie.Commit(); err != nil { + log.Error("Failed to commit stack account", "err", err) + } + } + if data := task.genBatch.ValueSize(); data > ethdb.IdealBatchSize || task.done { + keys := task.genBatch.KeyCount() + if err := task.genBatch.Write(); err != nil { + log.Error("Failed to persist stack account", "err", err) + } + task.genBatch.Reset() + + nodes += keys + bytes += common.StorageSize(keys*common.HashLength + data) + } + log.Debug("Persisted range of accounts", "accounts", len(res.accounts), "nodes", nodes, "bytes", bytes) } // OnAccounts is a callback method to invoke when a range of accounts are @@ -2091,7 +2145,6 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco s.lock.Unlock() return nil } - // Response is valid, but check if peer is signalling that it does not have // the requested data. For account range queries that means the state being // retrieved was either already pruned remotely, or the peer is not yet @@ -2123,22 +2176,13 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco if len(keys) > 0 { end = keys[len(keys)-1] } - db, tr, notary, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) + _, _, _, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) if err != nil { logger.Warn("Account range failed proof", "err", err) // Signal this request as failed, and ready for rescheduling s.scheduleRevertAccountRequest(req) return err } - // Partial trie reconstructed, send it to the scheduler for storage filling - bounds := make(map[common.Hash]struct{}) - - it := notary.Accessed().NewIterator(nil, nil) - for it.Next() { - bounds[common.BytesToHash(it.Key())] = struct{}{} - } - it.Release() - accs := make([]*state.Account, len(accounts)) for i, account := range accounts { acc := new(state.Account) @@ -2151,10 +2195,6 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco task: req.task, hashes: hashes, accounts: accs, - nodes: db, - trie: tr, - bounds: bounds, - overflow: light.NewNodeSet(), cont: cont, } select { @@ -2354,10 +2394,8 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Reconstruct the partial tries from the response and verify them var ( - dbs = make([]ethdb.KeyValueStore, len(hashes)) - tries = make([]*trie.Trie, len(hashes)) - notary *trie.KeyValueNotary - cont bool + dbs = make([]ethdb.KeyValueStore, len(hashes)) + cont bool ) for i := 0; i < len(hashes); i++ { // Convert the keys and proofs into an internal format @@ -2375,7 +2413,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(nodes) == 0 { // No proof has been attached, the response must cover the entire key // space and hash to the origin root. - dbs[i], tries[i], _, _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) + dbs[i], _, _, _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage slots failed proof", "err", err) @@ -2390,7 +2428,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(keys) > 0 { end = keys[len(keys)-1] } - dbs[i], tries[i], notary, cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) + dbs[i], _, _, cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage range failed proof", "err", err) @@ -2399,15 +2437,6 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo } } // Partial tries reconstructed, send them to the scheduler for storage filling - bounds := make(map[common.Hash]struct{}) - - if notary != nil { // if all contract storages are delivered in full, no notary will be created - it := notary.Accessed().NewIterator(nil, nil) - for it.Next() { - bounds[common.BytesToHash(it.Key())] = struct{}{} - } - it.Release() - } response := &storageResponse{ mainTask: req.mainTask, subTask: req.subTask, @@ -2416,9 +2445,6 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo hashes: hashes, slots: slots, nodes: dbs, - tries: tries, - bounds: bounds, - overflow: light.NewNodeSet(), cont: cont, } select { @@ -2658,7 +2684,7 @@ func (s *Syncer) report(force bool) { // reportSyncProgress calculates various status reports and provides it to the user. func (s *Syncer) reportSyncProgress(force bool) { // Don't report all the events, just occasionally - if !force && time.Since(s.logTime) < 3*time.Second { + if !force && time.Since(s.logTime) < 8*time.Second { return } // Don't report anything until we have a meaningful progress @@ -2697,7 +2723,7 @@ func (s *Syncer) reportSyncProgress(force bool) { // reportHealProgress calculates various status reports and provides it to the user. func (s *Syncer) reportHealProgress(force bool) { // Don't report all the events, just occasionally - if !force && time.Since(s.logTime) < 3*time.Second { + if !force && time.Since(s.logTime) < 8*time.Second { return } s.logTime = time.Now() @@ -2712,3 +2738,19 @@ func (s *Syncer) reportHealProgress(force bool) { log.Info("State heal in progress", "accounts", accounts, "slots", storage, "codes", bytecode, "nodes", trienode, "pending", s.healer.scheduler.Pending()) } + +// estimateRemainingSlots tries to determine roughly how many slots are left in +// a contract storage, based on the number of keys and the last hash. This method +// assumes that the hashes are lexicographically ordered and evenly distributed. +func estimateRemainingSlots(hashes int, last common.Hash) (uint64, error) { + if last == (common.Hash{}) { + return 0, errors.New("last hash empty") + } + space := new(big.Int).Mul(math.MaxBig256, big.NewInt(int64(hashes))) + space.Div(space, last.Big()) + if !space.IsUint64() { + // Gigantic address space probably due to too few or malicious slots + return 0, errors.New("too few slots for estimation") + } + return space.Uint64() - uint64(hashes), nil +} diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 3e9778dbc7..a1cc3581a8 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -135,6 +135,12 @@ type testPeer struct { trieRequestHandler trieHandlerFunc codeRequestHandler codeHandlerFunc term func() + + // counters + nAccountRequests int + nStorageRequests int + nBytecodeRequests int + nTrienodeRequests int } func newTestPeer(id string, t *testing.T, term func()) *testPeer { @@ -156,19 +162,30 @@ func newTestPeer(id string, t *testing.T, term func()) *testPeer { func (t *testPeer) ID() string { return t.id } func (t *testPeer) Log() log.Logger { return t.logger } +func (t *testPeer) Stats() string { + return fmt.Sprintf(`Account requests: %d +Storage requests: %d +Bytecode requests: %d +Trienode requests: %d +`, t.nAccountRequests, t.nStorageRequests, t.nBytecodeRequests, t.nTrienodeRequests) +} + func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { t.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) + t.nAccountRequests++ go t.accountRequestHandler(t, id, root, origin, limit, bytes) return nil } func (t *testPeer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { t.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) + t.nTrienodeRequests++ go t.trieRequestHandler(t, id, root, paths, bytes) return nil } func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { + t.nStorageRequests++ if len(accounts) == 1 && origin != nil { t.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) } else { @@ -179,6 +196,7 @@ func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts [] } func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { + t.nBytecodeRequests++ t.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) go t.codeRequestHandler(t, id, hashes, bytes) return nil @@ -1365,7 +1383,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { step := new(big.Int).Sub( new(big.Int).Div( new(big.Int).Exp(common.Big2, common.Big256, nil), - big.NewInt(accountConcurrency), + big.NewInt(int64(accountConcurrency)), ), common.Big1, ) for i := 0; i < accountConcurrency; i++ { @@ -1529,7 +1547,7 @@ func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) step := new(big.Int).Sub( new(big.Int).Div( new(big.Int).Exp(common.Big2, common.Big256, nil), - big.NewInt(accountConcurrency), + big.NewInt(int64(accountConcurrency)), ), common.Big1, ) for i := 0; i < accountConcurrency; i++ { @@ -1605,3 +1623,94 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } t.Logf("accounts: %d, slots: %d", accounts, slots) } + +// TestSyncAccountPerformance tests how efficient the snap algo is at minimizing +// state healing +func TestSyncAccountPerformance(t *testing.T) { + // Set the account concurrency to 1. This _should_ result in the + // range root to become correct, and there should be no healing needed + defer func(old int) { accountConcurrency = old }(accountConcurrency) + accountConcurrency = 1 + + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, term) + source.accountTrie = sourceAccountTrie + source.accountValues = elems + return source + } + src := mkSource("source") + syncer := setupSyncer(src) + if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + // The trie root will always be requested, since it is added when the snap + // sync cycle starts. When popping the queue, we do not look it up again. + // Doing so would bring this number down to zero in this artificial testcase, + // but only add extra IO for no reason in practice. + if have, want := src.nTrienodeRequests, 1; have != want { + fmt.Printf(src.Stats()) + t.Errorf("trie node heal requests wrong, want %d, have %d", want, have) + } +} + +func TestSlotEstimation(t *testing.T) { + for i, tc := range []struct { + last common.Hash + count int + want uint64 + }{ + { + // Half the space + common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + 100, + 100, + }, + { + // 1 / 16th + common.HexToHash("0x0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + 100, + 1500, + }, + { + // Bit more than 1 / 16th + common.HexToHash("0x1000000000000000000000000000000000000000000000000000000000000000"), + 100, + 1499, + }, + { + // Almost everything + common.HexToHash("0xF000000000000000000000000000000000000000000000000000000000000000"), + 100, + 6, + }, + { + // Almost nothing -- should lead to error + common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"), + 1, + 0, + }, + { + // Nothing -- should lead to error + common.Hash{}, + 100, + 0, + }, + } { + have, _ := estimateRemainingSlots(tc.count, tc.last) + if want := tc.want; have != want { + t.Errorf("test %d: have %d want %d", i, have, want) + } + } +} diff --git a/ethdb/batch.go b/ethdb/batch.go index e261415bff..5f8207fc46 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -25,6 +25,9 @@ const IdealBatchSize = 100 * 1024 type Batch interface { KeyValueWriter + // KeyCount retrieves the number of keys queued up for writing. + KeyCount() int + // ValueSize retrieves the amount of data queued up for writing. ValueSize() int diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 5d19cc3577..da00226e95 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -448,6 +448,7 @@ func (db *Database) meter(refresh time.Duration) { type batch struct { db *leveldb.DB b *leveldb.Batch + keys int size int } @@ -461,10 +462,16 @@ func (b *batch) Put(key, value []byte) error { // Delete inserts the a key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { b.b.Delete(key) + b.keys++ b.size += len(key) return nil } +// KeyCount retrieves the number of keys queued up for writing. +func (b *batch) KeyCount() int { + return b.keys +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -478,7 +485,7 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { b.b.Reset() - b.size = 0 + b.keys, b.size = 0, 0 } // Replay replays the batch contents. diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index fedc9e326c..ded9f5e66c 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -198,6 +198,7 @@ type keyvalue struct { type batch struct { db *Database writes []keyvalue + keys int size int } @@ -211,10 +212,16 @@ func (b *batch) Put(key, value []byte) error { // Delete inserts the a key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true}) + b.keys++ b.size += len(key) return nil } +// KeyCount retrieves the number of keys queued up for writing. +func (b *batch) KeyCount() int { + return b.keys +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -238,7 +245,7 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { b.writes = b.writes[:0] - b.size = 0 + b.keys, b.size = 0, 0 } // Replay replays the batch contents. diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 5cea7769c2..0013c919c9 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -90,6 +90,7 @@ func (b *spongeBatch) Put(key, value []byte) error { return nil } func (b *spongeBatch) Delete(key []byte) error { panic("implement me") } +func (b *spongeBatch) KeyCount() int { panic("not implemented") } func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} diff --git a/trie/trie_test.go b/trie/trie_test.go index 492b423c2f..44fddf87e4 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -706,6 +706,7 @@ func (b *spongeBatch) Put(key, value []byte) error { return nil } func (b *spongeBatch) Delete(key []byte) error { panic("implement me") } +func (b *spongeBatch) KeyCount() int { return 100 } func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} From 45fca44c24a1067285f341ccd9b4abdf1334604f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 27 Apr 2021 18:09:34 +0200 Subject: [PATCH 305/709] p2p/tracker: properly clean up fulfilled requests --- p2p/tracker/tracker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index b50a952f62..19852ad6ab 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -186,6 +186,7 @@ func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) { } // Everything matches, mark the request serviced and meter it t.expire.Remove(req.expire) + delete(t.pending, id) if req.expire.Prev() == nil { t.wake.Stop() t.schedule() From ff3535e8e03ea56207109f57ec3b4a30481d2fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Apr 2021 21:47:59 +0300 Subject: [PATCH 306/709] p2p/tracker: only reschedule wake if previous didn't run --- p2p/tracker/tracker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 19852ad6ab..69a49087e2 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -188,8 +188,9 @@ func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) { t.expire.Remove(req.expire) delete(t.pending, id) if req.expire.Prev() == nil { - t.wake.Stop() - t.schedule() + if t.wake.Stop() { + t.schedule() + } } g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) metrics.GetOrRegisterGauge(g, nil).Dec(1) From 0c998684167278fb09d2f5412a64fc225dc08392 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Apr 2021 08:48:07 +0200 Subject: [PATCH 307/709] cmd/devp2p, eth/protocols/eth: fix tests + make sanity checks earlier (#22749) --- cmd/devp2p/internal/ethtest/eth66_suite.go | 4 ++-- cmd/devp2p/internal/ethtest/large.go | 2 +- cmd/devp2p/internal/ethtest/suite_test.go | 12 ++++++++++-- eth/protocols/eth/handlers.go | 6 +++--- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 176d8bf33c..4265b25f6a 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -483,8 +483,8 @@ func (s *Suite) TestNewPooledTxs_66(t *utesting.T) { t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) } return - case *NewPooledTransactionHashes: - // ignore propagated txs from old tests + case *NewPooledTransactionHashes, *NewBlock, *NewBlockHashes: + // ignore propagated txs and blocks from old tests continue default: t.Fatalf("unexpected %s", pretty.Sdump(msg)) diff --git a/cmd/devp2p/internal/ethtest/large.go b/cmd/devp2p/internal/ethtest/large.go index deca00be53..22421355ab 100644 --- a/cmd/devp2p/internal/ethtest/large.go +++ b/cmd/devp2p/internal/ethtest/large.go @@ -70,7 +70,7 @@ func largeHeader() *types.Header { GasUsed: 0, Coinbase: common.Address{}, GasLimit: 0, - UncleHash: randHash(), + UncleHash: types.EmptyUncleHash, Time: 1337, ParentHash: randHash(), Root: randHash(), diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 2c628757bc..6e3217151a 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -19,6 +19,7 @@ package ethtest import ( "os" "testing" + "time" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -87,8 +88,15 @@ func setupGeth(stack *node.Node) error { } backend, err := eth.New(stack, ðconfig.Config{ - Genesis: &chain.genesis, - NetworkId: chain.genesis.Config.ChainID.Uint64(), // 19763 + Genesis: &chain.genesis, + NetworkId: chain.genesis.Config.ChainID.Uint64(), // 19763 + DatabaseCache: 10, + TrieCleanCache: 10, + TrieCleanCacheJournal: "", + TrieCleanCacheRejournal: 60 * time.Minute, + TrieDirtyCache: 16, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 10, }) if err != nil { return err diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index d0dec7b0b2..d7d993a23d 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -292,6 +292,9 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(ann); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + if err := ann.sanityCheck(); err != nil { + return err + } if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) return nil // TODO(karalabe): return error eventually, but wait a few releases @@ -300,9 +303,6 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) return nil // TODO(karalabe): return error eventually, but wait a few releases } - if err := ann.sanityCheck(); err != nil { - return err - } ann.Block.ReceivedAt = msg.Time() ann.Block.ReceivedFrom = peer From 256c5d68b245899f6d37c72636fdd795f66397ee Mon Sep 17 00:00:00 2001 From: Gregory Markou <16929357+GregTheGreek@users.noreply.github.com> Date: Wed, 28 Apr 2021 03:06:34 -0400 Subject: [PATCH 308/709] eth/gasprice: improve stability of estimated price (#22722) This PR makes the gas price oracle ignore transactions priced at `<=1 wei`. --- eth/gasprice/gasprice.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 5d8be08e0b..560722bec0 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -199,6 +199,9 @@ func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, bloc var prices []*big.Int for _, tx := range txs { + if tx.GasPriceIntCmp(common.Big1) <= 0 { + continue + } sender, err := types.Sender(signer, tx) if err == nil && sender != block.Coinbase() { prices = append(prices, tx.GasPrice()) From 9e5bb84c0e162c54f4532641084b9c0627b15e85 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Wed, 28 Apr 2021 05:04:25 -0500 Subject: [PATCH 309/709] tests/fuzzers: crypto/bn256 and crypto/bls12381 tests against gnark-crypto (#22755) Add more cross-fuzzers to fuzz bls with gnark versus geth's own bls12-381 library --- go.mod | 15 +- go.sum | 153 ++--------- oss-fuzz.sh | 5 + tests/fuzzers/bls12381/bls12381_fuzz.go | 244 ++++++++++++++++++ .../{bls_fuzzer.go => precompile_fuzzer.go} | 0 tests/fuzzers/bn256/bn256_fuzz.go | 48 ++-- 6 files changed, 306 insertions(+), 159 deletions(-) create mode 100644 tests/fuzzers/bls12381/bls12381_fuzz.go rename tests/fuzzers/bls12381/{bls_fuzzer.go => precompile_fuzzer.go} (100%) diff --git a/go.mod b/go.mod index 74c406f78b..63636caae1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,10 @@ module github.com/ethereum/go-ethereum go 1.15 require ( + github.com/Azure/azure-pipeline-go v0.2.2 // indirect github.com/Azure/azure-storage-blob-go v0.7.0 + github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect + github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.5.7 github.com/aws/aws-sdk-go-v2 v1.2.0 github.com/aws/aws-sdk-go-v2/config v1.1.1 @@ -12,15 +15,18 @@ require ( github.com/btcsuite/btcd v0.20.1-beta github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.14.0 - github.com/consensys/gurvy v0.3.8 + github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea + github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 github.com/edsrzf/mmap-go v1.0.0 github.com/fatih/color v1.7.0 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff + github.com/go-ole/go-ole v1.2.1 // indirect + github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 @@ -37,8 +43,10 @@ require ( github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 + github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 @@ -51,11 +59,12 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 golang.org/x/text v0.3.4 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 + gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index 0a9444323c..cadcd1de95 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,6 @@ cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbf cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= @@ -56,11 +55,6 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.2.0 h1:BS+UYpbsElC82gB+2E2jiCBg36i8HlubTB/dO/moQ9c= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1 h1:ZAoq32boMzcaTW9bcUacBswAmHTbvlvDJICgHFZuECo= @@ -81,11 +75,8 @@ github.com/aws/smithy-go v1.1.0 h1:D6CSsM3gdxaGaqXnPgOBCeL6Mophqzu7KJOu7zW78sU= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -107,20 +98,12 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cloudflare/cloudflare-go v0.14.0 h1:gFqGlGl/5f9UGXAaKapCGUfaTCgRKKnzu2VvzMZlOFA= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= -github.com/consensys/bavard v0.1.8-0.20210105233146-c16790d2aa8b/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= -github.com/consensys/goff v0.3.10/go.mod h1:xTldOBEHmFiYS0gPXd3NsaEqZWlnmeWcRLWgD3ba3xc= -github.com/consensys/gurvy v0.3.8 h1:H2hvjvT2OFMgdMn5ZbhXqHt+F8DJ2clZW7Vmc0kFFxc= -github.com/consensys/gurvy v0.3.8/go.mod h1:sN75xnsiD593XnhbhvG2PkOy194pZBzqShWF/kwuW/g= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -138,18 +121,13 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmak github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= -github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -158,7 +136,6 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -177,12 +154,10 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -225,51 +200,25 @@ github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 h1:bcAj8KroPf552TScjFPIakjH2/tdIrIH8F+cc4v4SRo= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= @@ -283,25 +232,20 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kilic/bls12-381 v0.0.0-20201226121925-69dacb279461/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= @@ -319,37 +263,24 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -361,8 +292,6 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -378,10 +307,7 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -394,62 +320,40 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= -github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= -github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -458,8 +362,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -467,41 +369,34 @@ github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -519,19 +414,16 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -545,9 +437,10 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d h1:1aflnvSoWWLI2k/dMUAl5lvU1YO4Mb4hz0gh+1rjcxU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -559,13 +452,12 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -585,14 +477,13 @@ golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -604,7 +495,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -624,7 +514,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -632,7 +521,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -688,17 +577,14 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -715,5 +601,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/oss-fuzz.sh b/oss-fuzz.sh index f8152f0fad..a9bac03257 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -114,5 +114,10 @@ compile_fuzzer tests/fuzzers/bls12381 FuzzPairing fuzz_pairing compile_fuzzer tests/fuzzers/bls12381 FuzzMapG1 fuzz_map_g1 compile_fuzzer tests/fuzzers/bls12381 FuzzMapG2 fuzz_map_g2 +compile_fuzzer tests/fuzzers/bls12381 FuzzCrossG1Add fuzz_cross_g1_add +compile_fuzzer tests/fuzzers/bls12381 FuzzCrossG1MultiExp fuzz_cross_g1_multiexp +compile_fuzzer tests/fuzzers/bls12381 FuzzCrossG2Add fuzz_cross_g2_add +compile_fuzzer tests/fuzzers/bls12381 FuzzCrossPairing fuzz_cross_pairing + #TODO: move this to tests/fuzzers, if possible compile_fuzzer crypto/blake2b Fuzz fuzzBlake2b diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go new file mode 100644 index 0000000000..298050ad36 --- /dev/null +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -0,0 +1,244 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build gofuzz + +package bls + +import ( + "bytes" + "crypto/rand" + "fmt" + "io" + "math/big" + + gnark "github.com/consensys/gnark-crypto/ecc/bls12-381" + "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" + "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" + "github.com/ethereum/go-ethereum/crypto/bls12381" +) + +func FuzzCrossPairing(data []byte) int { + input := bytes.NewReader(data) + + // get random G1 points + kpG1, cpG1, err := getG1Points(input) + if err != nil { + return 0 + } + + // get random G2 points + kpG2, cpG2, err := getG2Points(input) + if err != nil { + return 0 + } + + // compute pairing using geth + engine := bls12381.NewPairingEngine() + engine.AddPair(kpG1, kpG2) + kResult := engine.Result() + + // compute pairing using gnark + cResult, err := gnark.Pair([]gnark.G1Affine{*cpG1}, []gnark.G2Affine{*cpG2}) + if err != nil { + panic(fmt.Sprintf("gnark/bls12381 encountered error: %v", err)) + } + + // compare result + if !(bytes.Equal(cResult.Marshal(), bls12381.NewGT().ToBytes(kResult))) { + panic("pairing mismatch gnark / geth ") + } + + return 1 +} + +func FuzzCrossG1Add(data []byte) int { + input := bytes.NewReader(data) + + // get random G1 points + kp1, cp1, err := getG1Points(input) + if err != nil { + return 0 + } + + // get random G1 points + kp2, cp2, err := getG1Points(input) + if err != nil { + return 0 + } + + // compute kp = kp1 + kp2 + g1 := bls12381.NewG1() + kp := bls12381.PointG1{} + g1.Add(&kp, kp1, kp2) + + // compute cp = cp1 + cp2 + _cp1 := new(gnark.G1Jac).FromAffine(cp1) + _cp2 := new(gnark.G1Jac).FromAffine(cp2) + cp := new(gnark.G1Affine).FromJacobian(_cp1.AddAssign(_cp2)) + + // compare result + if !(bytes.Equal(cp.Marshal(), g1.ToBytes(&kp))) { + panic("G1 point addition mismatch gnark / geth ") + } + + return 1 +} + +func FuzzCrossG2Add(data []byte) int { + input := bytes.NewReader(data) + + // get random G2 points + kp1, cp1, err := getG2Points(input) + if err != nil { + return 0 + } + + // get random G2 points + kp2, cp2, err := getG2Points(input) + if err != nil { + return 0 + } + + // compute kp = kp1 + kp2 + g2 := bls12381.NewG2() + kp := bls12381.PointG2{} + g2.Add(&kp, kp1, kp2) + + // compute cp = cp1 + cp2 + _cp1 := new(gnark.G2Jac).FromAffine(cp1) + _cp2 := new(gnark.G2Jac).FromAffine(cp2) + cp := new(gnark.G2Affine).FromJacobian(_cp1.AddAssign(_cp2)) + + // compare result + if !(bytes.Equal(cp.Marshal(), g2.ToBytes(&kp))) { + panic("G2 point addition mismatch gnark / geth ") + } + + return 1 +} + +func FuzzCrossG1MultiExp(data []byte) int { + var ( + input = bytes.NewReader(data) + gethScalars []*big.Int + gnarkScalars []fr.Element + gethPoints []*bls12381.PointG1 + gnarkPoints []gnark.G1Affine + ) + // n random scalars (max 17) + for i := 0; i < 17; i++ { + // note that geth/crypto/bls12381 works only with scalars <= 32bytes + s, err := randomScalar(input, fr.Modulus()) + if err != nil { + break + } + // get a random G1 point as basis + kp1, cp1, err := getG1Points(input) + if err != nil { + break + } + gethScalars = append(gethScalars, s) + var gnarkScalar = &fr.Element{} + gnarkScalar = gnarkScalar.SetBigInt(s).FromMont() + gnarkScalars = append(gnarkScalars, *gnarkScalar) + + gethPoints = append(gethPoints, new(bls12381.PointG1).Set(kp1)) + gnarkPoints = append(gnarkPoints, *cp1) + } + if len(gethScalars) == 0{ + return 0 + } + // compute multi exponentiation + g1 := bls12381.NewG1() + kp := bls12381.PointG1{} + if _, err := g1.MultiExp(&kp, gethPoints, gethScalars); err != nil { + panic(fmt.Sprintf("G1 multi exponentiation errored (geth): %v", err)) + } + // note that geth/crypto/bls12381.MultiExp mutates the scalars slice (and sets all the scalars to zero) + + // gnark multi exp + cp := new(gnark.G1Affine) + cp.MultiExp(gnarkPoints, gnarkScalars) + + // compare result + if !(bytes.Equal(cp.Marshal(), g1.ToBytes(&kp))) { + panic("G1 multi exponentiation mismatch gnark / geth ") + } + + return 1 +} + +func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, error) { + // sample a random scalar + s, err := randomScalar(input, fp.Modulus()) + if err != nil { + return nil, nil, err + } + + // compute a random point + cp := new(gnark.G1Affine) + _, _, g1Gen, _ := gnark.Generators() + cp.ScalarMultiplication(&g1Gen, s) + cpBytes := cp.Marshal() + + // marshal gnark point -> geth point + g1 := bls12381.NewG1() + kp, err := g1.FromBytes(cpBytes) + if err != nil { + panic(fmt.Sprintf("Could not marshal gnark.G1 -> geth.G1: %v", err)) + } + if !bytes.Equal(g1.ToBytes(kp), cpBytes) { + panic("bytes(gnark.G1) != bytes(geth.G1)") + } + + return kp, cp, nil +} + +func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, error) { + // sample a random scalar + s, err := randomScalar(input, fp.Modulus()) + if err != nil { + return nil, nil, err + } + + // compute a random point + cp := new(gnark.G2Affine) + _, _, _, g2Gen := gnark.Generators() + cp.ScalarMultiplication(&g2Gen, s) + cpBytes := cp.Marshal() + + // marshal gnark point -> geth point + g2 := bls12381.NewG2() + kp, err := g2.FromBytes(cpBytes) + if err != nil { + panic(fmt.Sprintf("Could not marshal gnark.G2 -> geth.G2: %v", err)) + } + if !bytes.Equal(g2.ToBytes(kp), cpBytes) { + panic("bytes(gnark.G2) != bytes(geth.G2)") + } + + return kp, cp, nil +} + +func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { + for { + k, err = rand.Int(r, max) + if err != nil || k.Sign() > 0 { + return + } + } +} diff --git a/tests/fuzzers/bls12381/bls_fuzzer.go b/tests/fuzzers/bls12381/precompile_fuzzer.go similarity index 100% rename from tests/fuzzers/bls12381/bls_fuzzer.go rename to tests/fuzzers/bls12381/precompile_fuzzer.go diff --git a/tests/fuzzers/bn256/bn256_fuzz.go b/tests/fuzzers/bn256/bn256_fuzz.go index c98fbc33ae..030ac19b3f 100644 --- a/tests/fuzzers/bn256/bn256_fuzz.go +++ b/tests/fuzzers/bn256/bn256_fuzz.go @@ -12,12 +12,12 @@ import ( "io" "math/big" - gurvy "github.com/consensys/gurvy/bn256" + "github.com/consensys/gnark-crypto/ecc/bn254" cloudflare "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare" google "github.com/ethereum/go-ethereum/crypto/bn256/google" ) -func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1, *gurvy.G1Affine) { +func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1, *bn254.G1Affine) { _, xc, err := cloudflare.RandomG1(input) if err != nil { // insufficient input @@ -25,16 +25,16 @@ func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1, *gurvy.G1Affine) } xg := new(google.G1) if _, err := xg.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err)) } - xs := new(gurvy.G1Affine) + xs := new(bn254.G1Affine) if err := xs.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> consensys:", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> gnark: %v", err)) } return xc, xg, xs } -func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2, *gurvy.G2Affine) { +func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2, *bn254.G2Affine) { _, xc, err := cloudflare.RandomG2(input) if err != nil { // insufficient input @@ -42,11 +42,11 @@ func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2, *gurvy.G2Affine) } xg := new(google.G2) if _, err := xg.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err)) } - xs := new(gurvy.G2Affine) + xs := new(bn254.G2Affine) if err := xs.Unmarshal(xc.Marshal()); err != nil { - panic(fmt.Sprintf("Could not marshal cloudflare -> consensys:", err)) + panic(fmt.Sprintf("Could not marshal cloudflare -> gnark: %v", err)) } return xc, xg, xs } @@ -70,16 +70,16 @@ func FuzzAdd(data []byte) int { rg := new(google.G1) rg.Add(xg, yg) - tmpX := new(gurvy.G1Jac).FromAffine(xs) - tmpY := new(gurvy.G1Jac).FromAffine(ys) - rs := new(gurvy.G1Affine).FromJacobian(tmpX.AddAssign(tmpY)) + tmpX := new(bn254.G1Jac).FromAffine(xs) + tmpY := new(bn254.G1Jac).FromAffine(ys) + rs := new(bn254.G1Affine).FromJacobian(tmpX.AddAssign(tmpY)) if !bytes.Equal(rc.Marshal(), rg.Marshal()) { panic("add mismatch: cloudflare/google") } if !bytes.Equal(rc.Marshal(), rs.Marshal()) { - panic("add mismatch: cloudflare/consensys") + panic("add mismatch: cloudflare/gnark") } return 1 } @@ -112,16 +112,16 @@ func FuzzMul(data []byte) int { rg := new(google.G1) rg.ScalarMult(pg, new(big.Int).SetBytes(buf)) - rs := new(gurvy.G1Jac) - psJac := new(gurvy.G1Jac).FromAffine(ps) + rs := new(bn254.G1Jac) + psJac := new(bn254.G1Jac).FromAffine(ps) rs.ScalarMultiplication(psJac, new(big.Int).SetBytes(buf)) - rsAffine := new(gurvy.G1Affine).FromJacobian(rs) + rsAffine := new(bn254.G1Affine).FromJacobian(rs) if !bytes.Equal(rc.Marshal(), rg.Marshal()) { panic("scalar mul mismatch: cloudflare/google") } if !bytes.Equal(rc.Marshal(), rsAffine.Marshal()) { - panic("scalar mul mismatch: cloudflare/consensys") + panic("scalar mul mismatch: cloudflare/gnark") } return 1 } @@ -136,18 +136,20 @@ func FuzzPair(data []byte) int { if tc == nil { return 0 } + // Pair the two points and ensure they result in the same output - clPair := cloudflare.PairingCheck([]*cloudflare.G1{pc}, []*cloudflare.G2{tc}) - if clPair != google.PairingCheck([]*google.G1{pg}, []*google.G2{tg}) { + clPair := cloudflare.Pair(pc, tc).Marshal() + gPair := google.Pair(pg, tg).Marshal() + if !bytes.Equal(clPair, gPair) { panic("pairing mismatch: cloudflare/google") } - coPair, err := gurvy.PairingCheck([]gurvy.G1Affine{*ps}, []gurvy.G2Affine{*ts}) + cPair, err := bn254.Pair([]bn254.G1Affine{*ps}, []bn254.G2Affine{*ts}) if err != nil { - panic(fmt.Sprintf("gurvy encountered error: %v", err)) + panic(fmt.Sprintf("gnark/bn254 encountered error: %v", err)) } - if clPair != coPair { - panic("pairing mismatch: cloudflare/consensys") + if !bytes.Equal(clPair, cPair.Marshal()) { + panic("pairing mismatch: cloudflare/gnark") } return 1 From 6d7c9566df5d0dcb0797ef505db91287fa9ac7ce Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 28 Apr 2021 20:18:25 +0800 Subject: [PATCH 310/709] les, tests: fix les clientpool (#22756) * les, tests: fix les clientpool * tests: disable debug mode * les: polish code --- les/vflux/server/clientpool.go | 7 +-- tests/fuzzers/vflux/clientpool-fuzzer.go | 76 +++++++++++++++++++----- tests/fuzzers/vflux/debug/main.go | 3 + 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go index 079d511704..351961b74e 100644 --- a/les/vflux/server/clientpool.go +++ b/les/vflux/server/clientpool.go @@ -239,12 +239,11 @@ func (cp *ClientPool) SetCapacity(node *enode.Node, reqCap uint64, bias time.Dur maxTarget = curve.maxCapacity(func(capacity uint64) int64 { return balance.estimatePriority(capacity, 0, 0, bias, false) }) - if maxTarget <= capacity { + if maxTarget < reqCap { return } - if maxTarget > reqCap { - maxTarget = reqCap - } + maxTarget = reqCap + // Specify a narrow target range that allows a limited number of fine step // iterations minTarget = maxTarget - maxTarget/20 diff --git a/tests/fuzzers/vflux/clientpool-fuzzer.go b/tests/fuzzers/vflux/clientpool-fuzzer.go index 41b8627348..0414c001ec 100644 --- a/tests/fuzzers/vflux/clientpool-fuzzer.go +++ b/tests/fuzzers/vflux/clientpool-fuzzer.go @@ -28,11 +28,22 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/les/vflux" vfs "github.com/ethereum/go-ethereum/les/vflux/server" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" ) +var ( + debugMode = false + doLog = func(msg string, ctx ...interface{}) { + if !debugMode { + return + } + log.Info(msg, ctx...) + } +) + type fuzzer struct { peers [256]*clientPeer disconnectList []*clientPeer @@ -65,6 +76,7 @@ func (p *clientPeer) InactiveAllowance() time.Duration { } func (p *clientPeer) UpdateCapacity(newCap uint64, requested bool) { + origin, originTotal := p.capacity, p.fuzzer.activeCap p.fuzzer.activeCap -= p.capacity if p.capacity != 0 { p.fuzzer.activeCount-- @@ -74,9 +86,11 @@ func (p *clientPeer) UpdateCapacity(newCap uint64, requested bool) { if p.capacity != 0 { p.fuzzer.activeCount++ } + doLog("Update capacity", "peer", p.node.ID(), "origin", origin, "cap", newCap, "origintotal", originTotal, "total", p.fuzzer.activeCap, "requested", requested) } func (p *clientPeer) Disconnect() { + origin, originTotal := p.capacity, p.fuzzer.activeCap p.fuzzer.disconnectList = append(p.fuzzer.disconnectList, p) p.fuzzer.activeCap -= p.capacity if p.capacity != 0 { @@ -84,6 +98,7 @@ func (p *clientPeer) Disconnect() { } p.capacity = 0 p.balance = nil + doLog("Disconnect", "peer", p.node.ID(), "origin", origin, "origintotal", originTotal, "total", p.fuzzer.activeCap) } func newFuzzer(input []byte) *fuzzer { @@ -165,12 +180,16 @@ func (f *fuzzer) randomFactors() vfs.PriceFactors { } } -func (f *fuzzer) connectedBalanceOp(balance vfs.ConnectedBalance) { +func (f *fuzzer) connectedBalanceOp(balance vfs.ConnectedBalance, id enode.ID) { switch f.randomInt(3) { case 0: - balance.RequestServed(uint64(f.randomTokenAmount(false))) + cost := uint64(f.randomTokenAmount(false)) + balance.RequestServed(cost) + doLog("Serve request cost", "id", id, "amount", cost) case 1: - balance.SetPriceFactors(f.randomFactors(), f.randomFactors()) + posFactor, negFactor := f.randomFactors(), f.randomFactors() + balance.SetPriceFactors(posFactor, negFactor) + doLog("Set price factor", "pos", posFactor, "neg", negFactor) case 2: balance.GetBalance() balance.GetRawBalance() @@ -178,12 +197,16 @@ func (f *fuzzer) connectedBalanceOp(balance vfs.ConnectedBalance) { } } -func (f *fuzzer) atomicBalanceOp(balance vfs.AtomicBalanceOperator) { +func (f *fuzzer) atomicBalanceOp(balance vfs.AtomicBalanceOperator, id enode.ID) { switch f.randomInt(3) { case 0: - balance.AddBalance(f.randomTokenAmount(true)) + amount := f.randomTokenAmount(true) + balance.AddBalance(amount) + doLog("Add balance", "id", id, "amount", amount) case 1: - balance.SetBalance(uint64(f.randomTokenAmount(false)), uint64(f.randomTokenAmount(false))) + pos, neg := uint64(f.randomTokenAmount(false)), uint64(f.randomTokenAmount(false)) + balance.SetBalance(pos, neg) + doLog("Set balance", "id", id, "pos", pos, "neg", neg) case 2: balance.GetBalance() balance.GetRawBalance() @@ -212,33 +235,53 @@ func FuzzClientPool(input []byte) int { case 0: i := int(f.randomByte()) f.peers[i].balance = pool.Register(f.peers[i]) + doLog("Register peer", "id", f.peers[i].node.ID()) case 1: i := int(f.randomByte()) f.peers[i].Disconnect() + doLog("Disconnect peer", "id", f.peers[i].node.ID()) case 2: f.maxCount = uint64(f.randomByte()) f.maxCap = uint64(f.randomByte()) f.maxCap *= f.maxCap + + count, cap := pool.Limits() pool.SetLimits(f.maxCount, f.maxCap) + doLog("Set limits", "maxcount", f.maxCount, "maxcap", f.maxCap, "origincount", count, "oricap", cap) case 3: + bias := f.randomDelay() pool.SetConnectedBias(f.randomDelay()) + doLog("Set connection bias", "bias", bias) case 4: - pool.SetDefaultFactors(f.randomFactors(), f.randomFactors()) + pos, neg := f.randomFactors(), f.randomFactors() + pool.SetDefaultFactors(pos, neg) + doLog("Set default factors", "pos", pos, "neg", neg) case 5: - pool.SetExpirationTCs(uint64(f.randomInt(50000)), uint64(f.randomInt(50000))) + pos, neg := uint64(f.randomInt(50000)), uint64(f.randomInt(50000)) + pool.SetExpirationTCs(pos, neg) + doLog("Set expiration constants", "pos", pos, "neg", neg) case 6: - if _, err := pool.SetCapacity(f.peers[f.randomByte()].node, uint64(f.randomByte()), f.randomDelay(), f.randomBool()); err == vfs.ErrCantFindMaximum { + var ( + index = f.randomByte() + reqCap = uint64(f.randomByte()) + bias = f.randomDelay() + requested = f.randomBool() + ) + if _, err := pool.SetCapacity(f.peers[index].node, reqCap, bias, requested); err == vfs.ErrCantFindMaximum { panic(nil) } + doLog("Set capacity", "id", f.peers[index].node.ID(), "reqcap", reqCap, "bias", bias, "requested", requested) case 7: - if balance := f.peers[f.randomByte()].balance; balance != nil { - f.connectedBalanceOp(balance) + index := f.randomByte() + if balance := f.peers[index].balance; balance != nil { + f.connectedBalanceOp(balance, f.peers[index].node.ID()) } case 8: - pool.BalanceOperation(f.peers[f.randomByte()].node.ID(), f.peers[f.randomByte()].freeID, func(balance vfs.AtomicBalanceOperator) { + index := f.randomByte() + pool.BalanceOperation(f.peers[index].node.ID(), f.peers[index].freeID, func(balance vfs.AtomicBalanceOperator) { count := f.randomInt(4) for i := 0; i < count; i++ { - f.atomicBalanceOp(balance) + f.atomicBalanceOp(balance, f.peers[index].node.ID()) } }) case 9: @@ -272,13 +315,16 @@ func FuzzClientPool(input []byte) int { for _, peer := range f.disconnectList { pool.Unregister(peer) + doLog("Unregister peer", "id", peer.node.ID()) } f.disconnectList = nil if d := f.randomDelay(); d > 0 { clock.Run(d) } - //fmt.Println(f.activeCount, f.maxCount, f.activeCap, f.maxCap) - if activeCount, activeCap := pool.Active(); activeCount != f.activeCount || activeCap != f.activeCap { + doLog("Clientpool stats in fuzzer", "count", f.activeCap, "maxcount", f.maxCount, "cap", f.activeCap, "maxcap", f.maxCap) + activeCount, activeCap := pool.Active() + doLog("Clientpool stats in pool", "count", activeCount, "cap", activeCap) + if activeCount != f.activeCount || activeCap != f.activeCap { panic(nil) } if f.activeCount > f.maxCount || f.activeCap > f.maxCap { diff --git a/tests/fuzzers/vflux/debug/main.go b/tests/fuzzers/vflux/debug/main.go index de0b5d4124..1d4a5ff19c 100644 --- a/tests/fuzzers/vflux/debug/main.go +++ b/tests/fuzzers/vflux/debug/main.go @@ -21,10 +21,13 @@ import ( "io/ioutil" "os" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests/fuzzers/vflux" ) func main() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + if len(os.Args) != 2 { fmt.Fprintf(os.Stderr, "Usage: debug \n") fmt.Fprintf(os.Stderr, "Example\n") From 558bff40083ccb96165be17e4e40cad4fa8193b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Apr 2021 21:40:06 +0300 Subject: [PATCH 311/709] eth/protocols/snap: lower the packet size to avoid overloading link --- eth/protocols/snap/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 3ce4c8735f..2ad677f949 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -52,7 +52,7 @@ var ( const ( // maxRequestSize is the maximum number of bytes to request from a remote peer. - maxRequestSize = 512 * 1024 + maxRequestSize = 128 * 1024 // maxStorageSetRequestCount is the maximum number of contracts to request the // storage of in a single query. If this number is too low, we're not filling @@ -74,7 +74,7 @@ const ( // a single query. If this number is too low, we're not filling responses fully // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. - maxTrieRequestCount = 512 + maxTrieRequestCount = 256 ) var ( From e4270cacf4aa26875affc619dbf82ad18d06226e Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 28 Apr 2021 21:38:38 +0200 Subject: [PATCH 312/709] cmd/devp2p: fix flaky SameRequestID test (#22754) --- cmd/devp2p/internal/ethtest/eth66_suite.go | 71 ++++++++++++++----- .../internal/ethtest/eth66_suiteHelpers.go | 38 ++++++---- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 4265b25f6a..41177189dd 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -76,9 +76,14 @@ func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) { }, } // write message - headers := s.getBlockHeaders66(t, conn, req, req.RequestId) + headers, err := s.getBlockHeaders66(conn, req, req.RequestId) + if err != nil { + t.Fatalf("could not get block headers: %v", err) + } // check for correct headers - headersMatch(t, s.chain, headers) + if !headersMatch(t, s.chain, headers) { + t.Fatal("received wrong header(s)") + } } // TestSimultaneousRequests_66 sends two simultaneous `GetBlockHeader` requests @@ -115,12 +120,25 @@ func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { // wait for headers for first request headerChan := make(chan BlockHeaders, 1) go func(headers chan BlockHeaders) { - headers <- s.getBlockHeaders66(t, conn1, req1, req1.RequestId) + recvHeaders, err := s.getBlockHeaders66(conn1, req1, req1.RequestId) + if err != nil { + t.Fatalf("could not get block headers: %v", err) + return + } + headers <- recvHeaders }(headerChan) // check headers of second request - headersMatch(t, s.chain, s.getBlockHeaders66(t, conn2, req2, req2.RequestId)) + headers1, err := s.getBlockHeaders66(conn2, req2, req2.RequestId) + if err != nil { + t.Fatalf("could not get block headers: %v", err) + } + if !headersMatch(t, s.chain, headers1) { + t.Fatal("wrong header(s) in response to req2") + } // check headers of first request - headersMatch(t, s.chain, <-headerChan) + if !headersMatch(t, s.chain, <-headerChan) { + t.Fatal("wrong header(s) in response to req1") + } } // TestBroadcast_66 tests whether a block announcement is correctly @@ -377,26 +395,31 @@ func (s *Suite) TestZeroRequestID_66(t *utesting.T) { Amount: 2, }, } - headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req, req.RequestId)) + headers, err := s.getBlockHeaders66(conn, req, req.RequestId) + if err != nil { + t.Fatalf("could not get block headers: %v", err) + } + if !headersMatch(t, s.chain, headers) { + t.Fatal("received wrong header(s)") + } } // TestSameRequestID_66 sends two requests with the same request ID // concurrently to a single node. func (s *Suite) TestSameRequestID_66(t *utesting.T) { conn := s.setupConnection66(t) - defer conn.Close() - // create two separate requests with same ID + // create two requests with the same request ID reqID := uint64(1234) - req1 := ð.GetBlockHeadersPacket66{ + request1 := ð.GetBlockHeadersPacket66{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ - Number: 0, + Number: 1, }, Amount: 2, }, } - req2 := ð.GetBlockHeadersPacket66{ + request2 := ð.GetBlockHeadersPacket66{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -405,12 +428,26 @@ func (s *Suite) TestSameRequestID_66(t *utesting.T) { Amount: 2, }, } - // send requests concurrently - go func() { - headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req2, reqID)) - }() - // check response from first request - headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req1, reqID)) + // write the first request + err := conn.write66(request1, GetBlockHeaders{}.Code()) + if err != nil { + t.Fatalf("could not write to connection: %v", err) + } + // perform second request + headers2, err := s.getBlockHeaders66(conn, request2, reqID) + if err != nil { + t.Fatalf("could not get block headers: %v", err) + return + } + // wait for response to first request + headers1, err := s.waitForBlockHeadersResponse66(conn, reqID) + if err != nil { + t.Fatalf("could not get BlockHeaders response: %v", err) + } + // check if headers match + if !headersMatch(t, s.chain, headers1) || !headersMatch(t, s.chain, headers2) { + t.Fatal("received wrong header(s)") + } } // TestLargeTxRequest_66 tests whether a node can fulfill a large GetPooledTransactions diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go index 3af8295c61..fec02b5246 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -18,6 +18,7 @@ package ethtest import ( "fmt" + "reflect" "time" "github.com/ethereum/go-ethereum/core/types" @@ -150,8 +151,7 @@ func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID ui func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { start := time.Now() for time.Since(start) < timeout { - timeout := time.Now().Add(10 * time.Second) - c.SetReadDeadline(timeout) + c.SetReadDeadline(time.Now().Add(10 * time.Second)) reqID, msg := c.read66() @@ -257,6 +257,9 @@ func (c *Conn) waitForBlock66(block *types.Block) error { return nil } time.Sleep(100 * time.Millisecond) + case *NewPooledTransactionHashes: + // ignore old announcements + continue default: return fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) } @@ -269,31 +272,38 @@ func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) { sendSuccessfulTxWithConn(t, s, tx, sendConn) } -func (s *Suite) getBlockHeaders66(t *utesting.T, conn *Conn, req eth.Packet, expectedID uint64) BlockHeaders { - if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // check block headers response +// waitForBlockHeadersResponse66 waits for a BlockHeaders message with the given expected request ID +func (s *Suite) waitForBlockHeadersResponse66(conn *Conn, expectedID uint64) (BlockHeaders, error) { reqID, msg := conn.readAndServe66(s.chain, timeout) - switch msg := msg.(type) { case BlockHeaders: if reqID != expectedID { - t.Fatalf("request ID mismatch: wanted %d, got %d", expectedID, reqID) + return nil, fmt.Errorf("request ID mismatch: wanted %d, got %d", expectedID, reqID) } - return msg + return msg, nil default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - return nil + return nil, fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } } -func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) { +func (s *Suite) getBlockHeaders66(conn *Conn, req eth.Packet, expectedID uint64) (BlockHeaders, error) { + if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil { + return nil, fmt.Errorf("could not write to connection: %v", err) + } + return s.waitForBlockHeadersResponse66(conn, expectedID) +} + +func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) bool { + mismatched := 0 for _, header := range headers { num := header.Number.Uint64() t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash())) - assert.Equal(t, chain.blocks[int(num)].Header(), header) + if !reflect.DeepEqual(chain.blocks[int(num)].Header(), header) { + mismatched += 1 + t.Logf("received wrong header: %v", pretty.Sdump(header)) + } } + return mismatched == 0 } func (s *Suite) sendNextBlock66(t *utesting.T) { From a81cf0d2b3497e5d78b2c06427953b90c1a0d70f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Apr 2021 21:47:48 +0200 Subject: [PATCH 313/709] trie: remove redundant returns + use stacktrie where applicable (#22760) * trie: add benchmark for proofless range * trie: remove unused returns + use stacktrie --- core/state/snapshot/generate.go | 2 +- eth/protocols/snap/sync.go | 6 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 14 +-- trie/notary.go | 14 +-- trie/proof.go | 69 ++++++--------- trie/proof_test.go | 88 ++++++++++++------- 6 files changed, 96 insertions(+), 97 deletions(-) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 13b34f4d69..78fca45e44 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -368,7 +368,7 @@ func (dl *diskLayer) proveRange(stats *generatorStats, root common.Hash, prefix } // Verify the snapshot segment with range prover, ensure that all flat states // in this range correspond to merkle trie. - _, _, _, cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) + _, cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) return &proofResult{ keys: keys, vals: vals, diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 2ad677f949..287ac8d727 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2176,7 +2176,7 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco if len(keys) > 0 { end = keys[len(keys)-1] } - _, _, _, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) + _, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) if err != nil { logger.Warn("Account range failed proof", "err", err) // Signal this request as failed, and ready for rescheduling @@ -2413,7 +2413,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(nodes) == 0 { // No proof has been attached, the response must cover the entire key // space and hash to the origin root. - dbs[i], _, _, _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) + dbs[i], _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage slots failed proof", "err", err) @@ -2428,7 +2428,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(keys) > 0 { end = keys[len(keys)-1] } - dbs[i], _, _, cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) + dbs[i], cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage range failed proof", "err", err) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index b82a380723..984bb9d0a8 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -170,17 +170,11 @@ func (f *fuzzer) fuzz() int { } ok = 1 //nodes, subtrie - nodes, subtrie, notary, hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, last, keys, vals, proof) + nodes, hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, last, keys, vals, proof) if err != nil { if nodes != nil { panic("err != nil && nodes != nil") } - if subtrie != nil { - panic("err != nil && subtrie != nil") - } - if notary != nil { - panic("err != nil && notary != nil") - } if hasMore { panic("err != nil && hasMore == true") } @@ -188,12 +182,6 @@ func (f *fuzzer) fuzz() int { if nodes == nil { panic("err == nil && nodes == nil") } - if subtrie == nil { - panic("err == nil && subtrie == nil") - } - if notary == nil { - panic("err == nil && subtrie == nil") - } } } return ok diff --git a/trie/notary.go b/trie/notary.go index 5a64727aa7..10c7628f55 100644 --- a/trie/notary.go +++ b/trie/notary.go @@ -21,17 +21,17 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -// KeyValueNotary tracks which keys have been accessed through a key-value reader +// keyValueNotary tracks which keys have been accessed through a key-value reader // with te scope of verifying if certain proof datasets are maliciously bloated. -type KeyValueNotary struct { +type keyValueNotary struct { ethdb.KeyValueReader reads map[string]struct{} } -// NewKeyValueNotary wraps a key-value database with an access notary to track +// newKeyValueNotary wraps a key-value database with an access notary to track // which items have bene accessed. -func NewKeyValueNotary(db ethdb.KeyValueReader) *KeyValueNotary { - return &KeyValueNotary{ +func newKeyValueNotary(db ethdb.KeyValueReader) *keyValueNotary { + return &keyValueNotary{ KeyValueReader: db, reads: make(map[string]struct{}), } @@ -39,14 +39,14 @@ func NewKeyValueNotary(db ethdb.KeyValueReader) *KeyValueNotary { // Get retrieves an item from the underlying database, but also tracks it as an // accessed slot for bloat checks. -func (k *KeyValueNotary) Get(key []byte) ([]byte, error) { +func (k *keyValueNotary) Get(key []byte) ([]byte, error) { k.reads[string(key)] = struct{}{} return k.KeyValueReader.Get(key) } // Accessed returns s snapshot of the original key-value store containing only the // data accessed through the notary. -func (k *KeyValueNotary) Accessed() ethdb.KeyValueStore { +func (k *keyValueNotary) Accessed() ethdb.KeyValueStore { db := memorydb.New() for keystr := range k.reads { key := []byte(keystr) diff --git a/trie/proof.go b/trie/proof.go index 61c35a8423..2feed24de4 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -464,115 +464,100 @@ func hasRightElement(node node, key []byte) bool { // // Except returning the error to indicate the proof is valid or not, the function will // also return a flag to indicate whether there exists more accounts/slots in the trie. -func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (ethdb.KeyValueStore, *Trie, *KeyValueNotary, bool, error) { +func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (ethdb.KeyValueStore, bool, error) { if len(keys) != len(values) { - return nil, nil, nil, false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) + return nil, false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) } // Ensure the received batch is monotonic increasing. for i := 0; i < len(keys)-1; i++ { if bytes.Compare(keys[i], keys[i+1]) >= 0 { - return nil, nil, nil, false, errors.New("range is not monotonically increasing") + return nil, false, errors.New("range is not monotonically increasing") } } // Create a key-value notary to track which items from the given proof the // range prover actually needed to verify the data - notary := NewKeyValueNotary(proof) + notary := newKeyValueNotary(proof) // Special case, there is no edge proof at all. The given range is expected // to be the whole leaf-set in the trie. if proof == nil { var ( diskdb = memorydb.New() - triedb = NewDatabase(diskdb) + tr = NewStackTrie(diskdb) ) - tr, err := New(common.Hash{}, triedb) - if err != nil { - return nil, nil, nil, false, err - } for index, key := range keys { tr.TryUpdate(key, values[index]) } - if tr.Hash() != rootHash { - return nil, nil, nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) - } - // Proof seems valid, serialize all the nodes into the database - if _, err := tr.Commit(nil); err != nil { - return nil, nil, nil, false, err + if have, want := tr.Hash(), rootHash; have != want { + return nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", want, have) } - if err := triedb.Commit(rootHash, false, nil); err != nil { - return nil, nil, nil, false, err + // Proof seems valid, serialize remaining nodes into the database + if _, err := tr.Commit(); err != nil { + return nil, false, err } - return diskdb, tr, notary, false, nil // No more elements + return diskdb, false, nil // No more elements } // Special case, there is a provided edge proof but zero key/value // pairs, ensure there are no more accounts / slots in the trie. if len(keys) == 0 { root, val, err := proofToPath(rootHash, nil, firstKey, notary, true) if err != nil { - return nil, nil, nil, false, err + return nil, false, err } if val != nil || hasRightElement(root, firstKey) { - return nil, nil, nil, false, errors.New("more entries available") + return nil, false, errors.New("more entries available") } // Since the entire proof is a single path, we can construct a trie and a // node database directly out of the inputs, no need to generate them diskdb := notary.Accessed() - tr := &Trie{ - db: NewDatabase(diskdb), - root: root, - } - return diskdb, tr, notary, hasRightElement(root, firstKey), nil + return diskdb, hasRightElement(root, firstKey), nil } // Special case, there is only one element and two edge keys are same. // In this case, we can't construct two edge paths. So handle it here. if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { root, val, err := proofToPath(rootHash, nil, firstKey, notary, false) if err != nil { - return nil, nil, nil, false, err + return nil, false, err } if !bytes.Equal(firstKey, keys[0]) { - return nil, nil, nil, false, errors.New("correct proof but invalid key") + return nil, false, errors.New("correct proof but invalid key") } if !bytes.Equal(val, values[0]) { - return nil, nil, nil, false, errors.New("correct proof but invalid data") + return nil, false, errors.New("correct proof but invalid data") } // Since the entire proof is a single path, we can construct a trie and a // node database directly out of the inputs, no need to generate them diskdb := notary.Accessed() - tr := &Trie{ - db: NewDatabase(diskdb), - root: root, - } - return diskdb, tr, notary, hasRightElement(root, firstKey), nil + return diskdb, hasRightElement(root, firstKey), nil } // Ok, in all other cases, we require two edge paths available. // First check the validity of edge keys. if bytes.Compare(firstKey, lastKey) >= 0 { - return nil, nil, nil, false, errors.New("invalid edge keys") + return nil, false, errors.New("invalid edge keys") } // todo(rjl493456442) different length edge keys should be supported if len(firstKey) != len(lastKey) { - return nil, nil, nil, false, errors.New("inconsistent edge keys") + return nil, false, errors.New("inconsistent edge keys") } // Convert the edge proofs to edge trie paths. Then we can // have the same tree architecture with the original one. // For the first edge proof, non-existent proof is allowed. root, _, err := proofToPath(rootHash, nil, firstKey, notary, true) if err != nil { - return nil, nil, nil, false, err + return nil, false, err } // Pass the root node here, the second path will be merged // with the first one. For the last edge proof, non-existent // proof is also allowed. root, _, err = proofToPath(rootHash, root, lastKey, notary, true) if err != nil { - return nil, nil, nil, false, err + return nil, false, err } // Remove all internal references. All the removed parts should // be re-filled(or re-constructed) by the given leaves range. empty, err := unsetInternal(root, firstKey, lastKey) if err != nil { - return nil, nil, nil, false, err + return nil, false, err } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. @@ -588,16 +573,16 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key tr.TryUpdate(key, values[index]) } if tr.Hash() != rootHash { - return nil, nil, nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) + return nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) } // Proof seems valid, serialize all the nodes into the database if _, err := tr.Commit(nil); err != nil { - return nil, nil, nil, false, err + return nil, false, err } if err := triedb.Commit(rootHash, false, nil); err != nil { - return nil, nil, nil, false, err + return nil, false, err } - return diskdb, tr, notary, hasRightElement(root, keys[len(keys)-1]), nil + return diskdb, hasRightElement(root, keys[len(keys)-1]), nil } // get returns the child of the given node. Return nil if the diff --git a/trie/proof_test.go b/trie/proof_test.go index 304affa9f2..7a906e2540 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -182,7 +182,7 @@ func TestRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -233,7 +233,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -254,7 +254,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatal("Failed to verify whole rang with non-existent edges") } @@ -289,7 +289,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) + _, _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -311,7 +311,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) + _, _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -335,7 +335,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -350,7 +350,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -365,7 +365,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -380,7 +380,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -399,7 +399,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := tinyTrie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) + _, _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -421,7 +421,7 @@ func TestAllElementsProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) + _, _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -434,7 +434,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) + _, _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -449,7 +449,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -482,7 +482,7 @@ func TestSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) + _, _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -518,7 +518,7 @@ func TestReverseSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) + _, _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -590,7 +590,7 @@ func TestBadRangeProof(t *testing.T) { index = mrand.Intn(end - start) vals[index] = nil } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err == nil { t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) } @@ -624,7 +624,7 @@ func TestGappedRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err == nil { t.Fatal("expect error, got nil") } @@ -651,7 +651,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -667,7 +667,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -735,7 +735,7 @@ func TestHasRightElement(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, _, hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) + _, hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -768,25 +768,19 @@ func TestEmptyRangeProof(t *testing.T) { if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - db, tr, not, _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) + db, _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) if c.err && err == nil { t.Fatalf("Expected error, got nil") } if !c.err && err != nil { t.Fatalf("Expected no error, got %v", err) } - // If no error was returned, ensure the returned trie and database contains + // If no error was returned, ensure the returned database contains // the entire proof, since there's no value if !c.err { - if err := tr.Prove(first, 0, memorydb.New()); err != nil { - t.Errorf("returned trie doesn't contain original proof: %v", err) - } if memdb := db.(*memorydb.Database); memdb.Len() != proof.Len() { t.Errorf("database entry count mismatch: have %d, want %d", memdb.Len(), proof.Len()) } - if not == nil { - t.Errorf("missing notary") - } } } } @@ -805,6 +799,8 @@ func TestBloatedProof(t *testing.T) { var vals [][]byte proof := memorydb.New() + // In the 'malicious' case, we add proofs for every single item + // (but only one key/value pair used as leaf) for i, entry := range entries { trie.Prove(entry.k, 0, proof) if i == 50 { @@ -812,12 +808,15 @@ func TestBloatedProof(t *testing.T) { vals = append(vals, entry.v) } } + // For reference, we use the same function, but _only_ prove the first + // and last element want := memorydb.New() trie.Prove(keys[0], 0, want) trie.Prove(keys[len(keys)-1], 0, want) - _, _, notary, _, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) - if used := notary.Accessed().(*memorydb.Database); used.Len() != want.Len() { + db, _, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + // The db should not contain anything of the bloated data + if used := db.(*memorydb.Database); used.Len() != want.Len() { t.Fatalf("notary proof size mismatch: have %d, want %d", used.Len(), want.Len()) } } @@ -922,13 +921,40 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) + _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) if err != nil { b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } } } +func BenchmarkVerifyRangeNoProof10(b *testing.B) { benchmarkVerifyRangeNoProof(b, 100) } +func BenchmarkVerifyRangeNoProof500(b *testing.B) { benchmarkVerifyRangeNoProof(b, 500) } +func BenchmarkVerifyRangeNoProof1000(b *testing.B) { benchmarkVerifyRangeNoProof(b, 1000) } + +func benchmarkVerifyRangeNoProof(b *testing.B, size int) { + trie, vals := randomTrie(size) + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + sort.Sort(entries) + + var keys [][]byte + var values [][]byte + for _, entry := range entries { + keys = append(keys, entry.k) + values = append(values, entry.v) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, nil) + if err != nil { + b.Fatalf("Expected no error, got %v", err) + } + } +} + func randomTrie(n int) (*Trie, map[string]*kv) { trie := new(Trie) vals := make(map[string]*kv) From fae165a5def1a335594cf6761164e31fa4e8d27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Apr 2021 23:09:15 +0300 Subject: [PATCH 314/709] core, eth, ethdb, trie: simplify range proofs --- core/rawdb/table.go | 5 - core/state/snapshot/generate.go | 2 +- eth/protocols/snap/sync.go | 121 +++++++++--------- ethdb/batch.go | 28 +++- ethdb/leveldb/leveldb.go | 9 +- ethdb/memorydb/memorydb.go | 9 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 9 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 1 - trie/notary.go | 57 --------- trie/proof.go | 82 +++++------- trie/proof_test.go | 62 ++++----- trie/trie_test.go | 1 - 12 files changed, 149 insertions(+), 237 deletions(-) delete mode 100644 trie/notary.go diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 4daa6b5349..323ef6293c 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -176,11 +176,6 @@ func (b *tableBatch) Delete(key []byte) error { return b.batch.Delete(append([]byte(b.prefix), key...)) } -// KeyCount retrieves the number of keys queued up for writing. -func (b *tableBatch) KeyCount() int { - return b.batch.KeyCount() -} - // ValueSize retrieves the amount of data queued up for writing. func (b *tableBatch) ValueSize() int { return b.batch.ValueSize() diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 78fca45e44..8992d3f91b 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -368,7 +368,7 @@ func (dl *diskLayer) proveRange(stats *generatorStats, root common.Hash, prefix } // Verify the snapshot segment with range prover, ensure that all flat states // in this range correspond to merkle trie. - _, cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) + cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) return &proofResult{ keys: keys, vals: vals, diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 287ac8d727..d9c0cb9b1b 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -202,9 +202,8 @@ type storageResponse struct { accounts []common.Hash // Account hashes requested, may be only partially filled roots []common.Hash // Storage roots requested, may be only partially filled - hashes [][]common.Hash // Storage slot hashes in the returned range - slots [][][]byte // Storage slot values in the returned range - nodes []ethdb.KeyValueStore // Database containing the reconstructed trie nodes + hashes [][]common.Hash // Storage slot hashes in the returned range + slots [][][]byte // Storage slot values in the returned range cont bool // Whether the last storage range has a continuation } @@ -680,12 +679,22 @@ func (s *Syncer) loadSyncStatus() { } s.tasks = progress.Tasks for _, task := range s.tasks { - task.genBatch = s.db.NewBatch() + task.genBatch = ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.accountBytes += common.StorageSize(len(key) + len(value)) + }, + } task.genTrie = trie.NewStackTrie(task.genBatch) for _, subtasks := range task.SubTasks { for _, subtask := range subtasks { - subtask.genBatch = s.db.NewBatch() + subtask.genBatch = ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.storageBytes += common.StorageSize(len(key) + len(value)) + }, + } subtask.genTrie = trie.NewStackTrie(task.genBatch) } } @@ -729,7 +738,12 @@ func (s *Syncer) loadSyncStatus() { // Make sure we don't overflow if the step is not a proper divisor last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") } - batch := s.db.NewBatch() + batch := ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.accountBytes += common.StorageSize(len(key) + len(value)) + }, + } s.tasks = append(s.tasks, &accountTask{ Next: next, Last: last, @@ -746,19 +760,14 @@ func (s *Syncer) loadSyncStatus() { func (s *Syncer) saveSyncStatus() { // Serialize any partial progress to disk before spinning down for _, task := range s.tasks { - keys, bytes := task.genBatch.KeyCount(), task.genBatch.ValueSize() if err := task.genBatch.Write(); err != nil { log.Error("Failed to persist account slots", "err", err) } - s.accountBytes += common.StorageSize(keys*common.HashLength + bytes) - for _, subtasks := range task.SubTasks { for _, subtask := range subtasks { - keys, bytes := subtask.genBatch.KeyCount(), subtask.genBatch.ValueSize() if err := subtask.genBatch.Write(); err != nil { log.Error("Failed to persist storage slots", "err", err) } - s.accountBytes += common.StorageSize(keys*common.HashLength + bytes) } } } @@ -1763,12 +1772,15 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { if res.subTask != nil { res.subTask.req = nil } - batch := s.db.NewBatch() - + batch := ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.storageBytes += common.StorageSize(len(key) + len(value)) + }, + } var ( - slots int - nodes int - bytes common.StorageSize + slots int + oldStorageBytes = s.storageBytes ) // Iterate over all the accounts and reconstruct their storage tries from the // delivered slots @@ -1829,7 +1841,12 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { r := newHashRange(lastKey, chunks) // Our first task is the one that was just filled by this response. - batch := s.db.NewBatch() + batch := ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.storageBytes += common.StorageSize(len(key) + len(value)) + }, + } tasks = append(tasks, &storageTask{ Next: common.Hash{}, Last: r.End(), @@ -1838,7 +1855,12 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { genTrie: trie.NewStackTrie(batch), }) for r.Next() { - batch := s.db.NewBatch() + batch := ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.storageBytes += common.StorageSize(len(key) + len(value)) + }, + } tasks = append(tasks, &storageTask{ Next: r.Start(), Last: r.End(), @@ -1883,27 +1905,23 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { } } } - // Iterate over all the reconstructed trie nodes and push them to disk - // if the contract is fully delivered. If it's chunked, the trie nodes - // will be reconstructed later. + // Iterate over all the complete contracts, reconstruct the trie nodes and + // push them to disk. If the contract is chunked, the trie nodes will be + // reconstructed later. slots += len(res.hashes[i]) if i < len(res.hashes)-1 || res.subTask == nil { - it := res.nodes[i].NewIterator(nil, nil) - for it.Next() { - batch.Put(it.Key(), it.Value()) - - bytes += common.StorageSize(common.HashLength + len(it.Value())) - nodes++ + tr := trie.NewStackTrie(batch) + for j := 0; j < len(res.hashes[i]); j++ { + tr.Update(res.hashes[i][j][:], res.slots[i][j]) } - it.Release() + tr.Commit() } // Persist the received storage segements. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. for j := 0; j < len(res.hashes[i]); j++ { rawdb.WriteStorageSnapshot(batch, account, res.hashes[i][j], res.slots[i][j]) - bytes += common.StorageSize(1 + 2*common.HashLength + len(res.slots[i][j])) // If we're storing large contracts, generate the trie nodes // on the fly to not trash the gluing points @@ -1926,15 +1944,11 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { } } } - if data := res.subTask.genBatch.ValueSize(); data > ethdb.IdealBatchSize || res.subTask.done { - keys := res.subTask.genBatch.KeyCount() + if res.subTask.genBatch.ValueSize() > ethdb.IdealBatchSize || res.subTask.done { if err := res.subTask.genBatch.Write(); err != nil { log.Error("Failed to persist stack slots", "err", err) } res.subTask.genBatch.Reset() - - bytes += common.StorageSize(keys*common.HashLength + data) - nodes += keys } } // Flush anything written just now and update the stats @@ -1942,9 +1956,8 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { log.Crit("Failed to persist storage slots", "err", err) } s.storageSynced += uint64(slots) - s.storageBytes += bytes - log.Debug("Persisted set of storage slots", "accounts", len(res.hashes), "slots", slots, "nodes", nodes, "bytes", bytes) + log.Debug("Persisted set of storage slots", "accounts", len(res.hashes), "slots", slots, "bytes", s.storageBytes-oldStorageBytes) // If this delivery completed the last pending task, forward the account task // to the next chunk @@ -2042,18 +2055,20 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { // Persist the received account segements. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. - var ( - nodes int - bytes common.StorageSize - ) - batch := s.db.NewBatch() + oldAccountBytes := s.accountBytes + + batch := ethdb.HookedBatch{ + Batch: s.db.NewBatch(), + OnPut: func(key []byte, value []byte) { + s.accountBytes += common.StorageSize(len(key) + len(value)) + }, + } for i, hash := range res.hashes { if task.needCode[i] || task.needState[i] { break } slim := snapshot.SlimAccountRLP(res.accounts[i].Nonce, res.accounts[i].Balance, res.accounts[i].Root, res.accounts[i].CodeHash) rawdb.WriteAccountSnapshot(batch, hash, slim) - bytes += common.StorageSize(1 + common.HashLength + len(slim)) // If the task is complete, drop it into the stack trie to generate // account trie nodes for it @@ -2069,7 +2084,6 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { if err := batch.Write(); err != nil { log.Crit("Failed to persist accounts", "err", err) } - s.accountBytes += bytes s.accountSynced += uint64(len(res.accounts)) // Task filling persisted, push it the chunk marker forward to the first @@ -2091,17 +2105,13 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { log.Error("Failed to commit stack account", "err", err) } } - if data := task.genBatch.ValueSize(); data > ethdb.IdealBatchSize || task.done { - keys := task.genBatch.KeyCount() + if task.genBatch.ValueSize() > ethdb.IdealBatchSize || task.done { if err := task.genBatch.Write(); err != nil { log.Error("Failed to persist stack account", "err", err) } task.genBatch.Reset() - - nodes += keys - bytes += common.StorageSize(keys*common.HashLength + data) } - log.Debug("Persisted range of accounts", "accounts", len(res.accounts), "nodes", nodes, "bytes", bytes) + log.Debug("Persisted range of accounts", "accounts", len(res.accounts), "bytes", s.accountBytes-oldAccountBytes) } // OnAccounts is a callback method to invoke when a range of accounts are @@ -2176,7 +2186,7 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco if len(keys) > 0 { end = keys[len(keys)-1] } - _, cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) + cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) if err != nil { logger.Warn("Account range failed proof", "err", err) // Signal this request as failed, and ready for rescheduling @@ -2393,10 +2403,8 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo s.lock.Unlock() // Reconstruct the partial tries from the response and verify them - var ( - dbs = make([]ethdb.KeyValueStore, len(hashes)) - cont bool - ) + var cont bool + for i := 0; i < len(hashes); i++ { // Convert the keys and proofs into an internal format keys := make([][]byte, len(hashes[i])) @@ -2413,7 +2421,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(nodes) == 0 { // No proof has been attached, the response must cover the entire key // space and hash to the origin root. - dbs[i], _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) + _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage slots failed proof", "err", err) @@ -2428,7 +2436,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(keys) > 0 { end = keys[len(keys)-1] } - dbs[i], cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) + cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage range failed proof", "err", err) @@ -2444,7 +2452,6 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo roots: req.roots, hashes: hashes, slots: slots, - nodes: dbs, cont: cont, } select { diff --git a/ethdb/batch.go b/ethdb/batch.go index 5f8207fc46..1353693318 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -25,9 +25,6 @@ const IdealBatchSize = 100 * 1024 type Batch interface { KeyValueWriter - // KeyCount retrieves the number of keys queued up for writing. - KeyCount() int - // ValueSize retrieves the amount of data queued up for writing. ValueSize() int @@ -47,3 +44,28 @@ type Batcher interface { // until a final write is called. NewBatch() Batch } + +// HookedBatch wraps an arbitrary batch where each operation may be hooked into +// to monitor from black box code. +type HookedBatch struct { + Batch + + OnPut func(key []byte, value []byte) // Callback if a key is inserted + OnDelete func(key []byte) // Callback if a key is deleted +} + +// Put inserts the given value into the key-value data store. +func (b HookedBatch) Put(key []byte, value []byte) error { + if b.OnPut != nil { + b.OnPut(key, value) + } + return b.Batch.Put(key, value) +} + +// Delete removes the key from the key-value data store. +func (b HookedBatch) Delete(key []byte) error { + if b.OnDelete != nil { + b.OnDelete(key) + } + return b.Batch.Delete(key) +} diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index da00226e95..5d19cc3577 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -448,7 +448,6 @@ func (db *Database) meter(refresh time.Duration) { type batch struct { db *leveldb.DB b *leveldb.Batch - keys int size int } @@ -462,16 +461,10 @@ func (b *batch) Put(key, value []byte) error { // Delete inserts the a key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { b.b.Delete(key) - b.keys++ b.size += len(key) return nil } -// KeyCount retrieves the number of keys queued up for writing. -func (b *batch) KeyCount() int { - return b.keys -} - // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -485,7 +478,7 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { b.b.Reset() - b.keys, b.size = 0, 0 + b.size = 0 } // Replay replays the batch contents. diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index ded9f5e66c..fedc9e326c 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -198,7 +198,6 @@ type keyvalue struct { type batch struct { db *Database writes []keyvalue - keys int size int } @@ -212,16 +211,10 @@ func (b *batch) Put(key, value []byte) error { // Delete inserts the a key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true}) - b.keys++ b.size += len(key) return nil } -// KeyCount retrieves the number of keys queued up for writing. -func (b *batch) KeyCount() int { - return b.keys -} - // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -245,7 +238,7 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { b.writes = b.writes[:0] - b.keys, b.size = 0, 0 + b.size = 0 } // Replay replays the batch contents. diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 984bb9d0a8..09ee6bb9c7 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -170,18 +170,11 @@ func (f *fuzzer) fuzz() int { } ok = 1 //nodes, subtrie - nodes, hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, last, keys, vals, proof) + hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, last, keys, vals, proof) if err != nil { - if nodes != nil { - panic("err != nil && nodes != nil") - } if hasMore { panic("err != nil && hasMore == true") } - } else { - if nodes == nil { - panic("err == nil && nodes == nil") - } } } return ok diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 0013c919c9..5cea7769c2 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -90,7 +90,6 @@ func (b *spongeBatch) Put(key, value []byte) error { return nil } func (b *spongeBatch) Delete(key []byte) error { panic("implement me") } -func (b *spongeBatch) KeyCount() int { panic("not implemented") } func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} diff --git a/trie/notary.go b/trie/notary.go deleted file mode 100644 index 10c7628f55..0000000000 --- a/trie/notary.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" -) - -// keyValueNotary tracks which keys have been accessed through a key-value reader -// with te scope of verifying if certain proof datasets are maliciously bloated. -type keyValueNotary struct { - ethdb.KeyValueReader - reads map[string]struct{} -} - -// newKeyValueNotary wraps a key-value database with an access notary to track -// which items have bene accessed. -func newKeyValueNotary(db ethdb.KeyValueReader) *keyValueNotary { - return &keyValueNotary{ - KeyValueReader: db, - reads: make(map[string]struct{}), - } -} - -// Get retrieves an item from the underlying database, but also tracks it as an -// accessed slot for bloat checks. -func (k *keyValueNotary) Get(key []byte) ([]byte, error) { - k.reads[string(key)] = struct{}{} - return k.KeyValueReader.Get(key) -} - -// Accessed returns s snapshot of the original key-value store containing only the -// data accessed through the notary. -func (k *keyValueNotary) Accessed() ethdb.KeyValueStore { - db := memorydb.New() - for keystr := range k.reads { - key := []byte(keystr) - val, _ := k.KeyValueReader.Get(key) - db.Put(key, val) - } - return db -} diff --git a/trie/proof.go b/trie/proof.go index 2feed24de4..08a9e40422 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -464,108 +464,91 @@ func hasRightElement(node node, key []byte) bool { // // Except returning the error to indicate the proof is valid or not, the function will // also return a flag to indicate whether there exists more accounts/slots in the trie. -func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (ethdb.KeyValueStore, bool, error) { +// +// Note: This method does not verify that the proof is of minimal form. If the input +// proofs are 'bloated' with neighbour leaves or random data, aside from the 'useful' +// data, then the proof will still be accepted. +func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (bool, error) { if len(keys) != len(values) { - return nil, false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) + return false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) } // Ensure the received batch is monotonic increasing. for i := 0; i < len(keys)-1; i++ { if bytes.Compare(keys[i], keys[i+1]) >= 0 { - return nil, false, errors.New("range is not monotonically increasing") + return false, errors.New("range is not monotonically increasing") } } - // Create a key-value notary to track which items from the given proof the - // range prover actually needed to verify the data - notary := newKeyValueNotary(proof) - // Special case, there is no edge proof at all. The given range is expected // to be the whole leaf-set in the trie. if proof == nil { - var ( - diskdb = memorydb.New() - tr = NewStackTrie(diskdb) - ) + tr := NewStackTrie(nil) for index, key := range keys { tr.TryUpdate(key, values[index]) } if have, want := tr.Hash(), rootHash; have != want { - return nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", want, have) - } - // Proof seems valid, serialize remaining nodes into the database - if _, err := tr.Commit(); err != nil { - return nil, false, err + return false, fmt.Errorf("invalid proof, want hash %x, got %x", want, have) } - return diskdb, false, nil // No more elements + return false, nil // No more elements } // Special case, there is a provided edge proof but zero key/value // pairs, ensure there are no more accounts / slots in the trie. if len(keys) == 0 { - root, val, err := proofToPath(rootHash, nil, firstKey, notary, true) + root, val, err := proofToPath(rootHash, nil, firstKey, proof, true) if err != nil { - return nil, false, err + return false, err } if val != nil || hasRightElement(root, firstKey) { - return nil, false, errors.New("more entries available") + return false, errors.New("more entries available") } - // Since the entire proof is a single path, we can construct a trie and a - // node database directly out of the inputs, no need to generate them - diskdb := notary.Accessed() - return diskdb, hasRightElement(root, firstKey), nil + return hasRightElement(root, firstKey), nil } // Special case, there is only one element and two edge keys are same. // In this case, we can't construct two edge paths. So handle it here. if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { - root, val, err := proofToPath(rootHash, nil, firstKey, notary, false) + root, val, err := proofToPath(rootHash, nil, firstKey, proof, false) if err != nil { - return nil, false, err + return false, err } if !bytes.Equal(firstKey, keys[0]) { - return nil, false, errors.New("correct proof but invalid key") + return false, errors.New("correct proof but invalid key") } if !bytes.Equal(val, values[0]) { - return nil, false, errors.New("correct proof but invalid data") + return false, errors.New("correct proof but invalid data") } - // Since the entire proof is a single path, we can construct a trie and a - // node database directly out of the inputs, no need to generate them - diskdb := notary.Accessed() - return diskdb, hasRightElement(root, firstKey), nil + return hasRightElement(root, firstKey), nil } // Ok, in all other cases, we require two edge paths available. // First check the validity of edge keys. if bytes.Compare(firstKey, lastKey) >= 0 { - return nil, false, errors.New("invalid edge keys") + return false, errors.New("invalid edge keys") } // todo(rjl493456442) different length edge keys should be supported if len(firstKey) != len(lastKey) { - return nil, false, errors.New("inconsistent edge keys") + return false, errors.New("inconsistent edge keys") } // Convert the edge proofs to edge trie paths. Then we can // have the same tree architecture with the original one. // For the first edge proof, non-existent proof is allowed. - root, _, err := proofToPath(rootHash, nil, firstKey, notary, true) + root, _, err := proofToPath(rootHash, nil, firstKey, proof, true) if err != nil { - return nil, false, err + return false, err } // Pass the root node here, the second path will be merged // with the first one. For the last edge proof, non-existent // proof is also allowed. - root, _, err = proofToPath(rootHash, root, lastKey, notary, true) + root, _, err = proofToPath(rootHash, root, lastKey, proof, true) if err != nil { - return nil, false, err + return false, err } // Remove all internal references. All the removed parts should // be re-filled(or re-constructed) by the given leaves range. empty, err := unsetInternal(root, firstKey, lastKey) if err != nil { - return nil, false, err + return false, err } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - var ( - diskdb = memorydb.New() - triedb = NewDatabase(diskdb) - ) - tr := &Trie{root: root, db: triedb} + tr := &Trie{root: root, db: NewDatabase(memorydb.New())} if empty { tr.root = nil } @@ -573,16 +556,9 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key tr.TryUpdate(key, values[index]) } if tr.Hash() != rootHash { - return nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) - } - // Proof seems valid, serialize all the nodes into the database - if _, err := tr.Commit(nil); err != nil { - return nil, false, err - } - if err := triedb.Commit(rootHash, false, nil); err != nil { - return nil, false, err + return false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) } - return diskdb, hasRightElement(root, keys[len(keys)-1]), nil + return hasRightElement(root, keys[len(keys)-1]), nil } // get returns the child of the given node. Return nil if the diff --git a/trie/proof_test.go b/trie/proof_test.go index 7a906e2540..a35b7144c0 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -182,7 +182,7 @@ func TestRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -233,7 +233,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -254,7 +254,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatal("Failed to verify whole rang with non-existent edges") } @@ -289,7 +289,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) + _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -311,7 +311,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) + _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -335,7 +335,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - _, _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -350,7 +350,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -365,7 +365,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -380,7 +380,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -399,7 +399,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := tinyTrie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) + _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -421,7 +421,7 @@ func TestAllElementsProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) + _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -434,7 +434,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) + _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -449,7 +449,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -482,7 +482,7 @@ func TestSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) + _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -518,7 +518,7 @@ func TestReverseSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) + _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -590,7 +590,7 @@ func TestBadRangeProof(t *testing.T) { index = mrand.Intn(end - start) vals[index] = nil } - _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err == nil { t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) } @@ -624,7 +624,7 @@ func TestGappedRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err == nil { t.Fatal("expect error, got nil") } @@ -651,7 +651,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -667,7 +667,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -735,7 +735,7 @@ func TestHasRightElement(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) + hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -768,25 +768,19 @@ func TestEmptyRangeProof(t *testing.T) { if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - db, _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) + _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) if c.err && err == nil { t.Fatalf("Expected error, got nil") } if !c.err && err != nil { t.Fatalf("Expected no error, got %v", err) } - // If no error was returned, ensure the returned database contains - // the entire proof, since there's no value - if !c.err { - if memdb := db.(*memorydb.Database); memdb.Len() != proof.Len() { - t.Errorf("database entry count mismatch: have %d, want %d", memdb.Len(), proof.Len()) - } - } } } // TestBloatedProof tests a malicious proof, where the proof is more or less the -// whole trie. +// whole trie. Previously we didn't accept such packets, but the new APIs do, so +// lets leave this test as a bit weird, but present. func TestBloatedProof(t *testing.T) { // Use a small trie trie, kvs := nonRandomTrie(100) @@ -814,10 +808,8 @@ func TestBloatedProof(t *testing.T) { trie.Prove(keys[0], 0, want) trie.Prove(keys[len(keys)-1], 0, want) - db, _, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) - // The db should not contain anything of the bloated data - if used := db.(*memorydb.Database); used.Len() != want.Len() { - t.Fatalf("notary proof size mismatch: have %d, want %d", used.Len(), want.Len()) + if _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof); err != nil { + t.Fatalf("expected bloated proof to succeed, got %v", err) } } @@ -921,7 +913,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) if err != nil { b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -948,7 +940,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, nil) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, nil) if err != nil { b.Fatalf("Expected no error, got %v", err) } diff --git a/trie/trie_test.go b/trie/trie_test.go index 44fddf87e4..492b423c2f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -706,7 +706,6 @@ func (b *spongeBatch) Put(key, value []byte) error { return nil } func (b *spongeBatch) Delete(key []byte) error { panic("implement me") } -func (b *spongeBatch) KeyCount() int { return 100 } func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} From 06f44c0fd425a0a3d82c7f3470da0314d6dc369e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 29 Apr 2021 12:02:30 +0300 Subject: [PATCH 315/709] eth: restore eth_hashrate API endpoint --- eth/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eth/api.go b/eth/api.go index e02c0ca4d2..7387459c94 100644 --- a/eth/api.go +++ b/eth/api.go @@ -61,6 +61,11 @@ func (api *PublicEthereumAPI) Coinbase() (common.Address, error) { return api.Etherbase() } +// Hashrate returns the POW hashrate +func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { + return hexutil.Uint64(api.e.Miner().Hashrate()) +} + // PublicMinerAPI provides an API to control the miner. // It offers only methods that operate on data that pose no security risk when it is publicly accessible. type PublicMinerAPI struct { From c7d07294a6f855d1c0fef9dcb55e0f87fb0e4962 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Thu, 29 Apr 2021 16:42:21 +0200 Subject: [PATCH 316/709] catalyst: check if block exists in assemble-block call with unknown parent-hash (#22770) --- eth/catalyst/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index d6ea691d02..d7e2af1c1a 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -109,6 +109,11 @@ func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableD bc := api.eth.BlockChain() parent := bc.GetBlockByHash(params.ParentHash) + if parent == nil { + log.Warn("Cannot assemble block with parent hash to unknown block", "parentHash", params.ParentHash) + return nil, fmt.Errorf("cannot assemble block with unknown parent %s", params.ParentHash) + } + pool := api.eth.TxPool() if parent.Time() >= params.Timestamp { From 793c8f889f7c565c4775385e5197c199bfde01a5 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 29 Apr 2021 18:36:22 +0200 Subject: [PATCH 317/709] add myself as code owner for catalyst (#22778) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 58c1a4a62e..2015604e64 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,6 +9,7 @@ cmd/puppeth @karalabe consensus @karalabe core/ @karalabe @holiman @rjl493456442 eth/ @karalabe @holiman @rjl493456442 +eth/catalyst/ @gballet graphql/ @gballet les/ @zsfelfoldi @rjl493456442 light/ @zsfelfoldi @rjl493456442 From 56f533d00ceea30b9111e1d6572b29a32f490ac2 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Thu, 29 Apr 2021 13:23:07 -0400 Subject: [PATCH 318/709] docs: fix docstring on read head block (#22776) --- core/rawdb/accessors_chain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 92450313b4..76132bf37e 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -839,7 +839,7 @@ func ReadHeadHeader(db ethdb.Reader) *types.Header { return ReadHeader(db, headHeaderHash, *headHeaderNumber) } -// ReadHeadHeader returns the current canonical head block. +// ReadHeadBlock returns the current canonical head block. func ReadHeadBlock(db ethdb.Reader) *types.Block { headBlockHash := ReadHeadBlockHash(db) if headBlockHash == (common.Hash{}) { From 63bad18c33e215c6ef6f33aae7e686211cf945bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 29 Apr 2021 19:30:16 +0200 Subject: [PATCH 319/709] evm: remove unused errors left after EIP-2315 removal (#22767) --- core/vm/errors.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/vm/errors.go b/core/vm/errors.go index f6b156a02e..c813aa36af 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -23,9 +23,6 @@ import ( // List evm execution errors var ( - // ErrInvalidSubroutineEntry means that a BEGINSUB was reached via iteration, - // as opposed to from a JUMPSUB instruction - ErrInvalidSubroutineEntry = errors.New("invalid subroutine entry") ErrOutOfGas = errors.New("out of gas") ErrCodeStoreOutOfGas = errors.New("contract creation code storage out of gas") ErrDepth = errors.New("max call depth exceeded") @@ -37,8 +34,6 @@ var ( ErrWriteProtection = errors.New("write protection") ErrReturnDataOutOfBounds = errors.New("return data out of bounds") ErrGasUintOverflow = errors.New("gas uint64 overflow") - ErrInvalidRetsub = errors.New("invalid retsub") - ErrReturnStackExceeded = errors.New("return stack limit reached") ) // ErrStackUnderflow wraps an evm error when the items on the stack less From b50b17ac695ffcaa9a084f87a8ea6f53e37d2a75 Mon Sep 17 00:00:00 2001 From: ligi Date: Thu, 29 Apr 2021 19:30:37 +0200 Subject: [PATCH 320/709] github: add note about screenshots in issue template (#22764) --- .github/ISSUE_TEMPLATE/bug.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index c5a3654bde..2aa2c48a60 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -26,3 +26,5 @@ Commit hash : (if `develop`) ```` [backtrace] ```` + +When submitting logs: please submit them as text and not screenshots. \ No newline at end of file From bb43cd7a792ce3e89961d29410b692073b935ac8 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Thu, 29 Apr 2021 16:14:57 -0400 Subject: [PATCH 321/709] core/types: add license header (#22781) --- core/types/transaction_marshalling.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index 184a17d5b5..e561485556 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -1,3 +1,19 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package types import ( From 8130dd5cefb11fe6bf70264087368c318692b9ff Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Fri, 30 Apr 2021 06:46:34 -0400 Subject: [PATCH 322/709] core/vm: fix typo in comment (#22785) --- core/vm/operations_acl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index 45b51d80cd..c56941899e 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -30,7 +30,7 @@ const ( WarmStorageReadCostEIP2929 = uint64(100) // WARM_STORAGE_READ_COST ) -// gasSStoreEIP2929 implements gas cost for SSTORE according to EIP-2929" +// gasSStoreEIP2929 implements gas cost for SSTORE according to EIP-2929 // // When calling SSTORE, check if the (address, storage_key) pair is in accessed_storage_keys. // If it is not, charge an additional COLD_SLOAD_COST gas, and add the pair to accessed_storage_keys. From 1e57ab5de6612f9b3bea4d3eb6f08641be36944c Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Fri, 30 Apr 2021 06:47:05 -0400 Subject: [PATCH 323/709] core: remove unused else branch in reorg (#22783) --- core/blockchain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 49aa1c3e86..b1da30a1b2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2162,7 +2162,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { l := *log if removed { l.Removed = true - } else { } logs = append(logs, &l) } From dde6cb0b9248f5bc7ac3619304ee3b1fce35f5d4 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Fri, 30 Apr 2021 06:49:13 -0400 Subject: [PATCH 324/709] core/vm: replace repeated string with variable in tests (#22774) --- core/vm/instructions_test.go | 69 ++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 14f9e181f9..d17ccfab89 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -40,6 +40,7 @@ type twoOperandParams struct { y string } +var alphabetSoup = "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" var commonParams []*twoOperandParams var twoOpMethods map[string]executionFunc @@ -347,8 +348,8 @@ func BenchmarkOpSub256(b *testing.B) { } func BenchmarkOpMul(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opMul, x, y) } @@ -379,64 +380,64 @@ func BenchmarkOpSdiv(b *testing.B) { } func BenchmarkOpMod(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opMod, x, y) } func BenchmarkOpSmod(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opSmod, x, y) } func BenchmarkOpExp(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opExp, x, y) } func BenchmarkOpSignExtend(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opSignExtend, x, y) } func BenchmarkOpLt(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opLt, x, y) } func BenchmarkOpGt(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opGt, x, y) } func BenchmarkOpSlt(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opSlt, x, y) } func BenchmarkOpSgt(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opSgt, x, y) } func BenchmarkOpEq(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opEq, x, y) } @@ -446,45 +447,45 @@ func BenchmarkOpEq2(b *testing.B) { opBenchmark(b, opEq, x, y) } func BenchmarkOpAnd(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opAnd, x, y) } func BenchmarkOpOr(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opOr, x, y) } func BenchmarkOpXor(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opXor, x, y) } func BenchmarkOpByte(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup opBenchmark(b, opByte, x, y) } func BenchmarkOpAddmod(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - z := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup + z := alphabetSoup opBenchmark(b, opAddmod, x, y, z) } func BenchmarkOpMulmod(b *testing.B) { - x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" - z := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + x := alphabetSoup + y := alphabetSoup + z := alphabetSoup opBenchmark(b, opMulmod, x, y, z) } From b778e37daa915f1b0d53e8ce3beb3884ee58120b Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Fri, 30 Apr 2021 06:50:02 -0400 Subject: [PATCH 325/709] core: fix typo in comment (#22773) --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index b1da30a1b2..e5e10fd643 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -482,7 +482,7 @@ func (bc *BlockChain) SetHead(head uint64) error { // SetHeadBeyondRoot rewinds the local chain to a new head with the extra condition // that the rewind must pass the specified state root. This method is meant to be -// used when rewiding with snapshots enabled to ensure that we go back further than +// used when rewinding with snapshots enabled to ensure that we go back further than // persistent disk layer. Depending on whether the node was fast synced or full, and // in which state, the method will try to delete minimal data from disk whilst // retaining chain consistency. From ff75b21f255d112b9a828150da25e9528451a574 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 30 Apr 2021 12:52:25 +0200 Subject: [PATCH 326/709] README.md: update commands table, add note about web3.js version (#22748) --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4a083d117a..77a403f7f3 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ archives are published at https://geth.ethereum.org/downloads/. For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). -Building `geth` requires both a Go (version 1.13 or later) and a C compiler. You can install +Building `geth` requires both a Go (version 1.14 or later) and a C compiler. You can install them using your favourite package manager. Once the dependencies are installed, run ```shell @@ -37,10 +37,11 @@ directory. | Command | Description | | :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | +| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | +| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://eth.wiki/json-rpc/API) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. | | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | | `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | @@ -67,7 +68,8 @@ This command will: causing it to download more data in exchange for avoiding processing the entire history of the Ethereum network, which is very CPU intensive. * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), - (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://web3js.readthedocs.io/en/) + (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://web3js.readthedocs.io/en/) + (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). This tool is optional and if you leave it out you can always attach to an already running `geth` instance with `geth attach`. @@ -228,7 +230,8 @@ aware of and agree upon. This consists of a small JSON file (e.g. call it `genes "byzantiumBlock": 0, "constantinopleBlock": 0, "petersburgBlock": 0, - "istanbulBlock": 0 + "istanbulBlock": 0, + "berlinBlock": 0 }, "alloc": {}, "coinbase": "0x0000000000000000000000000000000000000000", From f66f1a16b3c480d3a43ac7e8a09ab3e362e96ae4 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Fri, 30 Apr 2021 07:00:48 -0400 Subject: [PATCH 327/709] eth/filters: fix comment on PublicFilterAPI timeoutLoop (#22782) --- eth/filters/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 4b36a5379e..e0b07e318e 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -72,8 +72,8 @@ func NewPublicFilterAPI(backend Backend, lightMode bool, timeout time.Duration) return api } -// timeoutLoop runs every 5 minutes and deletes filters that have not been recently used. -// Tt is started when the api is created. +// timeoutLoop runs at the interval set by 'timeout' and deletes filters +// that have not been recently used. It is started when the API is created. func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { var toUninstall []*Subscription ticker := time.NewTicker(timeout) From bbb57fd64b70e3c843b5171d0a4719cf457374fc Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Fri, 30 Apr 2021 07:10:12 -0400 Subject: [PATCH 328/709] core/state: remove toAddr helper in tests (#22772) --- core/state/state_test.go | 14 ++++++-------- core/state/statedb_test.go | 6 +++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/core/state/state_test.go b/core/state/state_test.go index 22e93d7a95..9566531466 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -27,8 +27,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" ) -var toAddr = common.BytesToAddress - type stateTest struct { db ethdb.Database state *StateDB @@ -46,11 +44,11 @@ func TestDump(t *testing.T) { s := &stateTest{db: db, state: sdb} // generate a few entries - obj1 := s.state.GetOrNewStateObject(toAddr([]byte{0x01})) + obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) obj1.AddBalance(big.NewInt(22)) - obj2 := s.state.GetOrNewStateObject(toAddr([]byte{0x01, 0x02})) + obj2 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) - obj3 := s.state.GetOrNewStateObject(toAddr([]byte{0x02})) + obj3 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x02})) obj3.SetBalance(big.NewInt(44)) // write some of them to the trie @@ -108,7 +106,7 @@ func TestNull(t *testing.T) { } func TestSnapshot(t *testing.T) { - stateobjaddr := toAddr([]byte("aa")) + stateobjaddr := common.BytesToAddress([]byte("aa")) var storageaddr common.Hash data1 := common.BytesToHash([]byte{42}) data2 := common.BytesToHash([]byte{43}) @@ -150,8 +148,8 @@ func TestSnapshotEmpty(t *testing.T) { func TestSnapshot2(t *testing.T) { state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) - stateobjaddr0 := toAddr([]byte("so0")) - stateobjaddr1 := toAddr([]byte("so1")) + stateobjaddr0 := common.BytesToAddress([]byte("so0")) + stateobjaddr1 := common.BytesToAddress([]byte("so1")) var storageaddr common.Hash data0 := common.BytesToHash([]byte{17}) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 220e28525c..9524e3730d 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -672,7 +672,7 @@ func TestDeleteCreateRevert(t *testing.T) { // Create an initial state with a single contract state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) - addr := toAddr([]byte("so")) + addr := common.BytesToAddress([]byte("so")) state.SetBalance(addr, big.NewInt(1)) root, _ := state.Commit(false) @@ -705,11 +705,11 @@ func TestMissingTrieNodes(t *testing.T) { db := NewDatabase(memDb) var root common.Hash state, _ := New(common.Hash{}, db, nil) - addr := toAddr([]byte("so")) + addr := common.BytesToAddress([]byte("so")) { state.SetBalance(addr, big.NewInt(1)) state.SetCode(addr, []byte{1, 2, 3}) - a2 := toAddr([]byte("another")) + a2 := common.BytesToAddress([]byte("another")) state.SetBalance(a2, big.NewInt(100)) state.SetCode(a2, []byte{1, 2, 4}) root, _ = state.Commit(false) From 745757ac6bb10c296ab30874ddde774f4fcdec1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 29 Apr 2021 17:33:45 +0300 Subject: [PATCH 329/709] core, eth: abort snapshot generation on snap sync and resume later --- core/blockchain.go | 3 +- core/rawdb/accessors_snapshot.go | 20 +++++++++ core/rawdb/database.go | 6 +-- core/rawdb/schema.go | 3 ++ core/state/snapshot/generate.go | 10 ----- core/state/snapshot/journal.go | 15 ++++--- core/state/snapshot/snapshot.go | 68 +++++++++++++++++++++++++++---- eth/downloader/downloader.go | 10 +++++ eth/downloader/downloader_test.go | 6 +++ eth/protocols/snap/sync.go | 5 --- 10 files changed, 115 insertions(+), 31 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 49aa1c3e86..c3562902d7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -640,7 +640,8 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { headBlockGauge.Update(int64(block.NumberU64())) bc.chainmu.Unlock() - // Destroy any existing state snapshot and regenerate it in the background + // Destroy any existing state snapshot and regenerate it in the background, + // also resuming the normal maintenance of any previously paused snapshot. if bc.snaps != nil { bc.snaps.Rebuild(block.Root()) } diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go index c3616ba3aa..88446e0792 100644 --- a/core/rawdb/accessors_snapshot.go +++ b/core/rawdb/accessors_snapshot.go @@ -24,6 +24,26 @@ import ( "github.com/ethereum/go-ethereum/log" ) +// ReadSnapshotDisabled retrieves if the snapshot maintenance is disabled. +func ReadSnapshotDisabled(db ethdb.KeyValueReader) bool { + disabled, _ := db.Has(snapshotDisabledKey) + return disabled +} + +// WriteSnapshotDisabled stores the snapshot pause flag. +func WriteSnapshotDisabled(db ethdb.KeyValueWriter) { + if err := db.Put(snapshotDisabledKey, []byte("42")); err != nil { + log.Crit("Failed to store snapshot disabled flag", "err", err) + } +} + +// DeleteSnapshotDisabled deletes the flag keeping the snapshot maintenance disabled. +func DeleteSnapshotDisabled(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotDisabledKey); err != nil { + log.Crit("Failed to remove snapshot disabled flag", "err", err) + } +} + // ReadSnapshotRoot retrieves the root of the block whose state is contained in // the persisted snapshot. func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 94759eb984..3a0a26c61d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -371,9 +371,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { var accounted bool for _, meta := range [][]byte{ databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey, - fastTrieProgressKey, snapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, - snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, - badBlockKey, + fastTrieProgressKey, snapshotDisabledKey, snapshotRootKey, snapshotJournalKey, + snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, + uncleanShutdownKey, badBlockKey, } { if bytes.Equal(key, meta) { metadata.Add(size) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 7a97389106..2505ce90b9 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -45,6 +45,9 @@ var ( // fastTrieProgressKey tracks the number of trie entries imported during fast sync. fastTrieProgressKey = []byte("TrieSync") + // snapshotDisabledKey flags that the snapshot should not be maintained due to initial sync. + snapshotDisabledKey = []byte("SnapshotDisabled") + // snapshotRootKey tracks the hash of the last snapshot. snapshotRootKey = []byte("SnapshotRoot") diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 8992d3f91b..7e29e51b21 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -141,16 +141,6 @@ func (gs *generatorStats) Log(msg string, root common.Hash, marker []byte) { log.Info(msg, ctx...) } -// ClearSnapshotMarker sets the snapshot marker to zero, meaning that snapshots -// are not usable. -func ClearSnapshotMarker(diskdb ethdb.KeyValueStore) { - batch := diskdb.NewBatch() - journalProgress(batch, []byte{}, nil) - if err := batch.Write(); err != nil { - log.Crit("Failed to write initialized state marker", "err", err) - } -} - // generateSnapshot regenerates a brand new snapshot based on an existing state // database and head block asynchronously. The snapshot is returned immediately // and generation is continued in the background until done. diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index f8cec4d4ea..5cfb9a9f2a 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -126,12 +126,17 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) { + // If snapshotting is disabled (initial sync in progress), don't do anything, + // wait for the chain to permit us to do something meaningful + if rawdb.ReadSnapshotDisabled(diskdb) { + return nil, true, nil + } // Retrieve the block number and hash of the snapshot, failing if no snapshot // is present in the database (or crashed mid-update). baseRoot := rawdb.ReadSnapshotRoot(diskdb) if baseRoot == (common.Hash{}) { - return nil, errors.New("missing or corrupted snapshot") + return nil, false, errors.New("missing or corrupted snapshot") } base := &diskLayer{ diskdb: diskdb, @@ -142,7 +147,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, snapshot, generator, err := loadAndParseJournal(diskdb, base) if err != nil { log.Warn("Failed to load new-format journal", "error", err) - return nil, err + return nil, false, err } // Entire snapshot journal loaded, sanity check the head. If the loaded // snapshot is not matched with current state root, print a warning log @@ -157,7 +162,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. if !recovery { - return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root) + return nil, false, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root) } // It's in snapshot recovery, the assumption is held that // the disk layer is always higher than chain head. It can @@ -187,7 +192,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, storage: common.StorageSize(generator.Storage), }) } - return snapshot, nil + return snapshot, false, nil } // loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 9ecbd4a6c8..cb8ec7a707 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -148,11 +148,11 @@ type snapshot interface { StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) } -// SnapshotTree is an Ethereum state snapshot tree. It consists of one persistent -// base layer backed by a key-value store, on top of which arbitrarily many in- -// memory diff layers are topped. The memory diffs can form a tree with branching, -// but the disk layer is singleton and common to all. If a reorg goes deeper than -// the disk layer, everything needs to be deleted. +// Tree is an Ethereum state snapshot tree. It consists of one persistent base +// layer backed by a key-value store, on top of which arbitrarily many in-memory +// diff layers are topped. The memory diffs can form a tree with branching, but +// the disk layer is singleton and common to all. If a reorg goes deeper than the +// disk layer, everything needs to be deleted. // // The goal of a state snapshot is twofold: to allow direct access to account and // storage data to avoid expensive multi-level trie lookups; and to allow sorted, @@ -186,7 +186,11 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm defer snap.waitBuild() } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + if disabled { + log.Warn("Snapshot maintenance disabled (syncing)") + return snap, nil + } if err != nil { if rebuild { log.Warn("Failed to load snapshot, regenerating", "err", err) @@ -224,6 +228,55 @@ func (t *Tree) waitBuild() { } } +// Disable interrupts any pending snapshot generator, deletes all the snapshot +// layers in memory and marks snapshots disabled globally. In order to resume +// the snapshot functionality, the caller must invoke Rebuild. +func (t *Tree) Disable() { + // Interrupt any live snapshot layers + t.lock.Lock() + defer t.lock.Unlock() + + for _, layer := range t.layers { + switch layer := layer.(type) { + case *diskLayer: + // If the base layer is generating, abort it + if layer.genAbort != nil { + abort := make(chan *generatorStats) + layer.genAbort <- abort + <-abort + } + // Layer should be inactive now, mark it as stale + layer.lock.Lock() + layer.stale = true + layer.lock.Unlock() + + case *diffLayer: + // If the layer is a simple diff, simply mark as stale + layer.lock.Lock() + atomic.StoreUint32(&layer.stale, 1) + layer.lock.Unlock() + + default: + panic(fmt.Sprintf("unknown layer type: %T", layer)) + } + } + t.layers = map[common.Hash]snapshot{} + + // Delete all snapshot liveness information from the database + batch := t.diskdb.NewBatch() + + rawdb.WriteSnapshotDisabled(batch) + rawdb.DeleteSnapshotRoot(batch) + rawdb.DeleteSnapshotJournal(batch) + rawdb.DeleteSnapshotGenerator(batch) + rawdb.DeleteSnapshotRecoveryNumber(batch) + // Note, we don't delete the sync progress + + if err := batch.Write(); err != nil { + log.Crit("Failed to disable snapshots", "err", err) + } +} + // Snapshot retrieves a snapshot belonging to the given block root, or nil if no // snapshot is maintained for that block. func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot { @@ -626,8 +679,9 @@ func (t *Tree) Rebuild(root common.Hash) { defer t.lock.Unlock() // Firstly delete any recovery flag in the database. Because now we are - // building a brand new snapshot. + // building a brand new snapshot. Also reenable the snapshot feature. rawdb.DeleteSnapshotRecoveryNumber(t.diskdb) + rawdb.DeleteSnapshotDisabled(t.diskdb) // Iterate over and mark all layers stale for _, layer := range t.layers { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index b8cb48914e..6f59b29a5e 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -214,6 +215,9 @@ type BlockChain interface { // InsertReceiptChain inserts a batch of receipts into the local chain. InsertReceiptChain(types.Blocks, []types.Receipts, uint64) (int, error) + + // Snapshots returns the blockchain snapshot tree to paused it during sync. + Snapshots() *snapshot.Tree } // New creates a new downloader to fetch hashes and blocks from remote peers. @@ -393,6 +397,12 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode // but until snap becomes prevalent, we should support both. TODO(karalabe). if mode == SnapSync { if !d.snapSync { + // Snap sync uses the snapshot namespace to store potentially flakey data until + // sync completely heals and finishes. Pause snapshot maintenance in the mean + // time to prevent access. + if snapshots := d.blockchain.Snapshots(); snapshots != nil { // Only nil in tests + snapshots.Disable() + } log.Warn("Enabling snapshot sync prototype") d.snapSync = true } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 1140a444c1..794160993b 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/ethdb" @@ -409,6 +410,11 @@ func (dl *downloadTester) dropPeer(id string) { dl.downloader.UnregisterPeer(id) } +// Snapshots implements the BlockChain interface for the downloader, but is a noop. +func (dl *downloadTester) Snapshots() *snapshot.Tree { + return nil +} + type downloadTesterPeer struct { dl *downloadTester id string diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index d9c0cb9b1b..2373149165 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -551,11 +551,6 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { log.Debug("Snapshot sync already completed") return nil } - // If sync is still not finished, we need to ensure that any marker is wiped. - // Otherwise, it may happen that requests for e.g. genesis-data is delivered - // from the snapshot data, instead of from the trie - snapshot.ClearSnapshotMarker(s.db) - defer func() { // Persist any progress, independent of failure for _, task := range s.tasks { s.forwardAccountTask(task) From 52b5d2d8699c2ffea6ea6a2de69b30a1911359bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 30 Apr 2021 18:24:34 +0300 Subject: [PATCH 330/709] eth/protocols/snap: use storage batch, not account batch in st task --- eth/protocols/snap/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 2373149165..e283473207 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -690,7 +690,7 @@ func (s *Syncer) loadSyncStatus() { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - subtask.genTrie = trie.NewStackTrie(task.genBatch) + subtask.genTrie = trie.NewStackTrie(subtask.genBatch) } } } From 8ff98108e53c01acb4266f23a272c9d707cb3dcd Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 30 Apr 2021 22:47:36 +0200 Subject: [PATCH 331/709] cmd/devp2p: fix flakey tests in CI (#22757) This PR fixes a couple of issues in the eth test suite that caused flakiness when run in the CI. --- cmd/devp2p/internal/ethtest/eth66_suite.go | 12 +-------- .../internal/ethtest/eth66_suiteHelpers.go | 19 ++++++++------ cmd/devp2p/internal/ethtest/suite.go | 25 ++++++++----------- cmd/devp2p/internal/ethtest/types.go | 13 +++++++--- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index 41177189dd..d4890d8de6 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -217,17 +217,7 @@ func (s *Suite) TestLargeAnnounce_66(t *utesting.T) { sendConn.Close() } // Test the last block as a valid block - sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) - defer sendConn.Close() - defer receiveConn.Close() - - s.testAnnounce66(t, sendConn, receiveConn, blocks[3]) - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) - // wait for client to update its chain - if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil { - t.Fatal(err) - } + s.sendNextBlock66(t) } func (s *Suite) TestOldAnnounce_66(t *utesting.T) { diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go index fec02b5246..3c5b22f0b5 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go @@ -229,8 +229,11 @@ func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *New func (c *Conn) waitForBlock66(block *types.Block) error { defer c.SetReadDeadline(time.Time{}) - timeout := time.Now().Add(20 * time.Second) - c.SetReadDeadline(timeout) + c.SetReadDeadline(time.Now().Add(20 * time.Second)) + // note: if the node has not yet imported the block, it will respond + // to the GetBlockHeaders request with an empty BlockHeaders response, + // so the GetBlockHeaders request must be sent again until the BlockHeaders + // response contains the desired header. for { req := eth.GetBlockHeadersPacket66{ RequestId: 54, @@ -253,8 +256,10 @@ func (c *Conn) waitForBlock66(block *types.Block) error { if reqID != req.RequestId { return fmt.Errorf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID) } - if len(msg) > 0 { - return nil + for _, header := range msg { + if header.Number.Uint64() == block.NumberU64() { + return nil + } } time.Sleep(100 * time.Millisecond) case *NewPooledTransactionHashes: @@ -319,10 +324,10 @@ func (s *Suite) sendNextBlock66(t *utesting.T) { } // send announcement and wait for node to request the header s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement) - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) // wait for client to update its chain - if err := receiveConn.waitForBlock66(s.chain.Head()); err != nil { + if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil { t.Fatal(err) } + // update test suite chain + s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 2fa31ad31d..abc6bcddce 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -260,22 +260,28 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { // TestBroadcast tests whether a block announcement is correctly // propagated to the given node's peer(s). func (s *Suite) TestBroadcast(t *utesting.T) { + s.sendNextBlock(t) +} + +func (s *Suite) sendNextBlock(t *utesting.T) { sendConn, receiveConn := s.setupConnection(t), s.setupConnection(t) defer sendConn.Close() defer receiveConn.Close() + // create new block announcement nextBlock := len(s.chain.blocks) blockAnnouncement := &NewBlock{ Block: s.fullChain.blocks[nextBlock], TD: s.fullChain.TD(nextBlock + 1), } + // send announcement and wait for node to request the header s.testAnnounce(t, sendConn, receiveConn, blockAnnouncement) - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) // wait for client to update its chain - if err := receiveConn.waitForBlock(s.chain.Head()); err != nil { + if err := receiveConn.waitForBlock(s.fullChain.blocks[nextBlock]); err != nil { t.Fatal(err) } + // update test suite chain + s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) } // TestMaliciousHandshake tries to send malicious data during the handshake. @@ -394,18 +400,7 @@ func (s *Suite) TestLargeAnnounce(t *utesting.T) { sendConn.Close() } // Test the last block as a valid block - sendConn := s.setupConnection(t) - receiveConn := s.setupConnection(t) - defer sendConn.Close() - defer receiveConn.Close() - - s.testAnnounce(t, sendConn, receiveConn, blocks[3]) - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) - // wait for client to update its chain - if err := receiveConn.waitForBlock(s.fullChain.blocks[nextBlock]); err != nil { - t.Fatal(err) - } + s.sendNextBlock(t) } func (s *Suite) TestOldAnnounce(t *utesting.T) { diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index 55adb75f85..50a69b9418 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -334,8 +334,11 @@ loop: func (c *Conn) waitForBlock(block *types.Block) error { defer c.SetReadDeadline(time.Time{}) - timeout := time.Now().Add(20 * time.Second) - c.SetReadDeadline(timeout) + c.SetReadDeadline(time.Now().Add(20 * time.Second)) + // note: if the node has not yet imported the block, it will respond + // to the GetBlockHeaders request with an empty BlockHeaders response, + // so the GetBlockHeaders request must be sent again until the BlockHeaders + // response contains the desired header. for { req := &GetBlockHeaders{Origin: eth.HashOrNumber{Hash: block.Hash()}, Amount: 1} if err := c.Write(req); err != nil { @@ -343,8 +346,10 @@ func (c *Conn) waitForBlock(block *types.Block) error { } switch msg := c.Read().(type) { case *BlockHeaders: - if len(*msg) > 0 { - return nil + for _, header := range *msg { + if header.Number.Uint64() == block.NumberU64() { + return nil + } } time.Sleep(100 * time.Millisecond) default: From 0e00ee42ec4e43ce3b9b1ffdadea3c66aa6eeba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Sat, 1 May 2021 13:19:24 +0200 Subject: [PATCH 332/709] core/vm: clean up contract creation error handling (#22766) Do not keep separate flag for "max code size exceeded" case, but instead assign appropriate error for it sooner. --- core/vm/evm.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 3f16f33b2d..bd54e855c6 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -463,13 +463,16 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, ret, err := run(evm, contract, nil, false) - // check whether the max code size has been exceeded - maxCodeSizeExceeded := evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize + // Check whether the max code size has been exceeded, assign err if the case. + if err == nil && evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize { + err = ErrMaxCodeSizeExceeded + } + // if the contract creation ran successfully and no errors were returned // calculate the gas required to store the code. If the code could not // be stored due to not enough gas set an error and let it be handled // by the error checking condition below. - if err == nil && !maxCodeSizeExceeded { + if err == nil { createDataGas := uint64(len(ret)) * params.CreateDataGas if contract.UseGas(createDataGas) { evm.StateDB.SetCode(address, ret) @@ -481,21 +484,17 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in homestead this also counts for code storage gas errors. - if maxCodeSizeExceeded || (err != nil && (evm.chainRules.IsHomestead || err != ErrCodeStoreOutOfGas)) { + if err != nil && (evm.chainRules.IsHomestead || err != ErrCodeStoreOutOfGas) { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { contract.UseGas(contract.Gas) } } - // Assign err if contract code size exceeds the max while the err is still empty. - if maxCodeSizeExceeded && err == nil { - err = ErrMaxCodeSizeExceeded - } + if evm.vmConfig.Debug && evm.depth == 0 { evm.vmConfig.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) } return ret, address, contract.Gas, err - } // Create creates a new contract using code as deployment code. From ca9c576e6214d7d9607e59dd9912e3def4095d85 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Mon, 3 May 2021 04:58:00 -0400 Subject: [PATCH 333/709] core/vm: fix interpreter comments (#22797) * Fix interpreter comment * Fix comment --- core/vm/interpreter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 3b67ad6dea..1022c355c1 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -144,7 +144,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer func() { in.evm.depth-- }() // Make sure the readOnly is only set if we aren't in readOnly yet. - // This makes also sure that the readOnly flag isn't removed for child calls. + // This also makes sure that the readOnly flag isn't removed for child calls. if readOnly && !in.readOnly { in.readOnly = true defer func() { in.readOnly = false }() @@ -226,7 +226,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } else if sLen > operation.maxStack { return nil, &ErrStackOverflow{stackLen: sLen, limit: operation.maxStack} } - // If the operation is valid, enforce and write restrictions + // If the operation is valid, enforce write restrictions if in.readOnly && in.evm.chainRules.IsByzantium { // If the interpreter is operating in readonly mode, make sure no // state-modifying operation is performed. The 3rd stack item From afb097eda83bbc5d69d7fa9aa5ed18b1869af4ba Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 3 May 2021 14:28:02 +0200 Subject: [PATCH 334/709] params: remove dependency on crypto (#22788) * params: remove dependency on crypto Package params should not depend on package crypto because building crypto requires cgo. Since build/ci.go needs package params to get the go-ethereum version number, C code must be compiled in order to run the build tool, which is annoying for certain cross-compilation setups. * params: add SectionHead --- params/config.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/params/config.go b/params/config.go index f4e2f5ea67..eb80bb2e27 100644 --- a/params/config.go +++ b/params/config.go @@ -22,7 +22,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" + "golang.org/x/crypto/sha3" ) // Genesis hashes to enforce below configs on. @@ -278,12 +278,18 @@ func (c *TrustedCheckpoint) HashEqual(hash common.Hash) bool { // Hash returns the hash of checkpoint's four key fields(index, sectionHead, chtRoot and bloomTrieRoot). func (c *TrustedCheckpoint) Hash() common.Hash { - buf := make([]byte, 8+3*common.HashLength) - binary.BigEndian.PutUint64(buf, c.SectionIndex) - copy(buf[8:], c.SectionHead.Bytes()) - copy(buf[8+common.HashLength:], c.CHTRoot.Bytes()) - copy(buf[8+2*common.HashLength:], c.BloomRoot.Bytes()) - return crypto.Keccak256Hash(buf) + var sectionIndex [8]byte + binary.BigEndian.PutUint64(sectionIndex[:], c.SectionIndex) + + w := sha3.NewLegacyKeccak256() + w.Write(sectionIndex[:]) + w.Write(c.SectionHead[:]) + w.Write(c.CHTRoot[:]) + w.Write(c.BloomRoot[:]) + + var h common.Hash + w.Sum(h[:0]) + return h } // Empty returns an indicator whether the checkpoint is regarded as empty. From 8f94fc26e3a9de7a756df455f12f6a8fe2a6ed09 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 3 May 2021 14:29:05 +0200 Subject: [PATCH 335/709] cmd/utils: don't crash on nonexistent datadir (#22738) --- cmd/utils/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 59cf32c983..a50fdd968a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1662,7 +1662,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(DataDirFlag.Name) { // Check if we have an already initialized chain and fall back to // that if so. Otherwise we need to generate a new genesis spec. - chaindb := MakeChainDatabase(ctx, stack, true) + chaindb := MakeChainDatabase(ctx, stack, false) // TODO (MariusVanDerWijden) make this read only if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { cfg.Genesis = nil // fallback to db content } From 856c379626c2ff6d37dc28e79596cfbb814bf1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 3 May 2021 15:42:43 +0300 Subject: [PATCH 336/709] eth: don't print db upgrade warning on db init --- eth/backend.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/backend.go b/eth/backend.go index 4c7374612e..7aac17eeb3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -165,7 +165,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if bcVersion != nil && *bcVersion > core.BlockChainVersion { return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, params.VersionWithMeta, core.BlockChainVersion) } else if bcVersion == nil || *bcVersion < core.BlockChainVersion { - log.Warn("Upgrade blockchain database version", "from", dbVer, "to", core.BlockChainVersion) + if bcVersion != nil { // only print warning on upgrade, not on init + log.Warn("Upgrade blockchain database version", "from", dbVer, "to", core.BlockChainVersion) + } rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } } From b8040a430e34117f121c67e8deee4a5e889e8372 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 4 May 2021 11:29:32 +0200 Subject: [PATCH 337/709] cmd/utils: use eth DNS tree for snap discovery (#22808) This removes auto-configuration of the snap.*.ethdisco.net DNS discovery tree. Since measurements have shown that > 75% of nodes in all.*.ethdisco.net support snap, we have decided to retire the dedicated index for snap and just use the eth tree instead. The dial iterators of eth and snap now use the same DNS tree in the default configuration, so both iterators should use the same DNS discovery client instance. This ensures that the record cache and rate limit are shared. Records will not be requested multiple times. While testing the change, I noticed that duplicate DNS requests do happen even when the client instance is shared. This is because the two iterators request the tree root, link tree root, and first levels of the tree in lockstep. To avoid this problem, the change also adds a singleflight.Group instance in the client. When one iterator attempts to resolve an entry which is already being resolved, the singleflight object waits for the existing resolve call to finish and returns the entry to both places. --- cmd/utils/flags.go | 6 +---- eth/backend.go | 10 +++++-- eth/discovery.go | 11 -------- eth/protocols/snap/handler.go | 6 +++++ go.mod | 1 + go.sum | 2 ++ p2p/dnsdisc/client.go | 51 ++++++++++++++++++++--------------- 7 files changed, 48 insertions(+), 39 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a50fdd968a..a81188342f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1690,11 +1690,7 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) { } if url := params.KnownDNSNetwork(genesis, protocol); url != "" { cfg.EthDiscoveryURLs = []string{url} - } - if cfg.SyncMode == downloader.SnapSync { - if url := params.KnownDNSNetwork(genesis, "snap"); url != "" { - cfg.SnapDiscoveryURLs = []string{url} - } + cfg.SnapDiscoveryURLs = cfg.EthDiscoveryURLs } } diff --git a/eth/backend.go b/eth/backend.go index 7aac17eeb3..7d8b0c52c6 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -50,6 +50,7 @@ import ( "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -239,14 +240,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) - eth.ethDialCandidates, err = setupDiscovery(eth.config.EthDiscoveryURLs) + // Setup DNS discovery iterators. + dnsclient := dnsdisc.NewClient(dnsdisc.Config{}) + eth.ethDialCandidates, err = dnsclient.NewIterator(eth.config.EthDiscoveryURLs...) if err != nil { return nil, err } - eth.snapDialCandidates, err = setupDiscovery(eth.config.SnapDiscoveryURLs) + eth.snapDialCandidates, err = dnsclient.NewIterator(eth.config.SnapDiscoveryURLs...) if err != nil { return nil, err } + // Start the RPC service eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, config.NetworkId) @@ -544,6 +548,8 @@ func (s *Ethereum) Start() error { // Ethereum protocol. func (s *Ethereum) Stop() error { // Stop all the peer-related stuff first. + s.ethDialCandidates.Close() + s.snapDialCandidates.Close() s.handler.Stop() // Then stop everything else. diff --git a/eth/discovery.go b/eth/discovery.go index 855ce3b0e1..70668b2b70 100644 --- a/eth/discovery.go +++ b/eth/discovery.go @@ -19,7 +19,6 @@ package eth import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" ) @@ -62,13 +61,3 @@ func (eth *Ethereum) currentEthEntry() *ethEntry { return ðEntry{ForkID: forkid.NewID(eth.blockchain.Config(), eth.blockchain.Genesis().Hash(), eth.blockchain.CurrentHeader().Number.Uint64())} } - -// setupDiscovery creates the node discovery source for the `eth` and `snap` -// protocols. -func setupDiscovery(urls []string) (enode.Iterator, error) { - if len(urls) == 0 { - return nil, nil - } - client := dnsdisc.NewClient(dnsdisc.Config{}) - return client.NewIterator(urls...) -} diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 9bfac6f03f..3d668a2ebb 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -84,6 +84,12 @@ type Backend interface { // MakeProtocols constructs the P2P protocol definitions for `snap`. func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { + // Filter the discovery iterator for nodes advertising snap support. + dnsdisc = enode.Filter(dnsdisc, func(n *enode.Node) bool { + var snap enrEntry + return n.Load(&snap) == nil + }) + protocols := make([]p2p.Protocol, len(ProtocolVersions)) for i, version := range ProtocolVersions { version := version // Closure diff --git a/go.mod b/go.mod index 63636caae1..079d40ae6f 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 golang.org/x/text v0.3.4 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 diff --git a/go.sum b/go.sum index cadcd1de95..3ae80f7b9c 100644 --- a/go.sum +++ b/go.sum @@ -455,6 +455,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 3e4b50aadd..f2a4bed4c6 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -32,15 +32,17 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" lru "github.com/hashicorp/golang-lru" + "golang.org/x/sync/singleflight" "golang.org/x/time/rate" ) // Client discovers nodes by querying DNS servers. type Client struct { - cfg Config - clock mclock.Clock - entries *lru.Cache - ratelimit *rate.Limiter + cfg Config + clock mclock.Clock + entries *lru.Cache + ratelimit *rate.Limiter + singleflight singleflight.Group } // Config holds configuration options for the client. @@ -135,17 +137,20 @@ func (c *Client) NewIterator(urls ...string) (enode.Iterator, error) { // resolveRoot retrieves a root entry via DNS. func (c *Client) resolveRoot(ctx context.Context, loc *linkEntry) (rootEntry, error) { - txts, err := c.cfg.Resolver.LookupTXT(ctx, loc.domain) - c.cfg.Logger.Trace("Updating DNS discovery root", "tree", loc.domain, "err", err) - if err != nil { - return rootEntry{}, err - } - for _, txt := range txts { - if strings.HasPrefix(txt, rootPrefix) { - return parseAndVerifyRoot(txt, loc) + e, err, _ := c.singleflight.Do(loc.str, func() (interface{}, error) { + txts, err := c.cfg.Resolver.LookupTXT(ctx, loc.domain) + c.cfg.Logger.Trace("Updating DNS discovery root", "tree", loc.domain, "err", err) + if err != nil { + return rootEntry{}, err } - } - return rootEntry{}, nameError{loc.domain, errNoRoot} + for _, txt := range txts { + if strings.HasPrefix(txt, rootPrefix) { + return parseAndVerifyRoot(txt, loc) + } + } + return rootEntry{}, nameError{loc.domain, errNoRoot} + }) + return e.(rootEntry), err } func parseAndVerifyRoot(txt string, loc *linkEntry) (rootEntry, error) { @@ -168,17 +173,21 @@ func (c *Client) resolveEntry(ctx context.Context, domain, hash string) (entry, if err := c.ratelimit.Wait(ctx); err != nil { return nil, err } - cacheKey := truncateHash(hash) if e, ok := c.entries.Get(cacheKey); ok { return e.(entry), nil } - e, err := c.doResolveEntry(ctx, domain, hash) - if err != nil { - return nil, err - } - c.entries.Add(cacheKey, e) - return e, nil + + ei, err, _ := c.singleflight.Do(cacheKey, func() (interface{}, error) { + e, err := c.doResolveEntry(ctx, domain, hash) + if err != nil { + return nil, err + } + c.entries.Add(cacheKey, e) + return e, nil + }) + e, _ := ei.(entry) + return e, err } // doResolveEntry fetches an entry via DNS. From effaf185234533b787926a67a6f73eb8d636e7af Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 4 May 2021 13:01:20 +0200 Subject: [PATCH 338/709] build: improve cross compilation setup (#22804) This PR cleans up the CI build system and fixes a couple of issues. - The go tool launcher code has been moved to internal/build. With the new toolchain functions, the environment of the host Go (i.e. the one that built ci.go) and the target Go (i.e. the toolchain downloaded by -dlgo) are isolated more strictly. This is important to make cross compilation and -dlgo work correctly in more cases. - The -dlgo option now skips the download and uses the host Go if the running Go version matches dlgoVersion exactly. - The 'test' command now supports -dlgo, -cc and -arch. Running unit tests with foreign GOARCH is occasionally useful. For example, it can be used to run 32-bit tests on Windows. It can also be used to run darwin/amd64 tests on darwin/arm64 using Rosetta 2. - The 'aar', 'xcode' and 'xgo' commands now use a slightly different method to install external tools. They previously used `go get`, but this comes with the annoying side effect of modifying go.mod. They now use `go install` instead, which is the recommended way of installing tools without modifying the local module. - The old build warning about outdated Go version has been removed because we're much better at keeping backwards compatibility now. --- Makefile | 11 +-- build/ci.go | 206 ++++++++++++--------------------------- internal/build/env.go | 3 + internal/build/gotool.go | 149 ++++++++++++++++++++++++++++ internal/build/util.go | 14 --- 5 files changed, 221 insertions(+), 162 deletions(-) create mode 100644 internal/build/gotool.go diff --git a/Makefile b/Makefile index ecee5f1894..cb5a87dad0 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ android: @echo "Import \"$(GOBIN)/geth.aar\" to use the library." @echo "Import \"$(GOBIN)/geth-sources.jar\" to add javadocs" @echo "For more info see https://stackoverflow.com/questions/20994336/android-studio-how-to-attach-javadoc" - + ios: $(GORUN) build/ci.go xcode --local @echo "Done building." @@ -46,12 +46,11 @@ clean: # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'. devtools: - env GOBIN= go get -u golang.org/x/tools/cmd/stringer - env GOBIN= go get -u github.com/kevinburke/go-bindata/go-bindata - env GOBIN= go get -u github.com/fjl/gencodec - env GOBIN= go get -u github.com/golang/protobuf/protoc-gen-go + env GOBIN= go install golang.org/x/tools/cmd/stringer@latest + env GOBIN= go install github.com/kevinburke/go-bindata/go-bindata@latest + env GOBIN= go install github.com/fjl/gencodec@latest + env GOBIN= go install github.com/golang/protobuf/protoc-gen-go@latest env GOBIN= go install ./cmd/abigen - @type "npm" 2> /dev/null || echo 'Please install node.js and npm' @type "solc" 2> /dev/null || echo 'Please install solc' @type "protoc" 2> /dev/null || echo 'Please install protoc' diff --git a/build/ci.go b/build/ci.go index b73435c5e6..a89c6e02fd 100644 --- a/build/ci.go +++ b/build/ci.go @@ -208,58 +208,25 @@ func doInstall(cmdline []string) { cc = flag.String("cc", "", "C compiler to cross build with") ) flag.CommandLine.Parse(cmdline) - env := build.Env() - // Check local Go version. People regularly open issues about compilation - // failure with outdated Go. This should save them the trouble. - if !strings.Contains(runtime.Version(), "devel") { - // Figure out the minor version number since we can't textually compare (1.10 < 1.9) - var minor int - fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 13 { - log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.13 and cannot") - log.Println("be compiled with an earlier version. Please upgrade your Go installation.") - os.Exit(1) - } - } - - // Choose which go command we're going to use. - var gobuild *exec.Cmd - if !*dlgo { - // Default behavior: use the go version which runs ci.go right now. - gobuild = goTool("build") - } else { - // Download of Go requested. This is for build environments where the - // installed version is too old and cannot be upgraded easily. - cachedir := filepath.Join("build", "cache") - goroot := downloadGo(runtime.GOARCH, runtime.GOOS, cachedir) - gobuild = localGoTool(goroot, "build") + // Configure the toolchain. + tc := build.GoToolchain{GOARCH: *arch, CC: *cc} + if *dlgo { + csdb := build.MustLoadChecksums("build/checksums.txt") + tc.Root = build.DownloadGo(csdb, dlgoVersion) } - // Configure environment for cross build. - if *arch != "" || *arch != runtime.GOARCH { - gobuild.Env = append(gobuild.Env, "CGO_ENABLED=1") - gobuild.Env = append(gobuild.Env, "GOARCH="+*arch) - } - - // Configure C compiler. - if *cc != "" { - gobuild.Env = append(gobuild.Env, "CC="+*cc) - } else if os.Getenv("CC") != "" { - gobuild.Env = append(gobuild.Env, "CC="+os.Getenv("CC")) - } + // Configure the build. + env := build.Env() + gobuild := tc.Go("build", buildFlags(env)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably // check for something in env instead. - if runtime.GOARCH == "arm64" { + if env.CI && runtime.GOARCH == "arm64" { gobuild.Args = append(gobuild.Args, "-p", "1") } - // Put the default settings in. - gobuild.Args = append(gobuild.Args, buildFlags(env)...) - // We use -trimpath to avoid leaking local paths into the built executables. gobuild.Args = append(gobuild.Args, "-trimpath") @@ -301,53 +268,30 @@ func buildFlags(env build.Environment) (flags []string) { return flags } -// goTool returns the go tool. This uses the Go version which runs ci.go. -func goTool(subcmd string, args ...string) *exec.Cmd { - cmd := build.GoTool(subcmd, args...) - goToolSetEnv(cmd) - return cmd -} - -// localGoTool returns the go tool from the given GOROOT. -func localGoTool(goroot string, subcmd string, args ...string) *exec.Cmd { - gotool := filepath.Join(goroot, "bin", "go") - cmd := exec.Command(gotool, subcmd) - goToolSetEnv(cmd) - cmd.Env = append(cmd.Env, "GOROOT="+goroot) - cmd.Args = append(cmd.Args, args...) - return cmd -} - -// goToolSetEnv forwards the build environment to the go tool. -func goToolSetEnv(cmd *exec.Cmd) { - cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) - for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOBIN=") || strings.HasPrefix(e, "CC=") { - continue - } - cmd.Env = append(cmd.Env, e) - } -} - // Running The Tests // // "tests" also includes static analysis tools such as vet. func doTest(cmdline []string) { - coverage := flag.Bool("coverage", false, "Whether to record code coverage") - verbose := flag.Bool("v", false, "Whether to log verbosely") + var ( + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Run tests for given architecture") + cc = flag.String("cc", "", "Sets C compiler binary") + coverage = flag.Bool("coverage", false, "Whether to record code coverage") + verbose = flag.Bool("v", false, "Whether to log verbosely") + ) flag.CommandLine.Parse(cmdline) - env := build.Env() - packages := []string{"./..."} - if len(flag.CommandLine.Args()) > 0 { - packages = flag.CommandLine.Args() + // Configure the toolchain. + tc := build.GoToolchain{GOARCH: *arch, CC: *cc} + if *dlgo { + csdb := build.MustLoadChecksums("build/checksums.txt") + tc.Root = build.DownloadGo(csdb, dlgoVersion) } + gotest := tc.Go("test") - // Run the actual tests. // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. - gotest := goTool("test", buildFlags(env)...) gotest.Args = append(gotest.Args, "-p", "1") if *coverage { gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") @@ -356,6 +300,10 @@ func doTest(cmdline []string) { gotest.Args = append(gotest.Args, "-v") } + packages := []string{"./..."} + if len(flag.CommandLine.Args()) > 0 { + packages = flag.CommandLine.Args() + } gotest.Args = append(gotest.Args, packages...) build.MustRun(gotest) } @@ -415,8 +363,7 @@ func doArchive(cmdline []string) { } var ( - env = build.Env() - + env = build.Env() basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) geth = "geth-" + basegeth + ext alltools = "geth-alltools-" + basegeth + ext @@ -492,15 +439,15 @@ func archiveUpload(archive string, blobstore string, signer string, signifyVar s // skips archiving for some build configurations. func maybeSkipArchive(env build.Environment) { if env.IsPullRequest { - log.Printf("skipping because this is a PR build") + log.Printf("skipping archive creation because this is a PR build") os.Exit(0) } if env.IsCronJob { - log.Printf("skipping because this is a cron job") + log.Printf("skipping archive creation because this is a cron job") os.Exit(0) } if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { - log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) + log.Printf("skipping archive creation because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) os.Exit(0) } } @@ -518,6 +465,7 @@ func doDebianSource(cmdline []string) { flag.CommandLine.Parse(cmdline) *workdir = makeWorkdir(*workdir) env := build.Env() + tc := new(build.GoToolchain) maybeSkipArchive(env) // Import the signing key. @@ -531,12 +479,12 @@ func doDebianSource(cmdline []string) { gobundle := downloadGoSources(*cachedir) // Download all the dependencies needed to build the sources and run the ci script - srcdepfetch := goTool("mod", "download") - srcdepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath")) + srcdepfetch := tc.Go("mod", "download") + srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) build.MustRun(srcdepfetch) - cidepfetch := goTool("run", "./build/ci.go") - cidepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath")) + cidepfetch := tc.Go("run", "./build/ci.go") + cidepfetch.Env = append(cidepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) cidepfetch.Run() // Command fails, don't care, we only need the deps to start it // Create Debian packages and upload them. @@ -592,41 +540,6 @@ func downloadGoSources(cachedir string) string { return dst } -// downloadGo downloads the Go binary distribution and unpacks it into a temporary -// directory. It returns the GOROOT of the unpacked toolchain. -func downloadGo(goarch, goos, cachedir string) string { - if goarch == "arm" { - goarch = "armv6l" - } - - csdb := build.MustLoadChecksums("build/checksums.txt") - file := fmt.Sprintf("go%s.%s-%s", dlgoVersion, goos, goarch) - if goos == "windows" { - file += ".zip" - } else { - file += ".tar.gz" - } - url := "https://golang.org/dl/" + file - dst := filepath.Join(cachedir, file) - if err := csdb.DownloadFile(url, dst); err != nil { - log.Fatal(err) - } - - ucache, err := os.UserCacheDir() - if err != nil { - log.Fatal(err) - } - godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", dlgoVersion, goos, goarch)) - if err := build.ExtractArchive(dst, godir); err != nil { - log.Fatal(err) - } - goroot, err := filepath.Abs(filepath.Join(godir, "go")) - if err != nil { - log.Fatal(err) - } - return goroot -} - func ppaUpload(workdir, ppa, sshUser string, files []string) { p := strings.Split(ppa, "/") if len(p) != 2 { @@ -901,13 +814,23 @@ func doAndroidArchive(cmdline []string) { ) flag.CommandLine.Parse(cmdline) env := build.Env() + tc := new(build.GoToolchain) // Sanity check that the SDK and NDK are installed and set if os.Getenv("ANDROID_HOME") == "" { log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") } + + // Build gomobile. + install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest") + install.Env = append(install.Env) + build.MustRun(install) + + // Ensure all dependencies are available. This is required to make + // gomobile bind work because it expects go.sum to contain all checksums. + build.MustRun(tc.Go("mod", "download")) + // Build the Android archive and Maven resources - build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) if *local { @@ -1027,10 +950,16 @@ func doXCodeFramework(cmdline []string) { ) flag.CommandLine.Parse(cmdline) env := build.Env() + tc := new(build.GoToolchain) + + // Build gomobile. + build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest")) + + // Ensure all dependencies are available. This is required to make + // gomobile bind work because it expects go.sum to contain all checksums. + build.MustRun(tc.Go("mod", "download")) // Build the iOS XCode framework - build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) - build.MustRun(gomobileTool("init")) bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") if *local { @@ -1039,17 +968,14 @@ func doXCodeFramework(cmdline []string) { build.MustRun(bind) return } + + // Create the archive. + maybeSkipArchive(env) archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) - if err := os.Mkdir(archive, os.ModePerm); err != nil { - log.Fatal(err) - } bind.Dir, _ = filepath.Abs(archive) build.MustRun(bind) build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) - // Skip CocoaPods deploy and Azure upload for PR builds - maybeSkipArchive(env) - // Sign and upload the framework to Azure if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil { log.Fatal(err) @@ -1115,10 +1041,10 @@ func doXgo(cmdline []string) { ) flag.CommandLine.Parse(cmdline) env := build.Env() + var tc build.GoToolchain // Make sure xgo is available for cross compilation - gogetxgo := goTool("get", "github.com/karalabe/xgo") - build.MustRun(gogetxgo) + build.MustRun(tc.Install(GOBIN, "github.com/karalabe/xgo@latest")) // If all tools building is requested, build everything the builder wants args := append(buildFlags(env), flag.Args()...) @@ -1129,27 +1055,23 @@ func doXgo(cmdline []string) { if strings.HasPrefix(res, GOBIN) { // Binary tool found, cross build it explicitly args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) - xgo := xgoTool(args) - build.MustRun(xgo) + build.MustRun(xgoTool(args)) args = args[:len(args)-1] } } return } - // Otherwise xxecute the explicit cross compilation + + // Otherwise execute the explicit cross compilation path := args[len(args)-1] args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...) - - xgo := xgoTool(args) - build.MustRun(xgo) + build.MustRun(xgoTool(args)) } func xgoTool(args []string) *exec.Cmd { cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, []string{ - "GOBIN=" + GOBIN, - }...) + cmd.Env = append(cmd.Env, []string{"GOBIN=" + GOBIN}...) return cmd } diff --git a/internal/build/env.go b/internal/build/env.go index a3017182e3..d70c0d50a4 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -38,6 +38,7 @@ var ( // Environment contains metadata provided by the build environment. type Environment struct { + CI bool Name string // name of the environment Repo string // name of GitHub repo Commit, Date, Branch, Tag string // Git info @@ -61,6 +62,7 @@ func Env() Environment { commit = os.Getenv("TRAVIS_COMMIT") } return Environment{ + CI: true, Name: "travis", Repo: os.Getenv("TRAVIS_REPO_SLUG"), Commit: commit, @@ -77,6 +79,7 @@ func Env() Environment { commit = os.Getenv("APPVEYOR_REPO_COMMIT") } return Environment{ + CI: true, Name: "appveyor", Repo: os.Getenv("APPVEYOR_REPO_NAME"), Commit: commit, diff --git a/internal/build/gotool.go b/internal/build/gotool.go new file mode 100644 index 0000000000..e644b5f695 --- /dev/null +++ b/internal/build/gotool.go @@ -0,0 +1,149 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package build + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +type GoToolchain struct { + Root string // GOROOT + + // Cross-compilation variables. These are set when running the go tool. + GOARCH string + GOOS string + CC string +} + +// Go creates an invocation of the go command. +func (g *GoToolchain) Go(command string, args ...string) *exec.Cmd { + tool := g.goTool(command, args...) + + // Configure environment for cross build. + if g.GOARCH != "" && g.GOARCH != runtime.GOARCH { + tool.Env = append(tool.Env, "CGO_ENABLED=1") + tool.Env = append(tool.Env, "GOARCH="+g.GOARCH) + } + if g.GOOS != "" && g.GOOS != runtime.GOOS { + tool.Env = append(tool.Env, "GOOS="+g.GOOS) + } + // Configure C compiler. + if g.CC != "" { + tool.Env = append(tool.Env, "CC="+g.CC) + } else if os.Getenv("CC") != "" { + tool.Env = append(tool.Env, "CC="+os.Getenv("CC")) + } + return tool +} + +// Install creates an invocation of 'go install'. The command is configured to output +// executables to the given 'gobin' directory. +// +// This can be used to install auxiliary build tools without modifying the local go.mod and +// go.sum files. To install tools which are not required by go.mod, ensure that all module +// paths in 'args' contain a module version suffix (e.g. "...@latest"). +func (g *GoToolchain) Install(gobin string, args ...string) *exec.Cmd { + if !filepath.IsAbs(gobin) { + panic("GOBIN must be an absolute path") + } + tool := g.goTool("install") + tool.Env = append(tool.Env, "GOBIN="+gobin) + tool.Args = append(tool.Args, "-mod=readonly") + tool.Args = append(tool.Args, args...) + + // Ensure GOPATH is set because go install seems to absolutely require it. This uses + // 'go env' because it resolves the default value when GOPATH is not set in the + // environment. Ignore errors running go env and leave any complaining about GOPATH to + // the install command. + pathTool := g.goTool("env", "GOPATH") + output, _ := pathTool.Output() + tool.Env = append(tool.Env, "GOPATH="+string(output)) + return tool +} + +func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { + if g.Root == "" { + g.Root = runtime.GOROOT() + } + tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) + tool.Args = append(tool.Args, args...) + tool.Env = append(tool.Env, "GOROOT="+g.Root) + + // Forward environment variables to the tool, but skip compiler target settings. + // TODO: what about GOARM? + skip := map[string]struct{}{"GOROOT": {}, "GOARCH": {}, "GOOS": {}, "GOBIN": {}, "CC": {}} + for _, e := range os.Environ() { + if i := strings.IndexByte(e, '='); i >= 0 { + if _, ok := skip[e[:i]]; ok { + continue + } + } + tool.Env = append(tool.Env, e) + } + return tool +} + +// DownloadGo downloads the Go binary distribution and unpacks it into a temporary +// directory. It returns the GOROOT of the unpacked toolchain. +func DownloadGo(csdb *ChecksumDB, version string) string { + // Shortcut: if the Go version that runs this script matches the + // requested version exactly, there is no need to download anything. + activeGo := strings.TrimPrefix(runtime.Version(), "go") + if activeGo == version { + log.Printf("-dlgo version matches active Go version %s, skipping download.", activeGo) + return runtime.GOROOT() + } + + ucache, err := os.UserCacheDir() + if err != nil { + log.Fatal(err) + } + + // For Arm architecture, GOARCH includes ISA version. + os := runtime.GOOS + arch := runtime.GOARCH + if arch == "arm" { + arch = "armv6l" + } + file := fmt.Sprintf("go%s.%s-%s", version, os, arch) + if os == "windows" { + file += ".zip" + } else { + file += ".tar.gz" + } + url := "https://golang.org/dl/" + file + dst := filepath.Join(ucache, file) + if err := csdb.DownloadFile(url, dst); err != nil { + log.Fatal(err) + } + + godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", version, os, arch)) + if err := ExtractArchive(dst, godir); err != nil { + log.Fatal(err) + } + goroot, err := filepath.Abs(filepath.Join(godir, "go")) + if err != nil { + log.Fatal(err) + } + return goroot +} diff --git a/internal/build/util.go b/internal/build/util.go index 91149926f7..2bdced82ee 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,7 +29,6 @@ import ( "os/exec" "path" "path/filepath" - "runtime" "strings" "text/template" ) @@ -111,19 +110,6 @@ func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x } } -// GoTool returns the command that runs a go tool. This uses go from GOROOT instead of PATH -// so that go commands executed by build use the same version of Go as the 'host' that runs -// build code. e.g. -// -// /usr/lib/go-1.12.1/bin/go run build/ci.go ... -// -// runs using go 1.12.1 and invokes go 1.12.1 tools from the same GOROOT. This is also important -// because runtime.Version checks on the host should match the tools that are run. -func GoTool(tool string, args ...string) *exec.Cmd { - args = append([]string{tool}, args...) - return exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) -} - // UploadSFTP uploads files to a remote host using the sftp command line tool. // The destination host may be specified either as [user@]host[: or as a URI in // the form sftp://[user@]host[:port]. From d107f90d1ce74c5d60c11a7171c0178bde12f14f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 4 May 2021 21:45:21 +0200 Subject: [PATCH 339/709] go.mod: go mod tidy (#22814) This updates go.mod for the addition of golang.org/x/sync. --- go.mod | 2 +- go.sum | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 079d40ae6f..512d541f41 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 golang.org/x/text v0.3.4 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 diff --git a/go.sum b/go.sum index 3ae80f7b9c..b6a27a2cf1 100644 --- a/go.sum +++ b/go.sum @@ -453,7 +453,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 973ad66b498546468c66ec3cc9d5aa111c1df948 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 4 May 2021 21:45:45 +0200 Subject: [PATCH 340/709] build: fix iOS framework build (#22813) This fixes a regression introduced in #22804. --- build/ci.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/ci.go b/build/ci.go index a89c6e02fd..d9f147ef0e 100644 --- a/build/ci.go +++ b/build/ci.go @@ -972,6 +972,9 @@ func doXCodeFramework(cmdline []string) { // Create the archive. maybeSkipArchive(env) archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) + if err := os.MkdirAll(archive, 0755); err != nil { + log.Fatal(err) + } bind.Dir, _ = filepath.Abs(archive) build.MustRun(bind) build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) From 3a2b29c1ed91e1099318c3829a44926a1c2a37d9 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 4 May 2021 22:39:09 +0200 Subject: [PATCH 341/709] appveyor.yml: upgrade to VisualStudio 2019 image (#22811) --- appveyor.yml | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 052280be15..a72163382a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,41 +1,29 @@ -os: Visual Studio 2015 - -# Clone directly into GOPATH. -clone_folder: C:\gopath\src\github.com\ethereum\go-ethereum +os: Visual Studio 2019 clone_depth: 5 version: "{branch}.{build}" environment: - global: - GO111MODULE: on - GOPATH: C:\gopath - CC: gcc.exe matrix: + # We use gcc from MSYS2 because it is the most recent compiler version available on + # AppVeyor. Note: gcc.exe only works properly if the corresponding bin/ directory is + # contained in PATH. - GETH_ARCH: amd64 - MSYS2_ARCH: x86_64 - MSYS2_BITS: 64 - MSYSTEM: MINGW64 - PATH: C:\msys64\mingw64\bin\;C:\Program Files (x86)\NSIS\;%PATH% + GETH_CC: C:\msys64\mingw64\bin\gcc.exe + PATH: C:\msys64\mingw64\bin;C:\Program Files (x86)\NSIS\;%PATH% - GETH_ARCH: 386 - MSYS2_ARCH: i686 - MSYS2_BITS: 32 - MSYSTEM: MINGW32 - PATH: C:\msys64\mingw32\bin\;C:\Program Files (x86)\NSIS\;%PATH% + GETH_CC: C:\msys64\mingw32\bin\gcc.exe + PATH: C:\msys64\mingw32\bin;C:\Program Files (x86)\NSIS\;%PATH% install: - - git submodule update --init - - rmdir C:\go /s /q - - appveyor DownloadFile https://dl.google.com/go/go1.16.windows-%GETH_ARCH%.zip - - 7z x go1.16.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - git submodule update --init --depth 1 - go version - - gcc --version + - "%GETH_CC% --version" build_script: - - go run build\ci.go install -dlgo + - go run build\ci.go install -dlgo -arch %GETH_ARCH% -cc %GETH_CC% after_build: - - go run build\ci.go archive -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds - - go run build\ci.go nsis -signer WINDOWS_SIGNING_KEY -upload gethstore/builds + - go run build\ci.go archive -arch %GETH_ARCH% -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds + - go run build\ci.go nsis -arch %GETH_ARCH% -signer WINDOWS_SIGNING_KEY -upload gethstore/builds test_script: - - set CGO_ENABLED=1 - - go run build\ci.go test -coverage + - go run build\ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC% -coverage From 41671d449f5b3a19119eef070151ec72fdbb31f8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 5 May 2021 12:19:51 +0200 Subject: [PATCH 342/709] build: fix windows installer build for NSIS v3.05 (#22821) With the update to a newer AppVeyor build image, creating the Windows installer no longer worked because of a string quoting error in EnvVarUpdate.nsh. This applies the fix recommended in https://stackoverflow.com/questions/62081765. --- build/nsis.envvarupdate.nsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/nsis.envvarupdate.nsh b/build/nsis.envvarupdate.nsh index 9c3ecbe337..95c2f1f639 100644 --- a/build/nsis.envvarupdate.nsh +++ b/build/nsis.envvarupdate.nsh @@ -43,7 +43,7 @@ !ifndef Un${StrFuncName}_INCLUDED ${Un${StrFuncName}} !endif - !define un.${StrFuncName} "${Un${StrFuncName}}" + !define un.${StrFuncName} '${Un${StrFuncName}}' !macroend !insertmacro _IncludeStrFunction StrTok From 0f3a1e7f9b069c7dcc934238bd75ace72c166f36 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 5 May 2021 12:27:27 +0200 Subject: [PATCH 343/709] cmd/devp2p/internal/ethtest: send simultaneous requests on one connection (#22801) This changes the SimultaneousRequests test to send the requests from the same connection, as it doesn't really make sense to test whether a node can respond to two requests with different request IDs from separate connections. --- cmd/devp2p/internal/ethtest/eth66_suite.go | 41 +++++++++++----------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go index d4890d8de6..903a90c7eb 100644 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ b/cmd/devp2p/internal/ethtest/eth66_suite.go @@ -91,9 +91,8 @@ func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) { // headers per request. func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { // create two connections - conn1, conn2 := s.setupConnection66(t), s.setupConnection66(t) - defer conn1.Close() - defer conn2.Close() + conn := s.setupConnection66(t) + defer conn.Close() // create two requests req1 := ð.GetBlockHeadersPacket66{ RequestId: 111, @@ -117,27 +116,29 @@ func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { Reverse: false, }, } - // wait for headers for first request - headerChan := make(chan BlockHeaders, 1) - go func(headers chan BlockHeaders) { - recvHeaders, err := s.getBlockHeaders66(conn1, req1, req1.RequestId) - if err != nil { - t.Fatalf("could not get block headers: %v", err) - return - } - headers <- recvHeaders - }(headerChan) - // check headers of second request - headers1, err := s.getBlockHeaders66(conn2, req2, req2.RequestId) + // write first request + if err := conn.write66(req1, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + // write second request + if err := conn.write66(req2, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + // wait for responses + headers1, err := s.waitForBlockHeadersResponse66(conn, req1.RequestId) if err != nil { - t.Fatalf("could not get block headers: %v", err) + t.Fatalf("error while waiting for block headers: %v", err) + } + headers2, err := s.waitForBlockHeadersResponse66(conn, req2.RequestId) + if err != nil { + t.Fatalf("error while waiting for block headers: %v", err) } + // check headers of both responses if !headersMatch(t, s.chain, headers1) { - t.Fatal("wrong header(s) in response to req2") + t.Fatalf("wrong header(s) in response to req1: got %v", headers1) } - // check headers of first request - if !headersMatch(t, s.chain, <-headerChan) { - t.Fatal("wrong header(s) in response to req1") + if !headersMatch(t, s.chain, headers2) { + t.Fatalf("wrong header(s) in response to req2: got %v", headers2) } } From 991384a7f6719e1125ca0be7fb27d0c4d1c5d2d3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 5 May 2021 13:20:06 +0200 Subject: [PATCH 344/709] params: go-ethereum v1.10.3 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index d350c5ff4a..1c7bf8d88e 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 37b5595456e7049e3ed487c41564281de52e00ab Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 5 May 2021 13:21:13 +0200 Subject: [PATCH 345/709] params: begin v1.10.4 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 1c7bf8d88e..89969705dc 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 4 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From df20b3b98286c9333c143c0a873d09e27b0e0693 Mon Sep 17 00:00:00 2001 From: Evgeny Danilenko <6655321@bk.ru> Date: Thu, 6 May 2021 11:46:27 +0300 Subject: [PATCH 346/709] core/vm: avoid duplicate log in json logger (#22825) --- core/vm/logger_json.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index e54be08596..93878b9806 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -87,8 +87,9 @@ func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, Time time.Duration `json:"time"` Err string `json:"error,omitempty"` } + var errMsg string if err != nil { - l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, err.Error()}) + errMsg = err.Error() } - l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, ""}) + l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, errMsg}) } From cc606be74c6f1f05b0b0a6226a400e734b9aac31 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 6 May 2021 11:07:42 +0200 Subject: [PATCH 347/709] all: define London+baikal, undefine yolov3, add london override flag (#22822) * all: define London+baikal, undefine yolov3, add london override flag * cmd, core, params: add baikal genesis definition --- cmd/geth/chaincmd.go | 2 +- cmd/geth/config.go | 4 ++-- cmd/geth/consolecmd.go | 4 ++-- cmd/geth/dbcmd.go | 16 +++++++------- cmd/geth/main.go | 8 +++---- cmd/geth/usage.go | 2 +- cmd/puppeth/wizard_genesis.go | 4 ++-- cmd/utils/flags.go | 36 +++++++++++++++---------------- core/blockchain_test.go | 2 +- core/genesis.go | 20 ++++++++--------- core/genesis_alloc.go | 2 +- core/genesis_test.go | 4 ++-- core/state/statedb.go | 2 +- core/types/transaction_signing.go | 2 +- core/vm/runtime/runtime.go | 2 +- eth/backend.go | 2 +- eth/ethconfig/config.go | 2 +- les/client.go | 2 +- params/bootnodes.go | 10 ++++----- params/config.go | 34 +++++++++++++++++------------ tests/init.go | 22 ++++++++++++++----- 21 files changed, 100 insertions(+), 82 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 3cae32aa9e..d00b4bc1f6 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -63,7 +63,7 @@ It expects the genesis file as argument.`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/config.go b/cmd/geth/config.go index c867877ee6..c4ebf64881 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -141,8 +141,8 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - if ctx.GlobalIsSet(utils.OverrideBerlinFlag.Name) { - cfg.Eth.OverrideBerlin = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideBerlinFlag.Name)) + if ctx.GlobalIsSet(utils.OverrideLondonFlag.Name) { + cfg.Eth.OverrideLondon = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideLondonFlag.Name)) } backend, eth := utils.RegisterEthService(stack, &cfg.Eth) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 9d8794eb15..5c715a1250 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -134,8 +134,8 @@ func remoteConsole(ctx *cli.Context) error { path = filepath.Join(path, "rinkeby") } else if ctx.GlobalBool(utils.GoerliFlag.Name) { path = filepath.Join(path, "goerli") - } else if ctx.GlobalBool(utils.YoloV3Flag.Name) { - path = filepath.Join(path, "yolo-v3") + } else if ctx.GlobalBool(utils.BaikalFlag.Name) { + path = filepath.Join(path, "baikal") } } endpoint = fmt.Sprintf("%s/geth.ipc", path) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 4c70373e9a..2cd481b706 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -75,7 +75,7 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, @@ -91,7 +91,7 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, } dbCompactCmd = cli.Command{ @@ -105,7 +105,7 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, utils.CacheFlag, utils.CacheDatabaseFlag, }, @@ -125,7 +125,7 @@ corruption if it is aborted during execution'!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -141,7 +141,7 @@ corruption if it is aborted during execution'!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, @@ -158,7 +158,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, @@ -175,7 +175,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -191,7 +191,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, }, Description: "This command displays information about the freezer index.", } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 78e65161da..bed4d8a488 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -66,7 +66,7 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, - utils.OverrideBerlinFlag, + utils.OverrideLondonFlag, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, @@ -138,7 +138,7 @@ var ( utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.YoloV3Flag, + utils.BaikalFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, @@ -275,8 +275,8 @@ func prepare(ctx *cli.Context) { case ctx.GlobalIsSet(utils.GoerliFlag.Name): log.Info("Starting Geth on Görli testnet...") - case ctx.GlobalIsSet(utils.YoloV3Flag.Name): - log.Info("Starting Geth on YOLOv3 testnet...") + case ctx.GlobalIsSet(utils.BaikalFlag.Name): + log.Info("Starting Geth on Baikal testnet...") case ctx.GlobalIsSet(utils.DeveloperFlag.Name): log.Info("Starting Geth in ephemeral dev mode...") diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 980794db73..aa0a4b2901 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -44,7 +44,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MainnetFlag, utils.GoerliFlag, utils.RinkebyFlag, - utils.YoloV3Flag, + utils.BaikalFlag, utils.RopstenFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 4f701fa1c3..ae5977b372 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -240,8 +240,8 @@ func (w *wizard) manageGenesis() { w.conf.Genesis.Config.BerlinBlock = w.readDefaultBigInt(w.conf.Genesis.Config.BerlinBlock) fmt.Println() - fmt.Printf("Which block should YOLOv3 come into effect? (default = %v)\n", w.conf.Genesis.Config.YoloV3Block) - w.conf.Genesis.Config.YoloV3Block = w.readDefaultBigInt(w.conf.Genesis.Config.YoloV3Block) + fmt.Printf("Which block should London come into effect? (default = %v)\n", w.conf.Genesis.Config.LondonBlock) + w.conf.Genesis.Config.LondonBlock = w.readDefaultBigInt(w.conf.Genesis.Config.LondonBlock) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a81188342f..aa00f96c92 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -151,9 +151,9 @@ var ( Name: "goerli", Usage: "Görli network: pre-configured proof-of-authority test network", } - YoloV3Flag = cli.BoolFlag{ - Name: "yolov3", - Usage: "YOLOv3 network: pre-configured proof-of-authority shortlived test network.", + BaikalFlag = cli.BoolFlag{ + Name: "baikal", + Usage: "Bailkal network: pre-configured proof-of-authority shortlived test network.", } RinkebyFlag = cli.BoolFlag{ Name: "rinkeby", @@ -233,9 +233,9 @@ var ( Usage: "Megabytes of memory allocated to bloom-filter for pruning", Value: 2048, } - OverrideBerlinFlag = cli.Uint64Flag{ - Name: "override.berlin", - Usage: "Manually specify Berlin fork-block, overriding the bundled setting", + OverrideLondonFlag = cli.Uint64Flag{ + Name: "override.london", + Usage: "Manually specify London fork-block, overriding the bundled setting", } // Light server and client settings LightServeFlag = cli.IntFlag{ @@ -778,8 +778,8 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(GoerliFlag.Name) { return filepath.Join(path, "goerli") } - if ctx.GlobalBool(YoloV3Flag.Name) { - return filepath.Join(path, "yolo-v3") + if ctx.GlobalBool(BaikalFlag.Name) { + return filepath.Join(path, "baikal") } return path } @@ -833,8 +833,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.RinkebyBootnodes case ctx.GlobalBool(GoerliFlag.Name): urls = params.GoerliBootnodes - case ctx.GlobalBool(YoloV3Flag.Name): - urls = params.YoloV3Bootnodes + case ctx.GlobalBool(BaikalFlag.Name): + urls = params.BaikalBootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -1275,8 +1275,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") - case ctx.GlobalBool(YoloV3Flag.Name) && cfg.DataDir == node.DefaultDataDir(): - cfg.DataDir = filepath.Join(node.DefaultDataDir(), "yolo-v3") + case ctx.GlobalBool(BaikalFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "baikal") } } @@ -1460,7 +1460,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV3Flag) + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, BaikalFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { @@ -1620,11 +1620,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) - case ctx.GlobalBool(YoloV3Flag.Name): + case ctx.GlobalBool(BaikalFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = new(big.Int).SetBytes([]byte("yolov3x")).Uint64() // "yolov3x" + cfg.NetworkId = 1642 // https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/client-integration-testnets/baikal.md } - cfg.Genesis = core.DefaultYoloV3GenesisBlock() + cfg.Genesis = core.DefaultBaikalGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1813,8 +1813,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() - case ctx.GlobalBool(YoloV3Flag.Name): - genesis = core.DefaultYoloV3GenesisBlock() + case ctx.GlobalBool(BaikalFlag.Name): + genesis = core.DefaultBaikalGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 5004abd1c7..75ef399c9a 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -3049,7 +3049,7 @@ func TestEIP2718Transition(t *testing.T) { address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) gspec = &Genesis{ - Config: params.YoloV3ChainConfig, + Config: params.TestChainConfig, Alloc: GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 diff --git a/core/genesis.go b/core/genesis.go index e05e27fe17..9ae718beb6 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -156,7 +156,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig return SetupGenesisBlockWithOverride(db, genesis, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideBerlin *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideLondon *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -202,8 +202,8 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - if overrideBerlin != nil { - newcfg.BerlinBlock = overrideBerlin + if overrideLondon != nil { + newcfg.LondonBlock = overrideLondon } if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err @@ -246,8 +246,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return params.RinkebyChainConfig case ghash == params.GoerliGenesisHash: return params.GoerliChainConfig - case ghash == params.YoloV3GenesisHash: - return params.YoloV3ChainConfig + case ghash == params.BaikalGenesisHash: + return params.BaikalChainConfig default: return params.AllEthashProtocolChanges } @@ -383,15 +383,15 @@ func DefaultGoerliGenesisBlock() *Genesis { } } -func DefaultYoloV3GenesisBlock() *Genesis { +func DefaultBaikalGenesisBlock() *Genesis { // Full genesis: https://gist.github.com/holiman/c6ed9269dce28304ad176314caa75e97 return &Genesis{ - Config: params.YoloV3ChainConfig, - Timestamp: 0x6027dd2e, - ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000001041afbcb359d5a8dc58c15b2ff51354ff8a217d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + Config: params.BaikalChainConfig, + Timestamp: 0x6092ca7f, + ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000005211cea3870c7ba7c6c44b185e62eecdb864cd8c560228ce57d31efbf64c200b2c200aacec78cf17a7148e784fe95a7a750335f8b9572ee28d72e7650000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), GasLimit: 0x47b760, Difficulty: big.NewInt(1), - Alloc: decodePrealloc(yoloV3AllocData), + Alloc: decodePrealloc(baikalAllocData), } } diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 5b0e933d7a..0828d1067e 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,4 +25,4 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const yoloV3AllocData = "\xf9\x05o\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xdf\n\x88\xb2\xb6\x8cg7\x13\xa8\xec\x82`\x03go'.5s\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" +const baikalAllocData = "\xf9\x05\x01\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" diff --git a/core/genesis_test.go b/core/genesis_test.go index 44c1ef253a..eec4c171e8 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -186,8 +186,8 @@ func TestGenesisHashes(t *testing.T) { hash: params.RinkebyGenesisHash, }, { - genesis: DefaultYoloV3GenesisBlock(), - hash: params.YoloV3GenesisHash, + genesis: DefaultBaikalGenesisBlock(), + hash: params.BaikalGenesisHash, }, } for i, c := range cases { diff --git a/core/state/statedb.go b/core/state/statedb.go index 90f4709bfc..203556c6b7 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -991,7 +991,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // - Add precompiles to access list (2929) // - Add the contents of the optional tx access list (2930) // -// This method should only be called if Yolov3/Berlin/2929+2930 is applicable at the current number. +// This method should only be called if Berlin/2929+2930 is applicable at the current number. func (s *StateDB) PrepareAccessList(sender common.Address, dst *common.Address, precompiles []common.Address, list types.AccessList) { s.AddAddressToAccessList(sender) if dst != nil { diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 126748efeb..5d94b26b36 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -61,7 +61,7 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { // have the current block number available, use MakeSigner instead. func LatestSigner(config *params.ChainConfig) Signer { if config.ChainID != nil { - if config.BerlinBlock != nil || config.YoloV3Block != nil { + if config.BerlinBlock != nil { return NewEIP2930Signer(config.ChainID) } if config.EIP155Block != nil { diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 72601441d5..be612fb0ef 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -66,7 +66,7 @@ func setDefaults(cfg *Config) { IstanbulBlock: new(big.Int), MuirGlacierBlock: new(big.Int), BerlinBlock: new(big.Int), - YoloV3Block: nil, + LondonBlock: nil, } } diff --git a/eth/backend.go b/eth/backend.go index 7d8b0c52c6..3e770fe83d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -131,7 +131,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideBerlin) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideLondon) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 0c6eb0bdd7..f5629e6a39 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -200,7 +200,7 @@ type Config struct { CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` // Berlin block override (TODO: remove after the fork) - OverrideBerlin *big.Int `toml:",omitempty"` + OverrideLondon *big.Int `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/les/client.go b/les/client.go index 7534eb3ea0..1d8a2c6f9a 100644 --- a/les/client.go +++ b/les/client.go @@ -88,7 +88,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideBerlin) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideLondon) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } diff --git a/params/bootnodes.go b/params/bootnodes.go index f36ad61729..20bf0b7cbf 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -67,11 +67,11 @@ var GoerliBootnodes = []string{ "enode://a59e33ccd2b3e52d578f1fbd70c6f9babda2650f0760d6ff3b37742fdcdfdb3defba5d56d315b40c46b70198c7621e63ffa3f987389c7118634b0fefbbdfa7fd@51.15.119.157:40303", } -// YoloV3Bootnodes are the enode URLs of the P2P bootstrap nodes running on the -// YOLOv3 ephemeral test network. -// TODO: Set Yolov3 bootnodes -var YoloV3Bootnodes = []string{ - "enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303", +// BaikalBootnodes are the enode URLs of the P2P bootstrap nodes running on the +// Baikal ephemeral test network. +// TODO: Set Baikal bootnodes +var BaikalBootnodes = []string{ + "", } var V5Bootnodes = []string{ diff --git a/params/config.go b/params/config.go index eb80bb2e27..2cafa0e449 100644 --- a/params/config.go +++ b/params/config.go @@ -31,7 +31,7 @@ var ( RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - YoloV3GenesisHash = common.HexToHash("0xf1f2876e8500c77afcc03228757b39477eceffccf645b734967fe3c7e16967b7") + BaikalGenesisHash = common.HexToHash("0xa0bc5d43c72a990cedeb59d305702602b34c3ee8585e77d03c7a4fa64d79636e") ) // TrustedCheckpoints associates each known checkpoint with the genesis hash of @@ -217,9 +217,8 @@ var ( Threshold: 2, } - // YoloV3ChainConfig contains the chain parameters to run a node on the YOLOv3 test network. - YoloV3ChainConfig = &ChainConfig{ - ChainID: new(big.Int).SetBytes([]byte("yolov3x")), + BaikalChainConfig = &ChainConfig{ + ChainID: big.NewInt(1642), HomesteadBlock: big.NewInt(0), DAOForkBlock: nil, DAOForkSupport: true, @@ -231,10 +230,10 @@ var ( PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), MuirGlacierBlock: nil, - BerlinBlock: nil, // Don't enable Berlin directly, we're YOLOing it - YoloV3Block: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), Clique: &CliqueConfig{ - Period: 15, + Period: 30, Epoch: 30000, }, } @@ -331,8 +330,8 @@ type ChainConfig struct { IstanbulBlock *big.Int `json:"istanbulBlock,omitempty"` // Istanbul switch block (nil = no fork, 0 = already on istanbul) MuirGlacierBlock *big.Int `json:"muirGlacierBlock,omitempty"` // Eip-2384 (bomb delay) switch block (nil = no fork, 0 = already activated) BerlinBlock *big.Int `json:"berlinBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) + LondonBlock *big.Int `json:"londonBlock,omitempty"` // London switch block (nil = no fork, 0 = already on london) - YoloV3Block *big.Int `json:"yoloV3Block,omitempty"` // YOLO v3: Gas repricings TODO @holiman add EIP references EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) CatalystBlock *big.Int `json:"catalystBlock,omitempty"` // Catalyst switch block (nil = no fork, 0 = already on catalyst) @@ -371,7 +370,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, YOLO v3: %v, Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, London: %v, Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -385,7 +384,7 @@ func (c *ChainConfig) String() string { c.IstanbulBlock, c.MuirGlacierBlock, c.BerlinBlock, - c.YoloV3Block, + c.LondonBlock, engine, ) } @@ -444,7 +443,12 @@ func (c *ChainConfig) IsIstanbul(num *big.Int) bool { // IsBerlin returns whether num is either equal to the Berlin fork block or greater. func (c *ChainConfig) IsBerlin(num *big.Int) bool { - return isForked(c.BerlinBlock, num) || isForked(c.YoloV3Block, num) + return isForked(c.BerlinBlock, num) +} + +// IsLondon returns whether num is either equal to the London fork block or greater. +func (c *ChainConfig) IsLondon(num *big.Int) bool { + return isForked(c.LondonBlock, num) } // IsCatalyst returns whether num is either equal to the Merge fork block or greater. @@ -496,6 +500,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "istanbulBlock", block: c.IstanbulBlock}, {name: "muirGlacierBlock", block: c.MuirGlacierBlock, optional: true}, {name: "berlinBlock", block: c.BerlinBlock}, + {name: "londonBlock", block: c.LondonBlock}, } { if lastFork.name != "" { // Next one must be higher number @@ -562,8 +567,8 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.BerlinBlock, newcfg.BerlinBlock, head) { return newCompatError("Berlin fork block", c.BerlinBlock, newcfg.BerlinBlock) } - if isForkIncompatible(c.YoloV3Block, newcfg.YoloV3Block, head) { - return newCompatError("YOLOv3 fork block", c.YoloV3Block, newcfg.YoloV3Block) + if isForkIncompatible(c.LondonBlock, newcfg.LondonBlock, head) { + return newCompatError("London fork block", c.LondonBlock, newcfg.LondonBlock) } if isForkIncompatible(c.EWASMBlock, newcfg.EWASMBlock, head) { return newCompatError("ewasm fork block", c.EWASMBlock, newcfg.EWASMBlock) @@ -635,7 +640,7 @@ type Rules struct { ChainID *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool - IsBerlin, IsCatalyst bool + IsBerlin, IsLondon, IsCatalyst bool } // Rules ensures c's ChainID is not nil. @@ -655,6 +660,7 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { IsPetersburg: c.IsPetersburg(num), IsIstanbul: c.IsIstanbul(num), IsBerlin: c.IsBerlin(num), + IsLondon: c.IsLondon(num), IsCatalyst: c.IsCatalyst(num), } } diff --git a/tests/init.go b/tests/init.go index 67f706eb50..240b7159d5 100644 --- a/tests/init.go +++ b/tests/init.go @@ -141,7 +141,7 @@ var Forks = map[string]*params.ChainConfig{ PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(5), }, - "YOLOv3": { + "Berlin": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), @@ -151,11 +151,22 @@ var Forks = map[string]*params.ChainConfig{ ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), - YoloV3Block: big.NewInt(0), + BerlinBlock: big.NewInt(0), }, - // This specification is subject to change, but is for now identical to YOLOv3 - // for cross-client testing purposes - "Berlin": { + "BerlinToLondonAt5": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(5), + }, + "London": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), @@ -166,6 +177,7 @@ var Forks = map[string]*params.ChainConfig{ PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), }, } From e69130d9f15d85c5955e1fa94abc74d606accba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 29 Apr 2021 11:46:03 +0200 Subject: [PATCH 348/709] core/vm, params: implement EIP 3541 --- core/vm/errors.go | 1 + core/vm/evm.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/core/vm/errors.go b/core/vm/errors.go index c813aa36af..c7cfeae53c 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -34,6 +34,7 @@ var ( ErrWriteProtection = errors.New("write protection") ErrReturnDataOutOfBounds = errors.New("return data out of bounds") ErrGasUintOverflow = errors.New("gas uint64 overflow") + ErrInvalidCode = errors.New("invalid code: must not begin with 0xef") ) // ErrStackUnderflow wraps an evm error when the items on the stack less diff --git a/core/vm/evm.go b/core/vm/evm.go index bd54e855c6..8e3c9fe00f 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -468,6 +468,11 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, err = ErrMaxCodeSizeExceeded } + // Reject code starting with 0xEF if EIP-3541 is enabled. + if err == nil && len(ret) >= 1 && ret[0] == 0xEF && evm.chainRules.IsLondon { + err = ErrInvalidCode + } + // if the contract creation ran successfully and no errors were returned // calculate the gas required to store the code. If the code could not // be stored due to not enough gas set an error and let it be handled From a5669ae292274d3b0a223cda07ebe64f943bded4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 7 May 2021 08:25:32 +0200 Subject: [PATCH 349/709] core, params: implement EIP-3529 (#22733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * core, params: implement EIP-3529 * core/vm: add london instructionset * core/vm: add method doc for EIP enabler Co-authored-by: Péter Szilágyi --- core/blockchain_test.go | 3 +- core/state_transition.go | 15 ++- core/vm/eips.go | 26 +++-- core/vm/interpreter.go | 2 + core/vm/jump_table.go | 9 ++ core/vm/operations_acl.go | 210 ++++++++++++++++++++------------------ params/protocol_params.go | 15 +++ 7 files changed, 166 insertions(+), 114 deletions(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 75ef399c9a..a8ce4c7b9a 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -3102,7 +3102,8 @@ func TestEIP2718Transition(t *testing.T) { block := chain.GetBlockByNumber(1) // Expected gas is intrinsic + 2 * pc + hot load + cold load, since only one load is in the access list - expected := params.TxGas + params.TxAccessListAddressGas + params.TxAccessListStorageKeyGas + vm.GasQuickStep*2 + vm.WarmStorageReadCostEIP2929 + vm.ColdSloadCostEIP2929 + expected := params.TxGas + params.TxAccessListAddressGas + params.TxAccessListStorageKeyGas + + vm.GasQuickStep*2 + params.WarmStorageReadCostEIP2929 + params.ColdSloadCostEIP2929 if block.GasUsed() != expected { t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expected, block.GasUsed()) diff --git a/core/state_transition.go b/core/state_transition.go index cdffc100a1..05becd9a00 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -241,6 +241,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { sender := vm.AccountRef(msg.From()) homestead := st.evm.ChainConfig().IsHomestead(st.evm.Context.BlockNumber) istanbul := st.evm.ChainConfig().IsIstanbul(st.evm.Context.BlockNumber) + eip3529 := st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) contractCreation := msg.To() == nil // Check clauses 4-5, subtract intrinsic gas if everything is correct @@ -273,7 +274,13 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { st.state.SetNonce(msg.From(), st.state.GetNonce(sender.Address())+1) ret, st.gas, vmerr = st.evm.Call(sender, st.to(), st.data, st.gas, st.value) } - st.refundGas() + if !eip3529 { + // Before EIP-3529: refunds were capped to gasUsed / 2 + st.refundGas(params.RefundQuotient) + } else { + // After EIP-3529: refunds are capped to gasUsed / 5 + st.refundGas(params.RefundQuotientEIP3529) + } st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.gasPrice)) return &ExecutionResult{ @@ -283,9 +290,9 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { }, nil } -func (st *StateTransition) refundGas() { - // Apply refund counter, capped to half of the used gas. - refund := st.gasUsed() / 2 +func (st *StateTransition) refundGas(refundQuotient uint64) { + // Apply refund counter, capped to a refund quotient + refund := st.gasUsed() / refundQuotient if refund > st.state.GetRefund() { refund = st.state.GetRefund() } diff --git a/core/vm/eips.go b/core/vm/eips.go index 6bb941d5f9..025502760b 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -25,6 +25,7 @@ import ( ) var activators = map[int]func(*JumpTable){ + 3529: enable3529, 2929: enable2929, 2200: enable2200, 1884: enable1884, @@ -115,28 +116,28 @@ func enable2929(jt *JumpTable) { jt[SLOAD].constantGas = 0 jt[SLOAD].dynamicGas = gasSLoadEIP2929 - jt[EXTCODECOPY].constantGas = WarmStorageReadCostEIP2929 + jt[EXTCODECOPY].constantGas = params.WarmStorageReadCostEIP2929 jt[EXTCODECOPY].dynamicGas = gasExtCodeCopyEIP2929 - jt[EXTCODESIZE].constantGas = WarmStorageReadCostEIP2929 + jt[EXTCODESIZE].constantGas = params.WarmStorageReadCostEIP2929 jt[EXTCODESIZE].dynamicGas = gasEip2929AccountCheck - jt[EXTCODEHASH].constantGas = WarmStorageReadCostEIP2929 + jt[EXTCODEHASH].constantGas = params.WarmStorageReadCostEIP2929 jt[EXTCODEHASH].dynamicGas = gasEip2929AccountCheck - jt[BALANCE].constantGas = WarmStorageReadCostEIP2929 + jt[BALANCE].constantGas = params.WarmStorageReadCostEIP2929 jt[BALANCE].dynamicGas = gasEip2929AccountCheck - jt[CALL].constantGas = WarmStorageReadCostEIP2929 + jt[CALL].constantGas = params.WarmStorageReadCostEIP2929 jt[CALL].dynamicGas = gasCallEIP2929 - jt[CALLCODE].constantGas = WarmStorageReadCostEIP2929 + jt[CALLCODE].constantGas = params.WarmStorageReadCostEIP2929 jt[CALLCODE].dynamicGas = gasCallCodeEIP2929 - jt[STATICCALL].constantGas = WarmStorageReadCostEIP2929 + jt[STATICCALL].constantGas = params.WarmStorageReadCostEIP2929 jt[STATICCALL].dynamicGas = gasStaticCallEIP2929 - jt[DELEGATECALL].constantGas = WarmStorageReadCostEIP2929 + jt[DELEGATECALL].constantGas = params.WarmStorageReadCostEIP2929 jt[DELEGATECALL].dynamicGas = gasDelegateCallEIP2929 // This was previously part of the dynamic cost, but we're using it as a constantGas @@ -144,3 +145,12 @@ func enable2929(jt *JumpTable) { jt[SELFDESTRUCT].constantGas = params.SelfdestructGasEIP150 jt[SELFDESTRUCT].dynamicGas = gasSelfdestructEIP2929 } + +// enable3529 enabled "EIP-3529: Reduction in refunds": +// - Removes refunds for selfdestructs +// - Reduces refunds for SSTORE +// - Reduces max refunds to 20% gas +func enable3529(jt *JumpTable) { + jt[SSTORE].dynamicGas = gasSStoreEIP3529 + jt[SELFDESTRUCT].dynamicGas = gasSelfdestructEIP3529 +} diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 1022c355c1..8b755038fd 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -98,6 +98,8 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { if cfg.JumpTable[STOP] == nil { var jt JumpTable switch { + case evm.chainRules.IsLondon: + jt = londonInstructionSet case evm.chainRules.IsBerlin: jt = berlinInstructionSet case evm.chainRules.IsIstanbul: diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index bb1800ea91..a0609a0d78 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -57,11 +57,20 @@ var ( constantinopleInstructionSet = newConstantinopleInstructionSet() istanbulInstructionSet = newIstanbulInstructionSet() berlinInstructionSet = newBerlinInstructionSet() + londonInstructionSet = newLondonInstructionSet() ) // JumpTable contains the EVM opcodes supported at a given fork. type JumpTable [256]*operation +// newLondonInstructionSet returns the frontier, homestead, byzantium, +// contantinople, istanbul, petersburg, berlin and london instructions. +func newLondonInstructionSet() JumpTable { + instructionSet := newBerlinInstructionSet() + enable3529(&instructionSet) // EIP-3529: Reduction in refunds https://eips.ethereum.org/EIPS/eip-3529 + return instructionSet +} + // newBerlinInstructionSet returns the frontier, homestead, byzantium, // contantinople, istanbul, petersburg and berlin instructions. func newBerlinInstructionSet() JumpTable { diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index c56941899e..483226eefa 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -24,91 +24,75 @@ import ( "github.com/ethereum/go-ethereum/params" ) -const ( - ColdAccountAccessCostEIP2929 = uint64(2600) // COLD_ACCOUNT_ACCESS_COST - ColdSloadCostEIP2929 = uint64(2100) // COLD_SLOAD_COST - WarmStorageReadCostEIP2929 = uint64(100) // WARM_STORAGE_READ_COST -) - -// gasSStoreEIP2929 implements gas cost for SSTORE according to EIP-2929 -// -// When calling SSTORE, check if the (address, storage_key) pair is in accessed_storage_keys. -// If it is not, charge an additional COLD_SLOAD_COST gas, and add the pair to accessed_storage_keys. -// Additionally, modify the parameters defined in EIP 2200 as follows: -// -// Parameter Old value New value -// SLOAD_GAS 800 = WARM_STORAGE_READ_COST -// SSTORE_RESET_GAS 5000 5000 - COLD_SLOAD_COST -// -//The other parameters defined in EIP 2200 are unchanged. -// see gasSStoreEIP2200(...) in core/vm/gas_table.go for more info about how EIP 2200 is specified -func gasSStoreEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { - // If we fail the minimum gas availability invariant, fail (0) - if contract.Gas <= params.SstoreSentryGasEIP2200 { - return 0, errors.New("not enough gas for reentrancy sentry") - } - // Gas sentry honoured, do the actual gas calculation based on the stored value - var ( - y, x = stack.Back(1), stack.peek() - slot = common.Hash(x.Bytes32()) - current = evm.StateDB.GetState(contract.Address(), slot) - cost = uint64(0) - ) - // Check slot presence in the access list - if addrPresent, slotPresent := evm.StateDB.SlotInAccessList(contract.Address(), slot); !slotPresent { - cost = ColdSloadCostEIP2929 - // If the caller cannot afford the cost, this change will be rolled back - evm.StateDB.AddSlotToAccessList(contract.Address(), slot) - if !addrPresent { - // Once we're done with YOLOv2 and schedule this for mainnet, might - // be good to remove this panic here, which is just really a - // canary to have during testing - panic("impossible case: address was not present in access list during sstore op") +func makeGasSStoreFunc(clearingRefund uint64) gasFunc { + return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + // If we fail the minimum gas availability invariant, fail (0) + if contract.Gas <= params.SstoreSentryGasEIP2200 { + return 0, errors.New("not enough gas for reentrancy sentry") } - } - value := common.Hash(y.Bytes32()) + // Gas sentry honoured, do the actual gas calculation based on the stored value + var ( + y, x = stack.Back(1), stack.peek() + slot = common.Hash(x.Bytes32()) + current = evm.StateDB.GetState(contract.Address(), slot) + cost = uint64(0) + ) + // Check slot presence in the access list + if addrPresent, slotPresent := evm.StateDB.SlotInAccessList(contract.Address(), slot); !slotPresent { + cost = params.ColdSloadCostEIP2929 + // If the caller cannot afford the cost, this change will be rolled back + evm.StateDB.AddSlotToAccessList(contract.Address(), slot) + if !addrPresent { + // Once we're done with YOLOv2 and schedule this for mainnet, might + // be good to remove this panic here, which is just really a + // canary to have during testing + panic("impossible case: address was not present in access list during sstore op") + } + } + value := common.Hash(y.Bytes32()) - if current == value { // noop (1) - // EIP 2200 original clause: - // return params.SloadGasEIP2200, nil - return cost + WarmStorageReadCostEIP2929, nil // SLOAD_GAS - } - original := evm.StateDB.GetCommittedState(contract.Address(), x.Bytes32()) - if original == current { - if original == (common.Hash{}) { // create slot (2.1.1) - return cost + params.SstoreSetGasEIP2200, nil + if current == value { // noop (1) + // EIP 2200 original clause: + // return params.SloadGasEIP2200, nil + return cost + params.WarmStorageReadCostEIP2929, nil // SLOAD_GAS } - if value == (common.Hash{}) { // delete slot (2.1.2b) - evm.StateDB.AddRefund(params.SstoreClearsScheduleRefundEIP2200) + original := evm.StateDB.GetCommittedState(contract.Address(), x.Bytes32()) + if original == current { + if original == (common.Hash{}) { // create slot (2.1.1) + return cost + params.SstoreSetGasEIP2200, nil + } + if value == (common.Hash{}) { // delete slot (2.1.2b) + evm.StateDB.AddRefund(clearingRefund) + } + // EIP-2200 original clause: + // return params.SstoreResetGasEIP2200, nil // write existing slot (2.1.2) + return cost + (params.SstoreResetGasEIP2200 - params.ColdSloadCostEIP2929), nil // write existing slot (2.1.2) } - // EIP-2200 original clause: - // return params.SstoreResetGasEIP2200, nil // write existing slot (2.1.2) - return cost + (params.SstoreResetGasEIP2200 - ColdSloadCostEIP2929), nil // write existing slot (2.1.2) - } - if original != (common.Hash{}) { - if current == (common.Hash{}) { // recreate slot (2.2.1.1) - evm.StateDB.SubRefund(params.SstoreClearsScheduleRefundEIP2200) - } else if value == (common.Hash{}) { // delete slot (2.2.1.2) - evm.StateDB.AddRefund(params.SstoreClearsScheduleRefundEIP2200) + if original != (common.Hash{}) { + if current == (common.Hash{}) { // recreate slot (2.2.1.1) + evm.StateDB.SubRefund(clearingRefund) + } else if value == (common.Hash{}) { // delete slot (2.2.1.2) + evm.StateDB.AddRefund(clearingRefund) + } } - } - if original == value { - if original == (common.Hash{}) { // reset to original inexistent slot (2.2.2.1) - // EIP 2200 Original clause: - //evm.StateDB.AddRefund(params.SstoreSetGasEIP2200 - params.SloadGasEIP2200) - evm.StateDB.AddRefund(params.SstoreSetGasEIP2200 - WarmStorageReadCostEIP2929) - } else { // reset to original existing slot (2.2.2.2) - // EIP 2200 Original clause: - // evm.StateDB.AddRefund(params.SstoreResetGasEIP2200 - params.SloadGasEIP2200) - // - SSTORE_RESET_GAS redefined as (5000 - COLD_SLOAD_COST) - // - SLOAD_GAS redefined as WARM_STORAGE_READ_COST - // Final: (5000 - COLD_SLOAD_COST) - WARM_STORAGE_READ_COST - evm.StateDB.AddRefund((params.SstoreResetGasEIP2200 - ColdSloadCostEIP2929) - WarmStorageReadCostEIP2929) + if original == value { + if original == (common.Hash{}) { // reset to original inexistent slot (2.2.2.1) + // EIP 2200 Original clause: + //evm.StateDB.AddRefund(params.SstoreSetGasEIP2200 - params.SloadGasEIP2200) + evm.StateDB.AddRefund(params.SstoreSetGasEIP2200 - params.WarmStorageReadCostEIP2929) + } else { // reset to original existing slot (2.2.2.2) + // EIP 2200 Original clause: + // evm.StateDB.AddRefund(params.SstoreResetGasEIP2200 - params.SloadGasEIP2200) + // - SSTORE_RESET_GAS redefined as (5000 - COLD_SLOAD_COST) + // - SLOAD_GAS redefined as WARM_STORAGE_READ_COST + // Final: (5000 - COLD_SLOAD_COST) - WARM_STORAGE_READ_COST + evm.StateDB.AddRefund((params.SstoreResetGasEIP2200 - params.ColdSloadCostEIP2929) - params.WarmStorageReadCostEIP2929) + } } + // EIP-2200 original clause: + //return params.SloadGasEIP2200, nil // dirty update (2.2) + return cost + params.WarmStorageReadCostEIP2929, nil // dirty update (2.2) } - // EIP-2200 original clause: - //return params.SloadGasEIP2200, nil // dirty update (2.2) - return cost + WarmStorageReadCostEIP2929, nil // dirty update (2.2) } // gasSLoadEIP2929 calculates dynamic gas for SLOAD according to EIP-2929 @@ -124,9 +108,9 @@ func gasSLoadEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, me // If the caller cannot afford the cost, this change will be rolled back // If he does afford it, we can skip checking the same thing later on, during execution evm.StateDB.AddSlotToAccessList(contract.Address(), slot) - return ColdSloadCostEIP2929, nil + return params.ColdSloadCostEIP2929, nil } - return WarmStorageReadCostEIP2929, nil + return params.WarmStorageReadCostEIP2929, nil } // gasExtCodeCopyEIP2929 implements extcodecopy according to EIP-2929 @@ -146,7 +130,7 @@ func gasExtCodeCopyEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memo evm.StateDB.AddAddressToAccessList(addr) var overflow bool // We charge (cold-warm), since 'warm' is already charged as constantGas - if gas, overflow = math.SafeAdd(gas, ColdAccountAccessCostEIP2929-WarmStorageReadCostEIP2929); overflow { + if gas, overflow = math.SafeAdd(gas, params.ColdAccountAccessCostEIP2929-params.WarmStorageReadCostEIP2929); overflow { return 0, ErrGasUintOverflow } return gas, nil @@ -168,7 +152,7 @@ func gasEip2929AccountCheck(evm *EVM, contract *Contract, stack *Stack, mem *Mem // If the caller cannot afford the cost, this change will be rolled back evm.StateDB.AddAddressToAccessList(addr) // The warm storage read cost is already charged as constantGas - return ColdAccountAccessCostEIP2929 - WarmStorageReadCostEIP2929, nil + return params.ColdAccountAccessCostEIP2929 - params.WarmStorageReadCostEIP2929, nil } return 0, nil } @@ -180,7 +164,7 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { warmAccess := evm.StateDB.AddressInAccessList(addr) // The WarmStorageReadCostEIP2929 (100) is already deducted in the form of a constant cost, so // the cost to charge for cold access, if any, is Cold - Warm - coldCost := ColdAccountAccessCostEIP2929 - WarmStorageReadCostEIP2929 + coldCost := params.ColdAccountAccessCostEIP2929 - params.WarmStorageReadCostEIP2929 if !warmAccess { evm.StateDB.AddAddressToAccessList(addr) // Charge the remaining difference here already, to correctly calculate available @@ -212,25 +196,49 @@ var ( gasDelegateCallEIP2929 = makeCallVariantGasCallEIP2929(gasDelegateCall) gasStaticCallEIP2929 = makeCallVariantGasCallEIP2929(gasStaticCall) gasCallCodeEIP2929 = makeCallVariantGasCallEIP2929(gasCallCode) + gasSelfdestructEIP2929 = makeSelfdestructGasFn(true) + // gasSelfdestructEIP3529 implements the changes in EIP-2539 (no refunds) + gasSelfdestructEIP3529 = makeSelfdestructGasFn(false) + + // gasSStoreEIP2929 implements gas cost for SSTORE according to EIP-2929 + // + // When calling SSTORE, check if the (address, storage_key) pair is in accessed_storage_keys. + // If it is not, charge an additional COLD_SLOAD_COST gas, and add the pair to accessed_storage_keys. + // Additionally, modify the parameters defined in EIP 2200 as follows: + // + // Parameter Old value New value + // SLOAD_GAS 800 = WARM_STORAGE_READ_COST + // SSTORE_RESET_GAS 5000 5000 - COLD_SLOAD_COST + // + //The other parameters defined in EIP 2200 are unchanged. + // see gasSStoreEIP2200(...) in core/vm/gas_table.go for more info about how EIP 2200 is specified + gasSStoreEIP2929 = makeGasSStoreFunc(params.SstoreClearsScheduleRefundEIP2200) + + // gasSStoreEIP2539 implements gas cost for SSTORE according to EPI-2539 + // Replace `SSTORE_CLEARS_SCHEDULE` with `SSTORE_RESET_GAS + ACCESS_LIST_STORAGE_KEY_COST` (4,800) + gasSStoreEIP3529 = makeGasSStoreFunc(params.SstoreClearsScheduleRefundEIP3529) ) -func gasSelfdestructEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { - var ( - gas uint64 - address = common.Address(stack.peek().Bytes20()) - ) - if !evm.StateDB.AddressInAccessList(address) { - // If the caller cannot afford the cost, this change will be rolled back - evm.StateDB.AddAddressToAccessList(address) - gas = ColdAccountAccessCostEIP2929 - } - // if empty and transfers value - if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 { - gas += params.CreateBySelfdestructGas - } - if !evm.StateDB.HasSuicided(contract.Address()) { - evm.StateDB.AddRefund(params.SelfdestructRefundGas) +// makeSelfdestructGasFn can create the selfdestruct dynamic gas function for EIP-2929 and EIP-2539 +func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { + gasFunc := func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var ( + gas uint64 + address = common.Address(stack.peek().Bytes20()) + ) + if !evm.StateDB.AddressInAccessList(address) { + // If the caller cannot afford the cost, this change will be rolled back + evm.StateDB.AddAddressToAccessList(address) + gas = params.ColdAccountAccessCostEIP2929 + } + // if empty and transfers value + if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 { + gas += params.CreateBySelfdestructGas + } + if refundsEnabled && !evm.StateDB.HasSuicided(contract.Address()) { + evm.StateDB.AddRefund(params.SelfdestructRefundGas) + } + return gas, nil } - return gas, nil - + return gasFunc } diff --git a/params/protocol_params.go b/params/protocol_params.go index 88f1a06e12..22b4c0651c 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -57,6 +57,16 @@ const ( SstoreResetGasEIP2200 uint64 = 5000 // Once per SSTORE operation from clean non-zero to something else SstoreClearsScheduleRefundEIP2200 uint64 = 15000 // Once per SSTORE operation for clearing an originally existing storage slot + ColdAccountAccessCostEIP2929 = uint64(2600) // COLD_ACCOUNT_ACCESS_COST + ColdSloadCostEIP2929 = uint64(2100) // COLD_SLOAD_COST + WarmStorageReadCostEIP2929 = uint64(100) // WARM_STORAGE_READ_COST + + // In EIP-2200: SstoreResetGas was 5000. + // In EIP-2929: SstoreResetGas was changed to '5000 - COLD_SLOAD_COST'. + // In EIP-3529: SSTORE_CLEARS_SCHEDULE is defined as SSTORE_RESET_GAS + ACCESS_LIST_STORAGE_KEY_COST + // Which becomes: 5000 - 2100 + 1900 = 4800 + SstoreClearsScheduleRefundEIP3529 uint64 = SstoreResetGasEIP2200 - ColdSloadCostEIP2929 + TxAccessListStorageKeyGas + JumpdestGas uint64 = 1 // Once per JUMPDEST operation. EpochDuration uint64 = 30000 // Duration between proof-of-work epochs. @@ -137,6 +147,11 @@ const ( Bls12381PairingPerPairGas uint64 = 23000 // Per-point pair gas price for BLS12-381 elliptic curve pairing check Bls12381MapG1Gas uint64 = 5500 // Gas price for BLS12-381 mapping field element to G1 operation Bls12381MapG2Gas uint64 = 110000 // Gas price for BLS12-381 mapping field element to G2 operation + + // The Refund Quotient is the cap on how much of the used gas can be refunded. Before EIP-3529, + // up to half the consumed gas could be refunded. Redefined as 1/5th in EIP-3529 + RefundQuotient uint64 = 2 + RefundQuotientEIP3529 uint64 = 5 ) // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations From 8a070e8f7d45eca745c29d4523cc2e05ff2f117f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 7 May 2021 10:31:01 +0200 Subject: [PATCH 350/709] consensus/clique: add some missing checks (#22836) --- consensus/clique/clique.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index c62e180faa..9954a023e5 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -20,6 +20,7 @@ package clique import ( "bytes" "errors" + "fmt" "io" "math/big" "math/rand" @@ -293,6 +294,15 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H return errInvalidDifficulty } } + // Verify that the gas limit is <= 2^63-1 + cap := uint64(0x7fffffffffffffff) + if header.GasLimit > cap { + return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, cap) + } + // Verify that the gasUsed is <= gasLimit + if header.GasUsed > header.GasLimit { + return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) + } // If all checks passed, validate any special fields for hard forks if err := misc.VerifyForkHashes(chain.Config(), header, false); err != nil { return err @@ -324,6 +334,15 @@ func (c *Clique) verifyCascadingFields(chain consensus.ChainHeaderReader, header if parent.Time+c.config.Period > header.Time { return errInvalidTimestamp } + // Verify that the gas limit remains within allowed bounds + diff := int64(parent.GasLimit) - int64(header.GasLimit) + if diff < 0 { + diff *= -1 + } + limit := parent.GasLimit / params.GasLimitBoundDivisor + if uint64(diff) >= limit || header.GasLimit < params.MinGasLimit { + return fmt.Errorf("invalid gas limit: have %d, want %d += %d", header.GasLimit, parent.GasLimit, limit) + } // Retrieve the snapshot needed to verify this header and cache it snap, err := c.snapshot(chain, number-1, header.ParentHash, parents) if err != nil { From 17b1be26617268504a8ef3a6ee3c5c6400bfb63a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 7 May 2021 14:04:54 +0200 Subject: [PATCH 351/709] consensus/ethash: implement EIP-3554 (bomb delay) --- consensus/ethash/consensus.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 249c2f65eb..492fc83538 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -45,6 +45,11 @@ var ( maxUncles = 2 // Maximum number of uncles allowed in a single block allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks + // calcDifficultyEip3554 is the difficulty adjustment algorithm as specified by EIP 3554. + // It offsets the bomb a total of 9.5M blocks. + // Specification EIP-3554: https://eips.ethereum.org/EIPS/eip-3554 + calcDifficultyEip3554 = makeDifficultyCalculator(big.NewInt(9500000)) + // calcDifficultyEip2384 is the difficulty adjustment algorithm as specified by EIP 2384. // It offsets the bomb 4M blocks from Constantinople, so in total 9M blocks. // Specification EIP-2384: https://eips.ethereum.org/EIPS/eip-2384 @@ -325,6 +330,8 @@ func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Heade switch { case config.IsCatalyst(next): return big.NewInt(1) + case config.IsLondon(next): + return calcDifficultyEip3554(time, parent) case config.IsMuirGlacier(next): return calcDifficultyEip2384(time, parent) case config.IsConstantinople(next): From 700df1442d714cb3c42a602c39c042ce88be463f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 7 May 2021 14:37:13 +0200 Subject: [PATCH 352/709] rlp: add support for optional struct fields (#22832) This adds support for a new struct tag "optional". Using this tag, structs used for RLP encoding/decoding can be extended in a backwards-compatible way, by adding new fields at the end. --- rlp/decode.go | 18 ++++- rlp/decode_test.go | 181 +++++++++++++++++++++++++++++++++++++++++++-- rlp/doc.go | 61 +++++++++++---- rlp/encode.go | 39 ++++++++-- rlp/encode_test.go | 22 +++++- rlp/typecache.go | 54 +++++++++++--- 6 files changed, 330 insertions(+), 45 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index 79b7ef0626..b340aa029e 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -229,7 +229,7 @@ func decodeBigInt(s *Stream, val reflect.Value) error { i = new(big.Int) val.Set(reflect.ValueOf(i)) } - // Reject leading zero bytes + // Reject leading zero bytes. if len(b) > 0 && b[0] == 0 { return wrapStreamError(ErrCanonInt, val.Type()) } @@ -394,9 +394,16 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { if _, err := s.List(); err != nil { return wrapStreamError(err, typ) } - for _, f := range fields { + for i, f := range fields { err := f.info.decoder(s, val.Field(f.index)) if err == EOL { + if f.optional { + // The field is optional, so reaching the end of the list before + // reaching the last field is acceptable. All remaining undecoded + // fields are zeroed. + zeroFields(val, fields[i:]) + break + } return &decodeError{msg: "too few elements", typ: typ} } else if err != nil { return addErrorContext(err, "."+typ.Field(f.index).Name) @@ -407,6 +414,13 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { return dec, nil } +func zeroFields(structval reflect.Value, fields []field) { + for _, f := range fields { + fv := structval.Field(f.index) + fv.Set(reflect.Zero(fv.Type())) + } +} + // makePtrDecoder creates a decoder that decodes into the pointer's element type. func makePtrDecoder(typ reflect.Type, tag tags) (decoder, error) { etype := typ.Elem() diff --git a/rlp/decode_test.go b/rlp/decode_test.go index d94c3969b2..87a3306332 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -369,6 +369,39 @@ type intField struct { X int } +type optionalFields struct { + A uint + B uint `rlp:"optional"` + C uint `rlp:"optional"` +} + +type optionalAndTailField struct { + A uint + B uint `rlp:"optional"` + Tail []uint `rlp:"tail"` +} + +type optionalBigIntField struct { + A uint + B *big.Int `rlp:"optional"` +} + +type optionalPtrField struct { + A uint + B *[3]byte `rlp:"optional"` +} + +type optionalPtrFieldNil struct { + A uint + B *[3]byte `rlp:"optional,nil"` +} + +type ignoredField struct { + A uint + B uint `rlp:"-"` + C uint +} + var ( veryBigInt = big.NewInt(0).Add( big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), @@ -376,12 +409,6 @@ var ( ) ) -type hasIgnoredField struct { - A uint - B uint `rlp:"-"` - C uint -} - var decodeTests = []decodeTest{ // booleans {input: "01", ptr: new(bool), value: true}, @@ -551,8 +578,8 @@ var decodeTests = []decodeTest{ // struct tag "-" { input: "C20102", - ptr: new(hasIgnoredField), - value: hasIgnoredField{A: 1, C: 2}, + ptr: new(ignoredField), + value: ignoredField{A: 1, C: 2}, }, // struct tag "nilList" @@ -592,6 +619,110 @@ var decodeTests = []decodeTest{ value: nilStringSlice{X: &[]uint{3}}, }, + // struct tag "optional" + { + input: "C101", + ptr: new(optionalFields), + value: optionalFields{1, 0, 0}, + }, + { + input: "C20102", + ptr: new(optionalFields), + value: optionalFields{1, 2, 0}, + }, + { + input: "C3010203", + ptr: new(optionalFields), + value: optionalFields{1, 2, 3}, + }, + { + input: "C401020304", + ptr: new(optionalFields), + error: "rlp: input list has too many elements for rlp.optionalFields", + }, + { + input: "C101", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1}, + }, + { + input: "C20102", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{}}, + }, + { + input: "C401020304", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{3, 4}}, + }, + { + input: "C101", + ptr: new(optionalBigIntField), + value: optionalBigIntField{A: 1, B: nil}, + }, + { + input: "C20102", + ptr: new(optionalBigIntField), + value: optionalBigIntField{A: 1, B: big.NewInt(2)}, + }, + { + input: "C101", + ptr: new(optionalPtrField), + value: optionalPtrField{A: 1}, + }, + { + input: "C20180", // not accepted because "optional" doesn't enable "nil" + ptr: new(optionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrField).B", + }, + { + input: "C20102", + ptr: new(optionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrField).B", + }, + { + input: "C50183010203", + ptr: new(optionalPtrField), + value: optionalPtrField{A: 1, B: &[3]byte{1, 2, 3}}, + }, + { + input: "C101", + ptr: new(optionalPtrFieldNil), + value: optionalPtrFieldNil{A: 1}, + }, + { + input: "C20180", // accepted because "nil" tag allows empty input + ptr: new(optionalPtrFieldNil), + value: optionalPtrFieldNil{A: 1}, + }, + { + input: "C20102", + ptr: new(optionalPtrFieldNil), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrFieldNil).B", + }, + + // struct tag "optional" field clearing + { + input: "C101", + ptr: &optionalFields{A: 9, B: 8, C: 7}, + value: optionalFields{A: 1, B: 0, C: 0}, + }, + { + input: "C20102", + ptr: &optionalFields{A: 9, B: 8, C: 7}, + value: optionalFields{A: 1, B: 2, C: 0}, + }, + { + input: "C20102", + ptr: &optionalAndTailField{A: 9, B: 8, Tail: []uint{7, 6, 5}}, + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{}}, + }, + { + input: "C101", + ptr: &optionalPtrField{A: 9, B: &[3]byte{8, 7, 6}}, + value: optionalPtrField{A: 1}, + }, + // RawValue {input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))}, {input: "82FFFF", ptr: new(RawValue), value: RawValue(unhex("82FFFF"))}, @@ -822,6 +953,40 @@ func TestDecoderFunc(t *testing.T) { x() } +// This tests the validity checks for fields with struct tag "optional". +func TestInvalidOptionalField(t *testing.T) { + type ( + invalid1 struct { + A uint `rlp:"optional"` + B uint + } + invalid2 struct { + T []uint `rlp:"tail,optional"` + } + invalid3 struct { + T []uint `rlp:"optional,tail"` + } + ) + + tests := []struct { + v interface{} + err string + }{ + {v: new(invalid1), err: `rlp: struct field rlp.invalid1.B needs "optional" tag`}, + {v: new(invalid2), err: `rlp: invalid struct tag "optional" for rlp.invalid2.T (also has "tail" tag)`}, + {v: new(invalid3), err: `rlp: invalid struct tag "tail" for rlp.invalid3.T (also has "optional" tag)`}, + } + for _, test := range tests { + err := DecodeBytes(unhex("C20102"), test.v) + if err == nil { + t.Errorf("no error for %T", test.v) + } else if err.Error() != test.err { + t.Errorf("wrong error for %T: %v", test.v, err.Error()) + } + } + +} + func ExampleDecode() { input, _ := hex.DecodeString("C90A1486666F6F626172") diff --git a/rlp/doc.go b/rlp/doc.go index 7e6ee85200..113828e39b 100644 --- a/rlp/doc.go +++ b/rlp/doc.go @@ -102,29 +102,60 @@ Signed integers, floating point numbers, maps, channels and functions cannot be Struct Tags -Package rlp honours certain struct tags: "-", "tail", "nil", "nilList" and "nilString". +As with other encoding packages, the "-" tag ignores fields. -The "-" tag ignores fields. + type StructWithIgnoredField struct{ + Ignored uint `rlp:"-"` + Field uint + } + +Go struct values encode/decode as RLP lists. There are two ways of influencing the mapping +of fields to list elements. The "tail" tag, which may only be used on the last exported +struct field, allows slurping up any excess list elements into a slice. + + type StructWithTail struct{ + Field uint + Tail []string `rlp:"tail"` + } -The "tail" tag, which may only be used on the last exported struct field, allows slurping -up any excess list elements into a slice. See examples for more details. +The "optional" tag says that the field may be omitted if it is zero-valued. If this tag is +used on a struct field, all subsequent public fields must also be declared optional. -The "nil" tag applies to pointer-typed fields and changes the decoding rules for the field -such that input values of size zero decode as a nil pointer. This tag can be useful when -decoding recursive types. +When encoding a struct with optional fields, the output RLP list contains all values up to +the last non-zero optional field. - type StructWithOptionalFoo struct { - Foo *[20]byte `rlp:"nil"` +When decoding into a struct, optional fields may be omitted from the end of the input +list. For the example below, this means input lists of one, two, or three elements are +accepted. + + type StructWithOptionalFields struct{ + Required uint + Optional1 uint `rlp:"optional"` + Optional2 uint `rlp:"optional"` + } + +The "nil", "nilList" and "nilString" tags apply to pointer-typed fields only, and change +the decoding rules for the field type. For regular pointer fields without the "nil" tag, +input values must always match the required input length exactly and the decoder does not +produce nil values. When the "nil" tag is set, input values of size zero decode as a nil +pointer. This is especially useful for recursive types. + + type StructWithNilField struct { + Field *[3]byte `rlp:"nil"` } +In the example above, Field allows two possible input sizes. For input 0xC180 (a list +containing an empty string) Field is set to nil after decoding. For input 0xC483000000 (a +list containing a 3-byte string), Field is set to a non-nil array pointer. + RLP supports two kinds of empty values: empty lists and empty strings. When using the -"nil" tag, the kind of empty value allowed for a type is chosen automatically. A struct -field whose Go type is a pointer to an unsigned integer, string, boolean or byte -array/slice expects an empty RLP string. Any other pointer field type encodes/decodes as -an empty RLP list. +"nil" tag, the kind of empty value allowed for a type is chosen automatically. A field +whose Go type is a pointer to an unsigned integer, string, boolean or byte array/slice +expects an empty RLP string. Any other pointer field type encodes/decodes as an empty RLP +list. The choice of null value can be made explicit with the "nilList" and "nilString" struct -tags. Using these tags encodes/decodes a Go nil pointer value as the kind of empty -RLP value defined by the tag. +tags. Using these tags encodes/decodes a Go nil pointer value as the empty RLP value kind +defined by the tag. */ package rlp diff --git a/rlp/encode.go b/rlp/encode.go index 77b591045d..b7e74a133f 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -546,15 +546,40 @@ func makeStructWriter(typ reflect.Type) (writer, error) { return nil, structFieldError{typ, f.index, f.info.writerErr} } } - writer := func(val reflect.Value, w *encbuf) error { - lh := w.list() - for _, f := range fields { - if err := f.info.writer(val.Field(f.index), w); err != nil { - return err + + var writer writer + firstOptionalField := firstOptionalField(fields) + if firstOptionalField == len(fields) { + // This is the writer function for structs without any optional fields. + writer = func(val reflect.Value, w *encbuf) error { + lh := w.list() + for _, f := range fields { + if err := f.info.writer(val.Field(f.index), w); err != nil { + return err + } } + w.listEnd(lh) + return nil + } + } else { + // If there are any "optional" fields, the writer needs to perform additional + // checks to determine the output list length. + writer = func(val reflect.Value, w *encbuf) error { + lastField := len(fields) - 1 + for ; lastField >= firstOptionalField; lastField-- { + if !val.Field(fields[lastField].index).IsZero() { + break + } + } + lh := w.list() + for i := 0; i <= lastField; i++ { + if err := fields[i].info.writer(val.Field(fields[i].index), w); err != nil { + return err + } + } + w.listEnd(lh) + return nil } - w.listEnd(lh) - return nil } return writer, nil } diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 418ee10a35..74e8ededcb 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -257,12 +257,30 @@ var encTests = []encTest{ {val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"}, {val: &recstruct{5, nil}, output: "C205C0"}, {val: &recstruct{5, &recstruct{4, &recstruct{3, nil}}}, output: "C605C404C203C0"}, + {val: &intField{X: 3}, error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)"}, + + // struct tag "-" + {val: &ignoredField{A: 1, B: 2, C: 3}, output: "C20103"}, + + // struct tag "tail" {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, output: "C3010203"}, {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, output: "C20102"}, {val: &tailRaw{A: 1, Tail: []RawValue{}}, output: "C101"}, {val: &tailRaw{A: 1, Tail: nil}, output: "C101"}, - {val: &hasIgnoredField{A: 1, B: 2, C: 3}, output: "C20103"}, - {val: &intField{X: 3}, error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)"}, + + // struct tag "optional" + {val: &optionalFields{}, output: "C180"}, + {val: &optionalFields{A: 1}, output: "C101"}, + {val: &optionalFields{A: 1, B: 2}, output: "C20102"}, + {val: &optionalFields{A: 1, B: 2, C: 3}, output: "C3010203"}, + {val: &optionalFields{A: 1, B: 0, C: 3}, output: "C3018003"}, + {val: &optionalAndTailField{A: 1}, output: "C101"}, + {val: &optionalAndTailField{A: 1, B: 2}, output: "C20102"}, + {val: &optionalAndTailField{A: 1, Tail: []uint{5, 6}}, output: "C401800506"}, + {val: &optionalAndTailField{A: 1, Tail: []uint{5, 6}}, output: "C401800506"}, + {val: &optionalBigIntField{A: 1}, output: "C101"}, + {val: &optionalPtrField{A: 1}, output: "C101"}, + {val: &optionalPtrFieldNil{A: 1}, output: "C101"}, // nil {val: (*uint)(nil), output: "80"}, diff --git a/rlp/typecache.go b/rlp/typecache.go index 6026e1a649..3910dcf080 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -38,15 +38,16 @@ type typeinfo struct { // tags represents struct tags. type tags struct { // rlp:"nil" controls whether empty input results in a nil pointer. - nilOK bool - - // This controls whether nil pointers are encoded/decoded as empty strings - // or empty lists. + // nilKind is the kind of empty value allowed for the field. nilKind Kind + nilOK bool + + // rlp:"optional" allows for a field to be missing in the input list. + // If this is set, all subsequent fields must also be optional. + optional bool - // rlp:"tail" controls whether this field swallows additional list - // elements. It can only be set for the last field, which must be - // of slice type. + // rlp:"tail" controls whether this field swallows additional list elements. It can + // only be set for the last field, which must be of slice type. tail bool // rlp:"-" ignores fields. @@ -104,28 +105,51 @@ func cachedTypeInfo1(typ reflect.Type, tags tags) *typeinfo { } type field struct { - index int - info *typeinfo + index int + info *typeinfo + optional bool } +// structFields resolves the typeinfo of all public fields in a struct type. func structFields(typ reflect.Type) (fields []field, err error) { - lastPublic := lastPublicField(typ) + var ( + lastPublic = lastPublicField(typ) + anyOptional = false + ) for i := 0; i < typ.NumField(); i++ { if f := typ.Field(i); f.PkgPath == "" { // exported tags, err := parseStructTag(typ, i, lastPublic) if err != nil { return nil, err } + + // Skip rlp:"-" fields. if tags.ignored { continue } + // If any field has the "optional" tag, subsequent fields must also have it. + if tags.optional || tags.tail { + anyOptional = true + } else if anyOptional { + return nil, fmt.Errorf(`rlp: struct field %v.%s needs "optional" tag`, typ, f.Name) + } info := cachedTypeInfo1(f.Type, tags) - fields = append(fields, field{i, info}) + fields = append(fields, field{i, info, tags.optional}) } } return fields, nil } +// anyOptionalFields returns the index of the first field with "optional" tag. +func firstOptionalField(fields []field) int { + for i, f := range fields { + if f.optional { + return i + } + } + return len(fields) +} + type structFieldError struct { typ reflect.Type field int @@ -166,11 +190,19 @@ func parseStructTag(typ reflect.Type, fi, lastPublic int) (tags, error) { case "nilList": ts.nilKind = List } + case "optional": + ts.optional = true + if ts.tail { + return ts, structTagError{typ, f.Name, t, `also has "tail" tag`} + } case "tail": ts.tail = true if fi != lastPublic { return ts, structTagError{typ, f.Name, t, "must be on last field"} } + if ts.optional { + return ts, structTagError{typ, f.Name, t, `also has "optional" tag`} + } if f.Type.Kind() != reflect.Slice { return ts, structTagError{typ, f.Name, t, "field type is not slice"} } From 7ab7acfded9d121e39ef0dc99ba427211693e4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Mon, 10 May 2021 12:18:42 +0200 Subject: [PATCH 353/709] build: upgrade -dlgo version to Go 1.16.4 (#22848) --- build/checksums.txt | 26 +++++++++++++------------- build/ci.go | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 0e33e07f2c..e667b30ce6 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,18 +1,18 @@ # This file contains sha256 checksums of optional build dependencies. -b298d29de9236ca47a023e382313bcc2d2eed31dfa706b60a04103ce83a71a25 go1.16.3.src.tar.gz -6bb1cf421f8abc2a9a4e39140b7397cdae6aca3e8d36dcff39a1a77f4f1170ac go1.16.3.darwin-amd64.tar.gz -f4e96bbcd5d2d1942f5b55d9e4ab19564da4fad192012f6d7b0b9b055ba4208f go1.16.3.darwin-arm64.tar.gz -48b2d1481db756c88c18b1f064dbfc3e265ce4a775a23177ca17e25d13a24c5d go1.16.3.linux-386.tar.gz -951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2 go1.16.3.linux-amd64.tar.gz -566b1d6f17d2bc4ad5f81486f0df44f3088c3ed47a3bec4099d8ed9939e90d5d go1.16.3.linux-arm64.tar.gz -0dae30385e3564a557dac7f12a63eedc73543e6da0f6017990e214ce8cc8797c go1.16.3.linux-armv6l.tar.gz -a3c16e1531bf9726f47911c4a9ed7cb665a6207a51c44f10ebad4db63b4bcc5a go1.16.3.windows-386.zip -a4400345135b36cb7942e52bbaf978b66814738b855eeff8de879a09fd99de7f go1.16.3.windows-amd64.zip -31ecd11d497684fa8b0f01ba784590c4c760943665fdc4fe0adaa1405c71736c go1.16.3.freebsd-386.tar.gz -ffbd920b309e62e807457b11d80e8c17fefe3ef6de423aaba4b1e270b2ca4c3d go1.16.3.freebsd-amd64.tar.gz -5eb046bbbbc7fe2591846a4303884cb5a01abb903e3e61e33459affe7874e811 go1.16.3.linux-ppc64le.tar.gz -3e8bd7bde533a73fd6fa75b5288678ef397e76c198cfb26b8ae086035383b1cf go1.16.3.linux-s390x.tar.gz +ae4f6b6e2a1677d31817984655a762074b5356da50fb58722b99104870d43503 go1.16.4.src.tar.gz +18fe94775763db3878717393b6d41371b0b45206055e49b3838328120c977d13 go1.16.4.darwin-amd64.tar.gz +cb6b972cc42e669f3585c648198cd5b6f6d7a0811d413ad64b50c02ba06ccc3a go1.16.4.darwin-arm64.tar.gz +cd1b146ef6e9006f27dd99e9687773e7fef30e8c985b7d41bff33e955a3bb53a go1.16.4.linux-386.tar.gz +7154e88f5a8047aad4b80ebace58a059e36e7e2e4eb3b383127a28c711b4ff59 go1.16.4.linux-amd64.tar.gz +8b18eb05ddda2652d69ab1b1dd1f40dd731799f43c6a58b512ad01ae5b5bba21 go1.16.4.linux-arm64.tar.gz +a53391a800ddec749ee90d38992babb27b95cfb864027350c737b9aa8e069494 go1.16.4.linux-armv6l.tar.gz +e75c0b114a09eb5499874162b208931dc260de0fedaeedac8621bf263c974605 go1.16.4.windows-386.zip +d40139b7ade8a3008e3240a6f86fe8f899a9c465c917e11dac8758af216f5eb0 go1.16.4.windows-amd64.zip +7cf2bc8a175d6d656861165bfc554f92dc78d2abf5afe5631db3579555d97409 go1.16.4.freebsd-386.tar.gz +ccdd2b76de1941b60734408fda0d750aaa69330d8a07430eed4c56bdb3502f6f go1.16.4.freebsd-amd64.tar.gz +80cfac566e344096a8df8f37bbd21f89e76a6fbe601406565d71a87a665fc125 go1.16.4.linux-ppc64le.tar.gz +d6431881b3573dc29ecc24fbeab5e5ec25d8c9273aa543769c86a1a3bbac1ddf go1.16.4.linux-s390x.tar.gz 7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c golangci-lint-1.39.0-darwin-amd64.tar.gz 574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064 golangci-lint-1.39.0-darwin-arm64.tar.gz diff --git a/build/ci.go b/build/ci.go index d9f147ef0e..3752b1bed9 100644 --- a/build/ci.go +++ b/build/ci.go @@ -152,7 +152,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.16.3" + dlgoVersion = "1.16.4" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From f19a679b09500e63b2fd134a4e829591174d9e48 Mon Sep 17 00:00:00 2001 From: Ceelog Date: Mon, 10 May 2021 18:19:32 +0800 Subject: [PATCH 354/709] cmd/geth: remove reference to monitor command (#22844) 'geth monitor' subcommand is no longer supported. --- cmd/geth/consolecmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 5c715a1250..85eabb527b 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -171,7 +171,7 @@ func remoteConsole(ctx *cli.Context) error { // dialRPC returns a RPC client which connects to the given endpoint. // The check for empty endpoint implements the defaulting logic -// for "geth attach" and "geth monitor" with no argument. +// for "geth attach" with no argument. func dialRPC(endpoint string) (*rpc.Client, error) { if endpoint == "" { endpoint = node.DefaultIPCEndpoint(clientIdentifier) From ae5fcdc67fbfa208de257487cb86271b429545df Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 10 May 2021 12:29:33 +0200 Subject: [PATCH 355/709] go.mod: upgrade to github.com/holiman/uint256 v1.2.0 (#22745) --- consensus/ethash/difficulty.go | 6 ++---- core/vm/instructions_test.go | 4 ++-- core/vm/logger_test.go | 4 ++-- go.mod | 2 +- go.sum | 2 ++ 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/consensus/ethash/difficulty.go b/consensus/ethash/difficulty.go index 59c4ac7419..66a18059c6 100644 --- a/consensus/ethash/difficulty.go +++ b/consensus/ethash/difficulty.go @@ -52,8 +52,7 @@ func CalcDifficultyFrontierU256(time uint64, parent *types.Header) *big.Int { - num = block.number */ - pDiff := uint256.NewInt() - pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff + pDiff, _ := uint256.FromBig(parent.Difficulty) // pDiff: pdiff adjust := pDiff.Clone() adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048 @@ -96,8 +95,7 @@ func CalcDifficultyHomesteadU256(time uint64, parent *types.Header) *big.Int { - num = block.number */ - pDiff := uint256.NewInt() - pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff + pDiff, _ := uint256.FromBig(parent.Difficulty) // pDiff: pdiff adjust := pDiff.Clone() adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048 diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index d17ccfab89..c881ba0aca 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -568,11 +568,11 @@ func BenchmarkOpSHA3(bench *testing.B) { env.interpreter = evmInterpreter mem.Resize(32) pc := uint64(0) - start := uint256.NewInt() + start := new(uint256.Int) bench.ResetTimer() for i := 0; i < bench.N; i++ { - stack.pushN(*uint256.NewInt().SetUint64(32), *start) + stack.pushN(*uint256.NewInt(32), *start) opSha3(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) } } diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 9c936af36a..0bbfd44692 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -60,8 +60,8 @@ func TestStoreCapture(t *testing.T) { Contract: contract, } ) - scope.Stack.push(uint256.NewInt().SetUint64(1)) - scope.Stack.push(uint256.NewInt()) + scope.Stack.push(uint256.NewInt(1)) + scope.Stack.push(new(uint256.Int)) var index common.Hash logger.CaptureState(env, 0, SSTORE, 0, 0, scope, nil, 0, nil) if len(logger.storage[contract.Address()]) == 0 { diff --git a/go.mod b/go.mod index 512d541f41..9a2e66e101 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 - github.com/holiman/uint256 v1.1.1 + github.com/holiman/uint256 v1.2.0 github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 github.com/influxdata/influxdb v1.8.3 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 diff --git a/go.sum b/go.sum index b6a27a2cf1..4cc4dbf20b 100644 --- a/go.sum +++ b/go.sum @@ -212,6 +212,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88 h1:bcAj8KroPf552TScjFPIakjH2/tdIrIH8F+cc4v4SRo= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= From c0e201b69039b50a28ec7f8beee79306c9844cff Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 10 May 2021 12:38:54 +0200 Subject: [PATCH 356/709] eth/protocols/eth, les: avoid Raw() when decoding HashOrNumber (#22841) Getting the raw value is not necessary to decode this type, and decoding it directly from the stream is faster. --- eth/protocols/eth/protocol.go | 24 ++++++++++++------------ les/protocol.go | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 62c018ef8e..de1b0ed1ee 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -155,19 +155,19 @@ func (hn *HashOrNumber) EncodeRLP(w io.Writer) error { // DecodeRLP is a specialized decoder for HashOrNumber to decode the contents // into either a block hash or a block number. func (hn *HashOrNumber) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - origin, err := s.Raw() - if err == nil { - switch { - case size == 32: - err = rlp.DecodeBytes(origin, &hn.Hash) - case size <= 8: - err = rlp.DecodeBytes(origin, &hn.Number) - default: - err = fmt.Errorf("invalid input size %d for origin", size) - } + _, size, err := s.Kind() + switch { + case err != nil: + return err + case size == 32: + hn.Number = 0 + return s.Decode(&hn.Hash) + case size <= 8: + hn.Hash = common.Hash{} + return s.Decode(&hn.Number) + default: + return fmt.Errorf("invalid input size %d for origin", size) } - return err } // BlockHeadersPacket represents a block header response. diff --git a/les/protocol.go b/les/protocol.go index 07a4452f40..06db9024eb 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -307,19 +307,19 @@ func (hn *hashOrNumber) EncodeRLP(w io.Writer) error { // DecodeRLP is a specialized decoder for hashOrNumber to decode the contents // into either a block hash or a block number. func (hn *hashOrNumber) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - origin, err := s.Raw() - if err == nil { - switch { - case size == 32: - err = rlp.DecodeBytes(origin, &hn.Hash) - case size <= 8: - err = rlp.DecodeBytes(origin, &hn.Number) - default: - err = fmt.Errorf("invalid input size %d for origin", size) - } + _, size, err := s.Kind() + switch { + case err != nil: + return err + case size == 32: + hn.Number = 0 + return s.Decode(&hn.Hash) + case size <= 8: + hn.Hash = common.Hash{} + return s.Decode(&hn.Number) + default: + return fmt.Errorf("invalid input size %d for origin", size) } - return err } // CodeData is the network response packet for a node data retrieval. From e536bb52ff87cfffe9b413d02dfcf80e7f265e5a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 10 May 2021 13:35:07 +0200 Subject: [PATCH 357/709] eth/protocols/snap: adapt to uint256 API changes (#22851) --- eth/protocols/snap/range.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/eth/protocols/snap/range.go b/eth/protocols/snap/range.go index dd380ff471..2627cb954b 100644 --- a/eth/protocols/snap/range.go +++ b/eth/protocols/snap/range.go @@ -42,15 +42,15 @@ func newHashRange(start common.Hash, num uint64) *hashRange { step256.SetFromBig(step) return &hashRange{ - current: uint256.NewInt().SetBytes32(start[:]), + current: new(uint256.Int).SetBytes32(start[:]), step: step256, } } // Next pushes the hash range to the next interval. func (r *hashRange) Next() bool { - next := new(uint256.Int) - if overflow := next.AddOverflow(r.current, r.step); overflow { + next, overflow := new(uint256.Int).AddOverflow(r.current, r.step) + if overflow { return false } r.current = next @@ -65,16 +65,17 @@ func (r *hashRange) Start() common.Hash { // End returns the last hash in the current interval. func (r *hashRange) End() common.Hash { // If the end overflows (non divisible range), return a shorter interval - next := new(uint256.Int) - if overflow := next.AddOverflow(r.current, r.step); overflow { + next, overflow := new(uint256.Int).AddOverflow(r.current, r.step) + if overflow { return common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") } - return new(uint256.Int).Sub(next, uint256.NewInt().SetOne()).Bytes32() + return next.SubUint64(next, 1).Bytes32() } // incHash returns the next hash, in lexicographical order (a.k.a plus one) func incHash(h common.Hash) common.Hash { - a := uint256.NewInt().SetBytes32(h[:]) - a.Add(a, uint256.NewInt().SetOne()) + var a uint256.Int + a.SetBytes32(h[:]) + a.AddUint64(&a, 1) return common.Hash(a.Bytes32()) } From 643fd0efc6ebc8397311d1ca68fc71d8da247643 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 11 May 2021 10:43:35 +0200 Subject: [PATCH 358/709] core/types: remove support for legacy receipt/log storage encoding (#22852) * core/types: remove support for legacy receipt storage encoding * core/types: remove support for legacy log storage encoding --- core/types/log.go | 39 ++---------- core/types/receipt.go | 89 +-------------------------- core/types/receipt_test.go | 123 ------------------------------------- 3 files changed, 7 insertions(+), 244 deletions(-) diff --git a/core/types/log.go b/core/types/log.go index 88274e39da..87865bdb25 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -71,18 +71,6 @@ type rlpLog struct { // rlpStorageLog is the storage encoding of a log. type rlpStorageLog rlpLog -// legacyRlpStorageLog is the previous storage encoding of a log including some redundant fields. -type legacyRlpStorageLog struct { - Address common.Address - Topics []common.Hash - Data []byte - BlockNumber uint64 - TxHash common.Hash - TxIndex uint - BlockHash common.Hash - Index uint -} - // EncodeRLP implements rlp.Encoder. func (l *Log) EncodeRLP(w io.Writer) error { return rlp.Encode(w, rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}) @@ -115,29 +103,10 @@ func (l *LogForStorage) EncodeRLP(w io.Writer) error { // // Note some redundant fields(e.g. block number, tx hash etc) will be assembled later. func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { - blob, err := s.Raw() - if err != nil { - return err - } var dec rlpStorageLog - err = rlp.DecodeBytes(blob, &dec) - if err == nil { - *l = LogForStorage{ - Address: dec.Address, - Topics: dec.Topics, - Data: dec.Data, - } - } else { - // Try to decode log with previous definition. - var dec legacyRlpStorageLog - err = rlp.DecodeBytes(blob, &dec) - if err == nil { - *l = LogForStorage{ - Address: dec.Address, - Topics: dec.Topics, - Data: dec.Data, - } - } + if err := s.Decode(&dec); err != nil { + return err } - return err + *l = LogForStorage{Address: dec.Address, Topics: dec.Topics, Data: dec.Data} + return nil } diff --git a/core/types/receipt.go b/core/types/receipt.go index e04259b9d8..6b519a79d2 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -97,27 +97,6 @@ type storedReceiptRLP struct { Logs []*LogForStorage } -// v4StoredReceiptRLP is the storage encoding of a receipt used in database version 4. -type v4StoredReceiptRLP struct { - PostStateOrStatus []byte - CumulativeGasUsed uint64 - TxHash common.Hash - ContractAddress common.Address - Logs []*LogForStorage - GasUsed uint64 -} - -// v3StoredReceiptRLP is the original storage encoding of a receipt including some unnecessary fields. -type v3StoredReceiptRLP struct { - PostStateOrStatus []byte - CumulativeGasUsed uint64 - Bloom Bloom - TxHash common.Hash - ContractAddress common.Address - Logs []*LogForStorage - GasUsed uint64 -} - // NewReceipt creates a barebone transaction receipt, copying the init fields. // Deprecated: create receipts using a struct literal instead. func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { @@ -237,8 +216,7 @@ func (r *Receipt) Size() common.StorageSize { // entire content of a receipt, as opposed to only the consensus fields originally. type ReceiptForStorage Receipt -// EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt -// into an RLP stream. +// EncodeRLP implements rlp.Encoder. func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { enc := &storedReceiptRLP{ PostStateOrStatus: (*Receipt)(r).statusEncoding(), @@ -251,82 +229,21 @@ func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { return rlp.Encode(w, enc) } -// DecodeRLP implements rlp.Decoder, and loads both consensus and implementation -// fields of a receipt from an RLP stream. +// DecodeRLP implements rlp.Decoder. func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error { - // Retrieve the entire receipt blob as we need to try multiple decoders - blob, err := s.Raw() - if err != nil { - return err - } - // Try decoding from the newest format for future proofness, then the older one - // for old nodes that just upgraded. V4 was an intermediate unreleased format so - // we do need to decode it, but it's not common (try last). - if err := decodeStoredReceiptRLP(r, blob); err == nil { - return nil - } - if err := decodeV3StoredReceiptRLP(r, blob); err == nil { - return nil - } - return decodeV4StoredReceiptRLP(r, blob) -} - -func decodeStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { var stored storedReceiptRLP - if err := rlp.DecodeBytes(blob, &stored); err != nil { - return err - } - if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { - return err - } - r.CumulativeGasUsed = stored.CumulativeGasUsed - r.Logs = make([]*Log, len(stored.Logs)) - for i, log := range stored.Logs { - r.Logs[i] = (*Log)(log) - } - r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) - - return nil -} - -func decodeV4StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { - var stored v4StoredReceiptRLP - if err := rlp.DecodeBytes(blob, &stored); err != nil { + if err := s.Decode(&stored); err != nil { return err } if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { return err } r.CumulativeGasUsed = stored.CumulativeGasUsed - r.TxHash = stored.TxHash - r.ContractAddress = stored.ContractAddress - r.GasUsed = stored.GasUsed r.Logs = make([]*Log, len(stored.Logs)) for i, log := range stored.Logs { r.Logs[i] = (*Log)(log) } r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) - - return nil -} - -func decodeV3StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { - var stored v3StoredReceiptRLP - if err := rlp.DecodeBytes(blob, &stored); err != nil { - return err - } - if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { - return err - } - r.CumulativeGasUsed = stored.CumulativeGasUsed - r.Bloom = stored.Bloom - r.TxHash = stored.TxHash - r.ContractAddress = stored.ContractAddress - r.GasUsed = stored.GasUsed - r.Logs = make([]*Log, len(stored.Logs)) - for i, log := range stored.Logs { - r.Logs[i] = (*Log)(log) - } return nil } diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 22a316c237..87fc16a510 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -20,7 +20,6 @@ import ( "bytes" "math" "math/big" - "reflect" "testing" "github.com/ethereum/go-ethereum/common" @@ -38,128 +37,6 @@ func TestDecodeEmptyTypedReceipt(t *testing.T) { } } -func TestLegacyReceiptDecoding(t *testing.T) { - tests := []struct { - name string - encode func(*Receipt) ([]byte, error) - }{ - { - "StoredReceiptRLP", - encodeAsStoredReceiptRLP, - }, - { - "V4StoredReceiptRLP", - encodeAsV4StoredReceiptRLP, - }, - { - "V3StoredReceiptRLP", - encodeAsV3StoredReceiptRLP, - }, - } - - tx := NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(1), 1, big.NewInt(1), nil) - receipt := &Receipt{ - Status: ReceiptStatusFailed, - CumulativeGasUsed: 1, - Logs: []*Log{ - { - Address: common.BytesToAddress([]byte{0x11}), - Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, - Data: []byte{0x01, 0x00, 0xff}, - }, - { - Address: common.BytesToAddress([]byte{0x01, 0x11}), - Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, - Data: []byte{0x01, 0x00, 0xff}, - }, - }, - TxHash: tx.Hash(), - ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), - GasUsed: 111111, - } - receipt.Bloom = CreateBloom(Receipts{receipt}) - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - enc, err := tc.encode(receipt) - if err != nil { - t.Fatalf("Error encoding receipt: %v", err) - } - var dec ReceiptForStorage - if err := rlp.DecodeBytes(enc, &dec); err != nil { - t.Fatalf("Error decoding RLP receipt: %v", err) - } - // Check whether all consensus fields are correct. - if dec.Status != receipt.Status { - t.Fatalf("Receipt status mismatch, want %v, have %v", receipt.Status, dec.Status) - } - if dec.CumulativeGasUsed != receipt.CumulativeGasUsed { - t.Fatalf("Receipt CumulativeGasUsed mismatch, want %v, have %v", receipt.CumulativeGasUsed, dec.CumulativeGasUsed) - } - if dec.Bloom != receipt.Bloom { - t.Fatalf("Bloom data mismatch, want %v, have %v", receipt.Bloom, dec.Bloom) - } - if len(dec.Logs) != len(receipt.Logs) { - t.Fatalf("Receipt log number mismatch, want %v, have %v", len(receipt.Logs), len(dec.Logs)) - } - for i := 0; i < len(dec.Logs); i++ { - if dec.Logs[i].Address != receipt.Logs[i].Address { - t.Fatalf("Receipt log %d address mismatch, want %v, have %v", i, receipt.Logs[i].Address, dec.Logs[i].Address) - } - if !reflect.DeepEqual(dec.Logs[i].Topics, receipt.Logs[i].Topics) { - t.Fatalf("Receipt log %d topics mismatch, want %v, have %v", i, receipt.Logs[i].Topics, dec.Logs[i].Topics) - } - if !bytes.Equal(dec.Logs[i].Data, receipt.Logs[i].Data) { - t.Fatalf("Receipt log %d data mismatch, want %v, have %v", i, receipt.Logs[i].Data, dec.Logs[i].Data) - } - } - }) - } -} - -func encodeAsStoredReceiptRLP(want *Receipt) ([]byte, error) { - stored := &storedReceiptRLP{ - PostStateOrStatus: want.statusEncoding(), - CumulativeGasUsed: want.CumulativeGasUsed, - Logs: make([]*LogForStorage, len(want.Logs)), - } - for i, log := range want.Logs { - stored.Logs[i] = (*LogForStorage)(log) - } - return rlp.EncodeToBytes(stored) -} - -func encodeAsV4StoredReceiptRLP(want *Receipt) ([]byte, error) { - stored := &v4StoredReceiptRLP{ - PostStateOrStatus: want.statusEncoding(), - CumulativeGasUsed: want.CumulativeGasUsed, - TxHash: want.TxHash, - ContractAddress: want.ContractAddress, - Logs: make([]*LogForStorage, len(want.Logs)), - GasUsed: want.GasUsed, - } - for i, log := range want.Logs { - stored.Logs[i] = (*LogForStorage)(log) - } - return rlp.EncodeToBytes(stored) -} - -func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) { - stored := &v3StoredReceiptRLP{ - PostStateOrStatus: want.statusEncoding(), - CumulativeGasUsed: want.CumulativeGasUsed, - Bloom: want.Bloom, - TxHash: want.TxHash, - ContractAddress: want.ContractAddress, - Logs: make([]*LogForStorage, len(want.Logs)), - GasUsed: want.GasUsed, - } - for i, log := range want.Logs { - stored.Logs[i] = (*LogForStorage)(log) - } - return rlp.EncodeToBytes(stored) -} - // Tests that receipt data can be correctly derived from the contextual infos func TestDeriveFields(t *testing.T) { // Create a few transactions to have receipts for From ca9808079844c55961cad6ddd9df04fbe21ff78c Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Tue, 11 May 2021 02:25:51 -0700 Subject: [PATCH 359/709] cmd/geth, eth/gasprice: add configurable threshold to gas price oracle (#22752) This adds a cmd line parameter `--gpo.ignoreprice`, to make the gas price oracle ignore transactions below the given threshold. --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 8 ++++++++ eth/ethconfig/config.go | 14 ++++++++------ eth/gasprice/gasprice.go | 39 +++++++++++++++++++++++++-------------- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index bed4d8a488..04274d75f7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -147,6 +147,7 @@ var ( utils.GpoBlocksFlag, utils.GpoPercentileFlag, utils.GpoMaxGasPriceFlag, + utils.GpoIgnoreGasPriceFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, utils.MinerNotifyFullFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index aa0a4b2901..117f6dc02b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -196,6 +196,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.GpoBlocksFlag, utils.GpoPercentileFlag, utils.GpoMaxGasPriceFlag, + utils.GpoIgnoreGasPriceFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index aa00f96c92..5c0bba4655 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -687,6 +687,11 @@ var ( Usage: "Maximum gas price will be recommended by gpo", Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), } + GpoIgnoreGasPriceFlag = cli.Int64Flag{ + Name: "gpo.ignoreprice", + Usage: "Gas price below which gpo will ignore transactions", + Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(), + } // Metrics flags MetricsEnabledFlag = cli.BoolFlag{ @@ -1296,6 +1301,9 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) { if ctx.GlobalIsSet(GpoMaxGasPriceFlag.Name) { cfg.MaxPrice = big.NewInt(ctx.GlobalInt64(GpoMaxGasPriceFlag.Name)) } + if ctx.GlobalIsSet(GpoIgnoreGasPriceFlag.Name) { + cfg.IgnorePrice = big.NewInt(ctx.GlobalInt64(GpoIgnoreGasPriceFlag.Name)) + } } func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f5629e6a39..f297019ba5 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -41,16 +41,18 @@ import ( // FullNodeGPO contains default gasprice oracle settings for full node. var FullNodeGPO = gasprice.Config{ - Blocks: 20, - Percentile: 60, - MaxPrice: gasprice.DefaultMaxPrice, + Blocks: 20, + Percentile: 60, + MaxPrice: gasprice.DefaultMaxPrice, + IgnorePrice: gasprice.DefaultIgnorePrice, } // LightClientGPO contains default gasprice oracle settings for light client. var LightClientGPO = gasprice.Config{ - Blocks: 2, - Percentile: 60, - MaxPrice: gasprice.DefaultMaxPrice, + Blocks: 2, + Percentile: 60, + MaxPrice: gasprice.DefaultMaxPrice, + IgnorePrice: gasprice.DefaultIgnorePrice, } // Defaults contains default settings for use on the Ethereum main net. diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 560722bec0..566c954ecc 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -32,12 +32,14 @@ import ( const sampleNumber = 3 // Number of transactions sampled in a block var DefaultMaxPrice = big.NewInt(500 * params.GWei) +var DefaultIgnorePrice = big.NewInt(2 * params.Wei) type Config struct { - Blocks int - Percentile int - Default *big.Int `toml:",omitempty"` - MaxPrice *big.Int `toml:",omitempty"` + Blocks int + Percentile int + Default *big.Int `toml:",omitempty"` + MaxPrice *big.Int `toml:",omitempty"` + IgnorePrice *big.Int `toml:",omitempty"` } // OracleBackend includes all necessary background APIs for oracle. @@ -50,12 +52,13 @@ type OracleBackend interface { // Oracle recommends gas prices based on the content of recent // blocks. Suitable for both light and full clients. type Oracle struct { - backend OracleBackend - lastHead common.Hash - lastPrice *big.Int - maxPrice *big.Int - cacheLock sync.RWMutex - fetchLock sync.Mutex + backend OracleBackend + lastHead common.Hash + lastPrice *big.Int + maxPrice *big.Int + ignorePrice *big.Int + cacheLock sync.RWMutex + fetchLock sync.Mutex checkBlocks int percentile int @@ -83,10 +86,18 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { maxPrice = DefaultMaxPrice log.Warn("Sanitizing invalid gasprice oracle price cap", "provided", params.MaxPrice, "updated", maxPrice) } + ignorePrice := params.IgnorePrice + if ignorePrice == nil || ignorePrice.Int64() <= 0 { + ignorePrice = DefaultIgnorePrice + log.Warn("Sanitizing invalid gasprice oracle ignore price", "provided", params.IgnorePrice, "updated", ignorePrice) + } else if ignorePrice.Int64() > 0 { + log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice) + } return &Oracle{ backend: backend, lastPrice: params.Default, maxPrice: maxPrice, + ignorePrice: ignorePrice, checkBlocks: blocks, percentile: percent, } @@ -123,7 +134,7 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) { txPrices []*big.Int ) for sent < gpo.checkBlocks && number > 0 { - go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, result, quit) + go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, gpo.ignorePrice, result, quit) sent++ exp++ number-- @@ -146,7 +157,7 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) { // meaningful returned, try to query more blocks. But the maximum // is 2*checkBlocks. if len(res.prices) == 1 && len(txPrices)+1+exp < gpo.checkBlocks*2 && number > 0 { - go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, result, quit) + go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, gpo.ignorePrice, result, quit) sent++ exp++ number-- @@ -183,7 +194,7 @@ func (t transactionsByGasPrice) Less(i, j int) bool { return t[i].GasPriceCmp(t[ // and sends it to the result channel. If the block is empty or all transactions // are sent by the miner itself(it doesn't make any sense to include this kind of // transaction prices for sampling), nil gasprice is returned. -func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, blockNum uint64, limit int, result chan getBlockPricesResult, quit chan struct{}) { +func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, blockNum uint64, limit int, ignoreUnder *big.Int, result chan getBlockPricesResult, quit chan struct{}) { block, err := gpo.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNum)) if block == nil { select { @@ -199,7 +210,7 @@ func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, bloc var prices []*big.Int for _, tx := range txs { - if tx.GasPriceIntCmp(common.Big1) <= 0 { + if ignoreUnder != nil && tx.GasPriceIntCmp(ignoreUnder) == -1 { continue } sender, err := types.Sender(signer, tx) From 0524cede37508068b3031912989493e10eb0544a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 May 2021 16:23:54 +0300 Subject: [PATCH 360/709] eth/tracers: do the JSON serialization via .js to capture C faults --- eth/tracers/tracer.go | 24 +++++++++++++++++++----- eth/tracers/tracer_test.go | 14 +++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index ba65925373..218903dd92 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -505,7 +505,7 @@ func (jst *Tracer) Stop(err error) { // call executes a method on a JS object, catching any errors, formatting and // returning them as error objects. -func (jst *Tracer) call(method string, args ...string) (json.RawMessage, error) { +func (jst *Tracer) call(noret bool, method string, args ...string) (json.RawMessage, error) { // Execute the JavaScript call and return any error jst.vm.PushString(method) for _, arg := range args { @@ -519,7 +519,21 @@ func (jst *Tracer) call(method string, args ...string) (json.RawMessage, error) return nil, errors.New(err) } // No error occurred, extract return value and return - return json.RawMessage(jst.vm.JsonEncode(-1)), nil + if noret { + return nil, nil + } + // Push a JSON marshaller onto the stack. We can't marshal from the out- + // side because duktape can crash on large nestings and we can't catch + // C++ exceptions ourselves from Go. TODO(karalabe): Yuck, why wrap?! + jst.vm.PushString("(JSON.stringify)") + jst.vm.Eval() + + jst.vm.Swap(-1, -2) + if code = jst.vm.Pcall(1); code != 0 { + err := jst.vm.SafeToString(-1) + return nil, errors.New(err) + } + return json.RawMessage(jst.vm.SafeToString(-1)), nil } func wrapError(context string, err error) error { @@ -578,7 +592,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost *jst.errorValue = err.Error() } - if _, err := jst.call("step", "log", "db"); err != nil { + if _, err := jst.call(true, "step", "log", "db"); err != nil { jst.err = wrapError("step", err) } } @@ -592,7 +606,7 @@ func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost jst.errorValue = new(string) *jst.errorValue = err.Error() - if _, err := jst.call("fault", "log", "db"); err != nil { + if _, err := jst.call(true, "fault", "log", "db"); err != nil { jst.err = wrapError("fault", err) } } @@ -640,7 +654,7 @@ func (jst *Tracer) GetResult() (json.RawMessage, error) { jst.vm.PutPropString(jst.stateObject, "ctx") // Finalize the trace and return the results - result, err := jst.call("result", "ctx", "db") + result, err := jst.call(false, "result", "ctx", "db") if err != nil { jst.err = wrapError("result", err) } diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 7cda2e5330..033824474d 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -78,7 +78,7 @@ func runTrace(tracer *Tracer, vmctx *vmContext) (json.RawMessage, error) { } func TestTracer(t *testing.T) { - execTracer := func(code string) []byte { + execTracer := func(code string) ([]byte, string) { t.Helper() ctx := &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} tracer, err := New(code, ctx.txCtx) @@ -87,13 +87,14 @@ func TestTracer(t *testing.T) { } ret, err := runTrace(tracer, ctx) if err != nil { - t.Fatal(err) + return nil, err.Error() // Stringify to allow comparison without nil checks } - return ret + return ret, "" } for i, tt := range []struct { code string want string + fail string }{ { // tests that we don't panic on bad arguments to memory access code: "{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}", @@ -116,10 +117,13 @@ func TestTracer(t *testing.T) { }, { // tests intrinsic gas code: "{depths: [], step: function() {}, fault: function() {}, result: function(ctx) { return ctx.gasPrice+'.'+ctx.gasUsed+'.'+ctx.intrinsicGas; }}", want: `"100000.6.21000"`, + }, { // tests too deep object / serialization crash + code: "{step: function() {}, fault: function() {}, result: function() { var o={}; var x=o; for (var i=0; i<1000; i++){ o.foo={}; o=o.foo; } return x; }}", + fail: "RangeError: json encode recursion limit in server-side tracer function 'result'", }, } { - if have := execTracer(tt.code); tt.want != string(have) { - t.Errorf("testcase %d: expected return value to be %s got %s\n\tcode: %v", i, tt.want, string(have), tt.code) + if have, err := execTracer(tt.code); tt.want != string(have) || tt.fail != err { + t.Errorf("testcase %d: expected return value to be '%s' got '%s', error to be '%s' got '%s'\n\tcode: %v", i, tt.want, string(have), tt.fail, err, tt.code) } } } From a2c456a526ae9c07944f12913c6830add93a553f Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 11 May 2021 17:12:10 +0200 Subject: [PATCH 361/709] core: ensure a broken trie invariant crashes genesis creation (#22780) * Ensure state could be created in ToBlock * Fix rebase errors * use a panic instead --- core/genesis.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/genesis.go b/core/genesis.go index 9ae718beb6..d7d08e0909 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -259,7 +259,10 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { db = rawdb.NewMemoryDatabase() } - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db), nil) + statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) + if err != nil { + panic(err) + } for addr, account := range g.Alloc { statedb.AddBalance(addr, account.Balance) statedb.SetCode(addr, account.Code) From addd8824cf3ad6133c1b1bbc3387a621eafba6a3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 12 May 2021 10:05:39 +0200 Subject: [PATCH 362/709] cmd/geth, eth, core: snapshot dump + unify with trie dump (#22795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cmd/geth, eth, core: snapshot dump + unify with trie dump * cmd/evm: dump API fixes * cmd/geth, core, eth: fix some remaining errors * cmd/evm: dump - add limit, support address startkey, address review concerns * cmd, core/state, eth: minor polishes, fix snap dump crash, unify format Co-authored-by: Péter Szilágyi --- cmd/evm/internal/t8ntool/transition.go | 7 +- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 10 +-- cmd/geth/chaincmd.go | 112 +++++++++++++++++-------- cmd/geth/snapshot.go | 97 +++++++++++++++++++++ cmd/utils/flags.go | 12 ++- core/state/dump.go | 80 ++++++++++++------ core/state/state_test.go | 19 +++-- eth/api.go | 19 ++++- eth/api_test.go | 23 +++-- 10 files changed, 293 insertions(+), 88 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index fedcd12435..22cd0dd851 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -212,16 +212,15 @@ func Main(ctx *cli.Context) error { // Iterate over all the tests, run them and aggregate the results // Run the test and aggregate the result - state, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) + s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { return err } body, _ := rlp.EncodeToBytes(txs) // Dump the excution result collector := make(Alloc) - state.DumpToCollector(collector, false, false, false, nil, -1) + s.DumpToCollector(collector, nil) return dispatchOutput(ctx, baseDir, result, collector, body) - } // txWithKey is a helper-struct, to allow us to use the types.Transaction along with @@ -303,7 +302,7 @@ func (g Alloc) OnAccount(addr common.Address, dumpAccount state.DumpAccount) { } } genesisAccount := core.GenesisAccount{ - Code: common.FromHex(dumpAccount.Code), + Code: dumpAccount.Code, Storage: storage, Balance: balance, Nonce: dumpAccount.Nonce, diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 4063767cb8..2d890ef1a2 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -270,7 +270,7 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump(false, false, true))) + fmt.Println(string(statedb.Dump(nil))) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index c4df936c75..d8bc4eae80 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -98,16 +98,16 @@ func stateTestCmd(ctx *cli.Context) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - _, state, err := test.Run(st, cfg, false) + _, s, err := test.Run(st, cfg, false) // print state root for evmlab tracing - if ctx.GlobalBool(MachineFlag.Name) && state != nil { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + if ctx.GlobalBool(MachineFlag.Name) && s != nil { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false)) } if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() - if ctx.GlobalBool(DumpFlag.Name) && state != nil { - dump := state.RawDump(false, false, true) + if ctx.GlobalBool(DumpFlag.Name) && s != nil { + dump := s.RawDump(nil) result.State = &dump } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d00b4bc1f6..b9bd88e213 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -18,6 +18,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "runtime" @@ -27,12 +28,16 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/node" "gopkg.in/urfave/cli.v1" ) @@ -152,20 +157,21 @@ The export-preimages command export hash preimages to an RLP encoded stream`, Action: utils.MigrateFlags(dump), Name: "dump", Usage: "Dump a specific block from storage", - ArgsUsage: "[ | ]...", + ArgsUsage: "[? | ]", Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.SyncModeFlag, utils.IterativeOutputFlag, utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.IncludeIncompletesFlag, + utils.StartKeyFlag, + utils.DumpLimitFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` -The arguments are interpreted as block numbers or hashes. -Use "ethereum dump 0" to dump the genesis block.`, +This command dumps out the state for a given block (or latest, if none provided). +`, } ) @@ -373,47 +379,85 @@ func exportPreimages(ctx *cli.Context) error { return nil } -func dump(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - +func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { db := utils.MakeChainDatabase(ctx, stack, true) - for _, arg := range ctx.Args() { - var header *types.Header + var header *types.Header + if ctx.NArg() > 1 { + return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) + } + if ctx.NArg() == 1 { + arg := ctx.Args().First() if hashish(arg) { hash := common.HexToHash(arg) - number := rawdb.ReadHeaderNumber(db, hash) - if number != nil { + if number := rawdb.ReadHeaderNumber(db, hash); number != nil { header = rawdb.ReadHeader(db, hash, *number) + } else { + return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { - number, _ := strconv.Atoi(arg) - hash := rawdb.ReadCanonicalHash(db, uint64(number)) - if hash != (common.Hash{}) { - header = rawdb.ReadHeader(db, hash, uint64(number)) - } - } - if header == nil { - fmt.Println("{}") - utils.Fatalf("block not found") - } else { - state, err := state.New(header.Root, state.NewDatabase(db), nil) + number, err := strconv.Atoi(arg) if err != nil { - utils.Fatalf("could not create new state: %v", err) + return nil, nil, common.Hash{}, err } - excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name) - excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name) - includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name) - if ctx.Bool(utils.IterativeOutputFlag.Name) { - state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout)) + if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, uint64(number)) } else { - if includeMissing { - fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" + - " otherwise the accounts will overwrite each other in the resulting mapping.") - } - fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false)) + return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } } + } else { + // Use latest + header = rawdb.ReadHeadHeader(db) + } + if header == nil { + return nil, nil, common.Hash{}, errors.New("no head block found") + } + startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) + var start common.Hash + switch len(startArg) { + case 0: // common.Hash + case 32: + start = common.BytesToHash(startArg) + case 20: + start = crypto.Keccak256Hash(startArg) + log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex()) + default: + return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) + } + var conf = &state.DumpConfig{ + SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), + SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name), + Start: start.Bytes(), + Max: ctx.Uint64(utils.DumpLimitFlag.Name), + } + log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), + "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, + "start", hexutil.Encode(conf.Start), "limit", conf.Max) + return conf, db, header.Root, nil +} + +func dump(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + conf, db, root, err := parseDumpConfig(ctx, stack) + if err != nil { + return err + } + state, err := state.New(root, state.NewDatabase(db), nil) + if err != nil { + return err + } + if ctx.Bool(utils.IterativeOutputFlag.Name) { + state.IterativeDump(conf, json.NewEncoder(os.Stdout)) + } else { + if conf.OnlyWithAddresses { + fmt.Fprintf(os.Stderr, "If you want to include accounts with missing preimages, you need iterative output, since"+ + " otherwise the accounts will overwrite each other in the resulting mapping.") + return fmt.Errorf("incompatible options") + } + fmt.Println(string(state.Dump(conf))) } return nil } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 1af458af20..35d027fb16 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,7 +18,9 @@ package main import ( "bytes" + "encoding/json" "errors" + "os" "time" "github.com/ethereum/go-ethereum/cmd/utils" @@ -142,6 +144,31 @@ verification. The default checking target is the HEAD state. It's basically iden to traverse-state, but the check granularity is smaller. It's also usable without snapshot enabled. +`, + }, + { + Name: "dump", + Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", + ArgsUsage: "[? | ]", + Action: utils.MigrateFlags(dumpState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.ExcludeCodeFlag, + utils.ExcludeStorageFlag, + utils.StartKeyFlag, + utils.DumpLimitFlag, + }, + Description: ` +This command is semantically equivalent to 'geth dump', but uses the snapshots +as the backend data source, making this command a lot faster. + +The argument is interpreted as block number or hash. If none is provided, the latest +block is used. `, }, }, @@ -430,3 +457,73 @@ func parseRoot(input string) (common.Hash, error) { } return h, nil } + +func dumpState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + conf, db, root, err := parseDumpConfig(ctx, stack) + if err != nil { + return err + } + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false) + if err != nil { + return err + } + accIt, err := snaptree.AccountIterator(root, common.BytesToHash(conf.Start)) + if err != nil { + return err + } + defer accIt.Release() + + log.Info("Snapshot dumping started", "root", root) + var ( + start = time.Now() + logged = time.Now() + accounts uint64 + ) + enc := json.NewEncoder(os.Stdout) + enc.Encode(struct { + Root common.Hash `json:"root"` + }{root}) + for accIt.Next() { + account, err := snapshot.FullAccount(accIt.Account()) + if err != nil { + return err + } + da := &state.DumpAccount{ + Balance: account.Balance.String(), + Nonce: account.Nonce, + Root: account.Root, + CodeHash: account.CodeHash, + SecureKey: accIt.Hash().Bytes(), + } + if !conf.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { + da.Code = rawdb.ReadCode(db, common.BytesToHash(account.CodeHash)) + } + if !conf.SkipStorage { + da.Storage = make(map[common.Hash]string) + + stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) + if err != nil { + return err + } + for stIt.Next() { + da.Storage[stIt.Hash()] = common.Bytes2Hex(stIt.Slot()) + } + } + enc.Encode(da) + accounts++ + if time.Since(logged) > 8*time.Second { + log.Info("Snapshot dumping in progress", "at", accIt.Hash(), "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + if conf.Max > 0 && accounts >= conf.Max { + break + } + } + log.Info("Snapshot dumping complete", "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5c0bba4655..d3fb3f2cbd 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -184,7 +184,7 @@ var ( Name: "exitwhensynced", Usage: "Exits after block synchronisation completes", } - IterativeOutputFlag = cli.BoolFlag{ + IterativeOutputFlag = cli.BoolTFlag{ Name: "iterative", Usage: "Print streaming JSON iteratively, delimited by newlines", } @@ -200,6 +200,16 @@ var ( Name: "nocode", Usage: "Exclude contract code (save db lookups)", } + StartKeyFlag = cli.StringFlag{ + Name: "start", + Usage: "Start position. Either a hash or address", + Value: "0x0000000000000000000000000000000000000000000000000000000000000000", + } + DumpLimitFlag = cli.Uint64Flag{ + Name: "limit", + Usage: "Max number of elements (0 = no limit)", + Value: 0, + } defaultSyncMode = ethconfig.Defaults.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", diff --git a/core/state/dump.go b/core/state/dump.go index b25da714fd..00faa4ed6a 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -19,6 +19,7 @@ package state import ( "encoding/json" "fmt" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -27,6 +28,16 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// DumpConfig is a set of options to control what portions of the statewill be +// iterated and collected. +type DumpConfig struct { + SkipCode bool + SkipStorage bool + OnlyWithAddresses bool + Start []byte + Max uint64 +} + // DumpCollector interface which the state trie calls during iteration type DumpCollector interface { // OnRoot is called with the state root @@ -39,9 +50,9 @@ type DumpCollector interface { type DumpAccount struct { Balance string `json:"balance"` Nonce uint64 `json:"nonce"` - Root string `json:"root"` - CodeHash string `json:"codeHash"` - Code string `json:"code,omitempty"` + Root hexutil.Bytes `json:"root"` + CodeHash hexutil.Bytes `json:"codeHash"` + Code hexutil.Bytes `json:"code,omitempty"` Storage map[common.Hash]string `json:"storage,omitempty"` Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key @@ -111,38 +122,50 @@ func (d iterativeDump) OnRoot(root common.Hash) { }{root}) } -func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) { - missingPreimages := 0 +// DumpToCollector iterates the state according to the given options and inserts +// the items into a collector for aggregation or serialization. +func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) { + // Sanitize the input to allow nil configs + if conf == nil { + conf = new(DumpConfig) + } + var ( + missingPreimages int + accounts uint64 + start = time.Now() + logged = time.Now() + ) + log.Info("Trie dumping started", "root", s.trie.Hash()) c.OnRoot(s.trie.Hash()) - var count int - it := trie.NewIterator(s.trie.NodeIterator(start)) + it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) for it.Next() { var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } account := DumpAccount{ - Balance: data.Balance.String(), - Nonce: data.Nonce, - Root: common.Bytes2Hex(data.Root[:]), - CodeHash: common.Bytes2Hex(data.CodeHash), + Balance: data.Balance.String(), + Nonce: data.Nonce, + Root: data.Root[:], + CodeHash: data.CodeHash, + SecureKey: it.Key, } addrBytes := s.trie.GetKey(it.Key) if addrBytes == nil { // Preimage missing missingPreimages++ - if excludeMissingPreimages { + if conf.OnlyWithAddresses { continue } account.SecureKey = it.Key } addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) - if !excludeCode { - account.Code = common.Bytes2Hex(obj.Code(s.db)) + if !conf.SkipCode { + account.Code = obj.Code(s.db) } - if !excludeStorage { + if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) for storageIt.Next() { @@ -155,8 +178,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, } } c.OnAccount(addr, account) - count++ - if maxResults > 0 && count >= maxResults { + accounts++ + if time.Since(logged) > 8*time.Second { + log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + if conf.Max > 0 && accounts >= conf.Max { if it.Next() { nextKey = it.Key } @@ -166,22 +194,24 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } + log.Info("Trie dumping complete", "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) return nextKey } // RawDump returns the entire state an a single large object -func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump { +func (s *StateDB) RawDump(opts *DumpConfig) Dump { dump := &Dump{ Accounts: make(map[common.Address]DumpAccount), } - s.DumpToCollector(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) + s.DumpToCollector(dump, opts) return *dump } // Dump returns a JSON string representing the entire state as a single json-object -func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte { - dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages) +func (s *StateDB) Dump(opts *DumpConfig) []byte { + dump := s.RawDump(opts) json, err := json.MarshalIndent(dump, "", " ") if err != nil { fmt.Println("Dump err", err) @@ -190,15 +220,15 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool } // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout -func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { - s.DumpToCollector(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) +func (s *StateDB) IterativeDump(opts *DumpConfig, output *json.Encoder) { + s.DumpToCollector(iterativeDump{output}, opts) } // IteratorDump dumps out a batch of accounts starts with the given start key -func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump { +func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump { iterator := &IteratorDump{ Accounts: make(map[common.Address]DumpAccount), } - iterator.Next = s.DumpToCollector(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults) + iterator.Next = s.DumpToCollector(iterator, opts) return *iterator } diff --git a/core/state/state_test.go b/core/state/state_test.go index 9566531466..0a55d7781f 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -57,28 +57,31 @@ func TestDump(t *testing.T) { s.state.Commit(false) // check that DumpToCollector contains the state objects that are in trie - got := string(s.state.Dump(false, false, true)) + got := string(s.state.Dump(nil)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { "0x0000000000000000000000000000000000000001": { "balance": "22", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "key": "0x1468288056310c82aa4c01a7e12a10f8111a0560e72b700555479031b86c357d" }, "0x0000000000000000000000000000000000000002": { "balance": "44", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "key": "0xd52688a8f926c816ca1e079067caba944f158e764817b83fc43594370ca9cf62" }, "0x0000000000000000000000000000000000000102": { "balance": "0", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", - "code": "03030303030303" + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0x87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", + "code": "0x03030303030303", + "key": "0xa17eacbc25cda025e81db9c5c62868822c73ce097cee2a63e33a2e41268358a1" } } }` diff --git a/eth/api.go b/eth/api.go index 7387459c94..6a22c9e416 100644 --- a/eth/api.go +++ b/eth/api.go @@ -264,12 +264,16 @@ func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { // DumpBlock retrieves the entire state of the database at a given block. func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { + opts := &state.DumpConfig{ + OnlyWithAddresses: true, + Max: AccountRangeMaxResults, // Sanity limit over RPC + } if blockNr == rpc.PendingBlockNumber { // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those _, stateDb := api.eth.miner.Pending() - return stateDb.RawDump(false, false, true), nil + return stateDb.RawDump(opts), nil } var block *types.Block if blockNr == rpc.LatestBlockNumber { @@ -284,7 +288,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error if err != nil { return state.Dump{}, err } - return stateDb.RawDump(false, false, true), nil + return stateDb.RawDump(opts), nil } // PrivateDebugAPI is the collection of Ethereum full node APIs exposed over @@ -386,10 +390,17 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta return state.IteratorDump{}, errors.New("either block number or block hash must be specified") } + opts := &state.DumpConfig{ + SkipCode: nocode, + SkipStorage: nostorage, + OnlyWithAddresses: !incompletes, + Start: start, + Max: uint64(maxResults), + } if maxResults > AccountRangeMaxResults || maxResults <= 0 { - maxResults = AccountRangeMaxResults + opts.Max = AccountRangeMaxResults } - return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil + return stateDb.IteratorDump(opts), nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. diff --git a/eth/api_test.go b/eth/api_test.go index b44eed40bc..39a1d58460 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -34,7 +34,13 @@ import ( var dumper = spew.ConfigState{Indent: " "} func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { - result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum) + result := statedb.IteratorDump(&state.DumpConfig{ + SkipCode: true, + SkipStorage: true, + OnlyWithAddresses: false, + Start: start.Bytes(), + Max: uint64(requestedNum), + }) if len(result.Accounts) != expectedNum { t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts)) @@ -131,12 +137,17 @@ func TestEmptyAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) - state, _ = state.New(common.Hash{}, statedb, nil) + statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) + st, _ = state.New(common.Hash{}, statedb, nil) ) - state.Commit(true) - state.IntermediateRoot(true) - results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults) + st.Commit(true) + st.IntermediateRoot(true) + results := st.IteratorDump(&state.DumpConfig{ + SkipCode: true, + SkipStorage: true, + OnlyWithAddresses: true, + Max: uint64(AccountRangeMaxResults), + }) if bytes.Equal(results.Next, (common.Hash{}).Bytes()) { t.Fatalf("Empty results should not return a second page") } From 597ecb39cc963eb3b85b01b759c75c45e71f41d0 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Mon, 17 May 2021 00:52:32 -0600 Subject: [PATCH 363/709] cmd/evm: return json error if unmarshalling from stdin fails (#22871) * cmd/evm: return json error if unmarshalling from stdin fails * cmd/evm: make error capitalizations uniform (all lowercase starts) * cmd/evm: capitalize error sent directly to stderror --- cmd/evm/internal/t8ntool/transition.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 22cd0dd851..9bb03c2c6a 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -142,7 +142,9 @@ func Main(ctx *cli.Context) error { // Figure out the prestate alloc if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) - decoder.Decode(inputData) + if err := decoder.Decode(inputData); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } } if allocStr != stdinSelector { inFile, err := os.Open(allocStr) @@ -152,7 +154,7 @@ func Main(ctx *cli.Context) error { defer inFile.Close() decoder := json.NewDecoder(inFile) if err := decoder.Decode(&inputData.Alloc); err != nil { - return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling alloc-file: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling alloc-file: %v", err)) } } prestate.Pre = inputData.Alloc @@ -167,7 +169,7 @@ func Main(ctx *cli.Context) error { decoder := json.NewDecoder(inFile) var env stEnv if err := decoder.Decode(&env); err != nil { - return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling env-file: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling env-file: %v", err)) } inputData.Env = &env } @@ -180,7 +182,7 @@ func Main(ctx *cli.Context) error { // Construct the chainconfig var chainConfig *params.ChainConfig if cConf, extraEips, err := tests.GetChainConfig(ctx.String(ForknameFlag.Name)); err != nil { - return NewError(ErrorVMConfig, fmt.Errorf("Failed constructing chain configuration: %v", err)) + return NewError(ErrorVMConfig, fmt.Errorf("failed constructing chain configuration: %v", err)) } else { chainConfig = cConf vmConfig.ExtraEips = extraEips @@ -197,7 +199,7 @@ func Main(ctx *cli.Context) error { defer inFile.Close() decoder := json.NewDecoder(inFile) if err := decoder.Decode(&txsWithKeys); err != nil { - return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling txs-file: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) } } else { txsWithKeys = inputData.Txs @@ -206,7 +208,7 @@ func Main(ctx *cli.Context) error { signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number))) if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil { - return NewError(ErrorJson, fmt.Errorf("Failed signing transactions: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed signing transactions: %v", err)) } // Iterate over all the tests, run them and aggregate the results @@ -277,7 +279,7 @@ func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Tran // This transaction needs to be signed signed, err := types.SignTx(tx, signer, key) if err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("Tx %d: failed to sign tx: %v", i, err)) + return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err)) } signedTxs = append(signedTxs, signed) } else { From 14bc6e5130ebd291026da82f9ad9684bde895479 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 17 May 2021 10:49:23 +0200 Subject: [PATCH 364/709] consensus/ethash: change eip3554 from 9.5M to 9.7M (#22870) --- consensus/ethash/consensus.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 492fc83538..b04cb24fb4 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -46,9 +46,9 @@ var ( allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks // calcDifficultyEip3554 is the difficulty adjustment algorithm as specified by EIP 3554. - // It offsets the bomb a total of 9.5M blocks. + // It offsets the bomb a total of 9.7M blocks. // Specification EIP-3554: https://eips.ethereum.org/EIPS/eip-3554 - calcDifficultyEip3554 = makeDifficultyCalculator(big.NewInt(9500000)) + calcDifficultyEip3554 = makeDifficultyCalculator(big.NewInt(9700000)) // calcDifficultyEip2384 is the difficulty adjustment algorithm as specified by EIP 2384. // It offsets the bomb 4M blocks from Constantinople, so in total 9M blocks. From 94451c2788295901c302c9bf5fa2f7b021c924e2 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 17 May 2021 15:13:22 +0200 Subject: [PATCH 365/709] all: implement EIP-1559 (#22837) This is the initial implementation of EIP-1559 in packages core/types and core. Mining, RPC, etc. will be added in subsequent commits. Co-authored-by: Marius van der Wijden Co-authored-by: lightclient@protonmail.com Co-authored-by: Felix Lange --- accounts/abi/bind/backends/simulated.go | 2 + accounts/external/backend.go | 12 +- cmd/evm/internal/t8ntool/execution.go | 8 +- cmd/evm/internal/t8ntool/gen_stenv.go | 22 ++- cmd/evm/internal/t8ntool/transition.go | 10 +- cmd/evm/poststate.json | 23 --- cmd/evm/testdata/10/alloc.json | 23 +++ cmd/evm/testdata/10/env.json | 12 ++ cmd/evm/testdata/10/readme.md | 79 +++++++++ cmd/evm/testdata/10/txs.json | 70 ++++++++ cmd/evm/testdata/11/alloc.json | 25 +++ cmd/evm/testdata/11/env.json | 12 ++ cmd/evm/testdata/11/readme.md | 13 ++ cmd/evm/testdata/11/txs.json | 14 ++ cmd/evm/testdata/9/alloc.json | 11 ++ cmd/evm/testdata/9/env.json | 8 + cmd/evm/testdata/9/readme.md | 75 +++++++++ cmd/evm/testdata/9/txs.json | 37 +++++ consensus/clique/clique.go | 34 ++-- consensus/ethash/consensus.go | 30 ++-- consensus/misc/eip1559.go | 93 +++++++++++ consensus/misc/eip1559_test.go | 133 ++++++++++++++++ consensus/misc/gaslimit.go | 42 +++++ core/blockchain_test.go | 157 ++++++++++++++++++ core/chain_makers.go | 8 +- core/error.go | 4 + core/evm.go | 10 +- core/genesis.go | 3 + core/state_prefetcher.go | 2 +- core/state_processor.go | 6 +- core/state_processor_test.go | 203 +++++++++++++++++------- core/state_transition.go | 22 ++- core/types/access_list_tx.go | 3 +- core/types/block.go | 14 ++ core/types/block_test.go | 65 ++++++++ core/types/dynamic_fee_tx.go | 104 ++++++++++++ core/types/legacy_tx.go | 3 +- core/types/receipt.go | 9 +- core/types/transaction.go | 31 +++- core/types/transaction_marshalling.go | 104 ++++++++++-- core/types/transaction_signing.go | 77 ++++++++- core/vm/eips.go | 20 +++ core/vm/evm.go | 1 + core/vm/jump_table.go | 1 + core/vm/opcodes.go | 3 + eth/state_accessor.go | 2 +- eth/tracers/api.go | 8 +- eth/tracers/api_test.go | 2 +- eth/tracers/tracers_test.go | 4 +- interfaces.go | 3 + internal/ethapi/api.go | 6 +- les/odr_test.go | 4 +- les/state_accessor.go | 2 +- light/odr_test.go | 2 +- params/bootnodes.go | 3 +- params/config.go | 2 +- params/protocol_params.go | 4 + tests/init.go | 13 ++ tests/state_test_util.go | 2 +- 59 files changed, 1522 insertions(+), 173 deletions(-) delete mode 100644 cmd/evm/poststate.json create mode 100644 cmd/evm/testdata/10/alloc.json create mode 100644 cmd/evm/testdata/10/env.json create mode 100644 cmd/evm/testdata/10/readme.md create mode 100644 cmd/evm/testdata/10/txs.json create mode 100644 cmd/evm/testdata/11/alloc.json create mode 100644 cmd/evm/testdata/11/env.json create mode 100644 cmd/evm/testdata/11/readme.md create mode 100644 cmd/evm/testdata/11/txs.json create mode 100644 cmd/evm/testdata/9/alloc.json create mode 100644 cmd/evm/testdata/9/env.json create mode 100644 cmd/evm/testdata/9/readme.md create mode 100644 cmd/evm/testdata/9/txs.json create mode 100644 consensus/misc/eip1559.go create mode 100644 consensus/misc/eip1559_test.go create mode 100644 consensus/misc/gaslimit.go create mode 100644 core/types/dynamic_fee_tx.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index d6d525eae1..9de427ae43 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -716,6 +716,8 @@ func (m callMsg) Nonce() uint64 { return 0 } func (m callMsg) CheckNonce() bool { return false } func (m callMsg) To() *common.Address { return m.CallMsg.To } func (m callMsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } +func (m callMsg) FeeCap() *big.Int { return m.CallMsg.FeeCap } +func (m callMsg) Tip() *big.Int { return m.CallMsg.Tip } func (m callMsg) Gas() uint64 { return m.CallMsg.Gas } func (m callMsg) Value() *big.Int { return m.CallMsg.Value } func (m callMsg) Data() []byte { return m.CallMsg.Data } diff --git a/accounts/external/backend.go b/accounts/external/backend.go index de241385c2..59766217d2 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -217,12 +217,12 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio if chainID != nil { args.ChainID = (*hexutil.Big)(chainID) } - // However, if the user asked for a particular chain id, then we should - // use that instead. - if tx.Type() != types.LegacyTxType && tx.ChainId() != nil { - args.ChainID = (*hexutil.Big)(tx.ChainId()) - } - if tx.Type() == types.AccessListTxType { + if tx.Type() != types.LegacyTxType { + // However, if the user asked for a particular chain id, then we should + // use that instead. + if tx.ChainId() != nil { + args.ChainID = (*hexutil.Big)(tx.ChainId()) + } accessList := tx.AccessList() args.AccessList = &accessList } diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index c3f1b16efc..cf6974bc43 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -69,6 +69,7 @@ type stEnv struct { Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` Ommers []ommer `json:"ommers,omitempty"` + BaseFee *big.Int `json:"currentBaseFee,omitempty"` } type stEnvMarshaling struct { @@ -77,6 +78,7 @@ type stEnvMarshaling struct { GasLimit math.HexOrDecimal64 Number math.HexOrDecimal64 Timestamp math.HexOrDecimal64 + BaseFee *math.HexOrDecimal256 } // Apply applies a set of transactions to a pre-state @@ -120,6 +122,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, GasLimit: pre.Env.GasLimit, GetHash: getHash, } + // If currentBaseFee is defined, add it to the vmContext. + if pre.Env.BaseFee != nil { + vmContext.BaseFee = new(big.Int).Set(pre.Env.BaseFee) + } // If DAO is supported/enabled, we need to handle it here. In geth 'proper', it's // done in StateProcessor.Process(block, ...), right before transactions are applied. if chainConfig.DAOForkSupport && @@ -129,7 +135,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } for i, tx := range txs { - msg, err := tx.AsMessage(signer) + msg, err := tx.AsMessage(signer, pre.Env.BaseFee) if err != nil { log.Info("rejected tx", "index", i, "hash", tx.Hash(), "error", err) rejectedTxs = append(rejectedTxs, i) diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index ab5951534e..695fdba1e1 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -16,13 +16,14 @@ var _ = (*stEnvMarshaling)(nil) // MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { - Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` - GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) @@ -32,19 +33,21 @@ func (s stEnv) MarshalJSON() ([]byte, error) { enc.Timestamp = math.HexOrDecimal64(s.Timestamp) enc.BlockHashes = s.BlockHashes enc.Ommers = s.Ommers + enc.BaseFee = (*math.HexOrDecimal256)(s.BaseFee) return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { - Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` - GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -76,5 +79,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.Ommers != nil { s.Ommers = dec.Ommers } + if dec.BaseFee != nil { + s.BaseFee = (*big.Int)(dec.BaseFee) + } return nil } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 9bb03c2c6a..bab6e63faa 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -19,6 +19,7 @@ package t8ntool import ( "crypto/ecdsa" "encoding/json" + "errors" "fmt" "io/ioutil" "math/big" @@ -210,9 +211,12 @@ func Main(ctx *cli.Context) error { if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil { return NewError(ErrorJson, fmt.Errorf("failed signing transactions: %v", err)) } - - // Iterate over all the tests, run them and aggregate the results - + // Sanity check, to not `panic` in state_transition + if chainConfig.IsLondon(big.NewInt(int64(prestate.Env.Number))) { + if prestate.Env.BaseFee == nil { + return NewError(ErrorVMConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section")) + } + } // Run the test and aggregate the result s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { diff --git a/cmd/evm/poststate.json b/cmd/evm/poststate.json deleted file mode 100644 index 9ee17f18d1..0000000000 --- a/cmd/evm/poststate.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "root": "f4157bb27bcb1d1a63001434a249a80948f2e9fe1f53d551244c1dae826b5b23", - "accounts": { - "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192": { - "balance": "4276951709", - "nonce": 1, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "6916764286133345652", - "nonce": 172, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - }, - "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "42500", - "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - } - } -} \ No newline at end of file diff --git a/cmd/evm/testdata/10/alloc.json b/cmd/evm/testdata/10/alloc.json new file mode 100644 index 0000000000..6e98e7513c --- /dev/null +++ b/cmd/evm/testdata/10/alloc.json @@ -0,0 +1,23 @@ +{ + "0x1111111111111111111111111111111111111111" : { + "balance" : "0x010000000000", + "code" : "0xfe", + "nonce" : "0x01", + "storage" : { + } + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0x010000000000", + "code" : "0x", + "nonce" : "0x01", + "storage" : { + } + }, + "0xd02d72e067e77158444ef2020ff2d325f929b363" : { + "balance" : "0x01000000000000", + "code" : "0x", + "nonce" : "0x01", + "storage" : { + } + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/10/env.json b/cmd/evm/testdata/10/env.json new file mode 100644 index 0000000000..3a82d46a77 --- /dev/null +++ b/cmd/evm/testdata/10/env.json @@ -0,0 +1,12 @@ +{ + "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "0x020000", + "currentNumber" : "0x01", + "currentTimestamp" : "0x079e", + "previousHash" : "0xcb23ee65a163121f640673b41788ee94633941405f95009999b502eedfbbfd4f", + "currentGasLimit" : "0x40000000", + "currentBaseFee" : "0x036b", + "blockHashes" : { + "0" : "0xcb23ee65a163121f640673b41788ee94633941405f95009999b502eedfbbfd4f" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/10/readme.md b/cmd/evm/testdata/10/readme.md new file mode 100644 index 0000000000..c34be80bb7 --- /dev/null +++ b/cmd/evm/testdata/10/readme.md @@ -0,0 +1,79 @@ +## EIP-1559 testing + +This test contains testcases for EIP-1559, which were reported by Ori as misbehaving. + +``` +[user@work evm]$ dir=./testdata/10 && ./evm t8n --state.fork=London --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout --output.result=stdout 2>&1 +INFO [05-09|22:11:59.436] rejected tx index=3 hash=db07bf..ede1e8 from=0xd02d72E067e77158444ef2020Ff2d325f929B363 error="gas limit reached" +``` +Output: +```json +{ + "alloc": { + "0x1111111111111111111111111111111111111111": { + "code": "0xfe", + "balance": "0x10000000000", + "nonce": "0x1" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x10000000000", + "nonce": "0x1" + }, + "0xd02d72e067e77158444ef2020ff2d325f929b363": { + "balance": "0xff5beffffc95", + "nonce": "0x4" + } + }, + "result": { + "stateRoot": "0xf91a7ec08e4bfea88719aab34deabb000c86902360532b52afa9599d41f2bb8b", + "txRoot": "0xda925f2306a52fa24c15d5cd212d736ee016415fd8dd0c45fd368de7917d64bb", + "receiptRoot": "0x439a25f7fc424c10fb1f89800e4aa1df74156b137239d9ac3eaa7c911c353cd5", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [ + { + "type": "0x2", + "root": "0x", + "status": "0x0", + "cumulativeGasUsed": "0x10000001", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x88980f6efcc5358d9c359663e7b9414722d430497637340ea056b076bc206701", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x10000001", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + }, + { + "type": "0x2", + "root": "0x", + "status": "0x0", + "cumulativeGasUsed": "0x20000001", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0xd7bf3886f4e2aef74d525ae072c680f3846f550254401b67cbfda4a233757582", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x10000000", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x1" + }, + { + "type": "0x2", + "root": "0x", + "status": "0x0", + "cumulativeGasUsed": "0x30000001", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x50308296760f01f1eeec7500e9e73cad67469249b1f59e9a9f55e6625a4923db", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x10000000", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x2" + } + ], + "rejected": [ + 3 + ] + } +} +``` diff --git a/cmd/evm/testdata/10/txs.json b/cmd/evm/testdata/10/txs.json new file mode 100644 index 0000000000..014f9db9af --- /dev/null +++ b/cmd/evm/testdata/10/txs.json @@ -0,0 +1,70 @@ +[ + { + "input" : "0x", + "gas" : "0x10000001", + "nonce" : "0x1", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "v" : "0x0", + "r" : "0x7a45f00bcde9036b026cdf1628b023cd8a31a95c62b5e4dbbee2fa7debe668fb", + "s" : "0x3cc9d6f2cd00a045b0263f2d6dad7d60938d5d13d061af4969f95928aa934d4a", + "secretKey" : "0x41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + "chainId" : "0x1", + "type" : "0x2", + "feeCap" : "0xfa0", + "tip" : "0x0", + "accessList" : [ + ] + }, + { + "input" : "0x", + "gas" : "0x10000000", + "nonce" : "0x2", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "v" : "0x0", + "r" : "0x4c564b94b0281a8210eeec2dd1fe2e16ff1c1903a8c3a1078d735d7f8208b2af", + "s" : "0x56432b2593e6de95db1cb997b7385217aca03f1615327e231734446b39f266d", + "secretKey" : "0x41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + "chainId" : "0x1", + "type" : "0x2", + "feeCap" : "0xfa0", + "tip" : "0x0", + "accessList" : [ + ] + }, + { + "input" : "0x", + "gas" : "0x10000000", + "nonce" : "0x3", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "v" : "0x0", + "r" : "0x2ed2ef52f924f59d4a21e1f2a50d3b1109303ce5e32334a7ece9b46f4fbc2a57", + "s" : "0x2980257129cbd3da987226f323d50ba3975a834d165e0681f991b75615605c44", + "secretKey" : "0x41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + "chainId" : "0x1", + "type" : "0x2", + "feeCap" : "0xfa0", + "tip" : "0x0", + "accessList" : [ + ] + }, + { + "input" : "0x", + "gas" : "0x10000000", + "nonce" : "0x4", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "v" : "0x0", + "r" : "0x5df7d7f8f8e15b36fc9f189cacb625040fad10398d08fc90812595922a2c49b2", + "s" : "0x565fc1803f77a84d754ffe3c5363ab54a8d93a06ea1bb9d4c73c73a282b35917", + "secretKey" : "0x41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + "chainId" : "0x1", + "type" : "0x2", + "feeCap" : "0xfa0", + "tip" : "0x0", + "accessList" : [ + ] + } +] \ No newline at end of file diff --git a/cmd/evm/testdata/11/alloc.json b/cmd/evm/testdata/11/alloc.json new file mode 100644 index 0000000000..86938230fa --- /dev/null +++ b/cmd/evm/testdata/11/alloc.json @@ -0,0 +1,25 @@ +{ + "0x0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6" : { + "balance" : "0x0de0b6b3a7640000", + "code" : "0x61ffff5060046000f3", + "nonce" : "0x01", + "storage" : { + } + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0x0de0b6b3a7640000", + "code" : "0x", + "nonce" : "0x00", + "storage" : { + "0x00" : "0x00" + } + }, + "0xb94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0x00", + "code" : "0x6001600055", + "nonce" : "0x00", + "storage" : { + } + } +} + diff --git a/cmd/evm/testdata/11/env.json b/cmd/evm/testdata/11/env.json new file mode 100644 index 0000000000..37dedf0947 --- /dev/null +++ b/cmd/evm/testdata/11/env.json @@ -0,0 +1,12 @@ +{ + "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "0x020000", + "currentNumber" : "0x01", + "currentTimestamp" : "0x03e8", + "previousHash" : "0xfda4419b3660e99f37e536dae1ab081c180136bb38c837a93e93d9aab58553b2", + "currentGasLimit" : "0x0f4240", + "blockHashes" : { + "0" : "0xfda4419b3660e99f37e536dae1ab081c180136bb38c837a93e93d9aab58553b2" + } +} + diff --git a/cmd/evm/testdata/11/readme.md b/cmd/evm/testdata/11/readme.md new file mode 100644 index 0000000000..d499f8e99f --- /dev/null +++ b/cmd/evm/testdata/11/readme.md @@ -0,0 +1,13 @@ +## Test missing basefee + +In this test, the `currentBaseFee` is missing from the env portion. +On a live blockchain, the basefee is present in the header, and verified as part of header validation. + +In `evm t8n`, we don't have blocks, so it needs to be added in the `env`instead. + +When it's missing, an error is expected. + +``` +dir=./testdata/11 && ./evm t8n --state.fork=London --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout --output.result=stdout 2>&1>/dev/null +ERROR(3): EIP-1559 config but missing 'currentBaseFee' in env section +``` \ No newline at end of file diff --git a/cmd/evm/testdata/11/txs.json b/cmd/evm/testdata/11/txs.json new file mode 100644 index 0000000000..c54b0a1f5b --- /dev/null +++ b/cmd/evm/testdata/11/txs.json @@ -0,0 +1,14 @@ +[ + { + "input" : "0x38600060013960015160005560006000f3", + "gas" : "0x61a80", + "gasPrice" : "0x1", + "nonce" : "0x0", + "value" : "0x186a0", + "v" : "0x1c", + "r" : "0x2e1391fd903387f1cc2b51df083805fb4bbb0d4710a2cdf4a044d191ff7be63e", + "s" : "0x7f10a933c42ab74927db02b1db009e923d9d2ab24ac24d63c399f2fe5d9c9b22", + "secretKey" : "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + } +] + diff --git a/cmd/evm/testdata/9/alloc.json b/cmd/evm/testdata/9/alloc.json new file mode 100644 index 0000000000..c14e38e845 --- /dev/null +++ b/cmd/evm/testdata/9/alloc.json @@ -0,0 +1,11 @@ +{ + "0x000000000000000000000000000000000000aaaa": { + "balance": "0x03", + "code": "0x58585454", + "nonce": "0x1" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x100000000000000", + "nonce": "0x00" + } +} diff --git a/cmd/evm/testdata/9/env.json b/cmd/evm/testdata/9/env.json new file mode 100644 index 0000000000..ec5164b995 --- /dev/null +++ b/cmd/evm/testdata/9/env.json @@ -0,0 +1,8 @@ +{ + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x20000", + "currentGasTarget": "0x1000000000", + "currentBaseFee": "0x3B9ACA00", + "currentNumber": "0x1000000", + "currentTimestamp": "0x04" +} diff --git a/cmd/evm/testdata/9/readme.md b/cmd/evm/testdata/9/readme.md new file mode 100644 index 0000000000..88f0f12aaa --- /dev/null +++ b/cmd/evm/testdata/9/readme.md @@ -0,0 +1,75 @@ +## EIP-1559 testing + +This test contains testcases for EIP-1559, which uses an new transaction type and has a new block parameter. + +### Prestate + +The alloc portion contains one contract (`0x000000000000000000000000000000000000aaaa`), containing the +following code: `0x58585454`: `PC; PC; SLOAD; SLOAD`. + +Essentialy, this contract does `SLOAD(0)` and `SLOAD(1)`. + +The alloc also contains some funds on `0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b`. + +## Transactions + +There are two transactions, each invokes the contract above. + +1. EIP-1559 ACL-transaction, which contains the `0x0` slot for `0xaaaa` +2. Legacy transaction + +## Execution + +Running it yields: +``` +$ dir=./testdata/9 && ./evm t8n --state.fork=London --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --trace && cat trace-* | grep SLOAD +{"pc":2,"op":84,"gas":"0x48c28","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0","0x1"],"returnStack":null,"returnD +ata":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":3,"op":84,"gas":"0x483f4","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x0","0x0"],"returnStack":null,"returnDa +ta":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":2,"op":84,"gas":"0x49cf4","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0","0x1"],"returnStack":null,"returnD +ata":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +{"pc":3,"op":84,"gas":"0x494c0","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0","0x0"],"returnStack":null,"returnD +ata":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""} +``` + +We can also get the post-alloc: +``` +$ dir=./testdata/9 && ./evm t8n --state.fork=London --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout +{ + "alloc": { + "0x000000000000000000000000000000000000aaaa": { + "code": "0x58585454", + "balance": "0x3", + "nonce": "0x1" + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0xbfc02677a000" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0xff104fcfea7800", + "nonce": "0x2" + } + } +} +``` + +If we try to execute it on older rules: +``` +dir=./testdata/9 && ./evm t8n --state.fork=Berlin --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout +ERROR(10): Failed signing transactions: ERROR(10): Tx 0: failed to sign tx: transaction type not supported +``` + +It fails, due to the `evm t8n` cannot sign them in with the given signer. We can bypass that, however, +by feeding it presigned transactions, located in `txs_signed.json`. + +``` +dir=./testdata/9 && ./evm t8n --state.fork=Berlin --input.alloc=$dir/alloc.json --input.txs=$dir/txs_signed.json --input.env=$dir/env.json +INFO [05-07|12:28:42.072] rejected tx index=0 hash=b4821e..536819 error="transaction type not supported" +INFO [05-07|12:28:42.072] rejected tx index=1 hash=a9c6c6..fa4036 from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0" +INFO [05-07|12:28:42.073] Wrote file file=alloc.json +INFO [05-07|12:28:42.073] Wrote file file=result.json +``` + +Number `0` is not applicable, and therefore number `1` has wrong nonce, and both are rejected. + diff --git a/cmd/evm/testdata/9/txs.json b/cmd/evm/testdata/9/txs.json new file mode 100644 index 0000000000..f349ae4a24 --- /dev/null +++ b/cmd/evm/testdata/9/txs.json @@ -0,0 +1,37 @@ +[ + { + "gas": "0x4ef00", + "tip": "0x2", + "feeCap": "0x12A05F200", + "chainId": "0x1", + "input": "0x", + "nonce": "0x0", + "to": "0x000000000000000000000000000000000000aaaa", + "value": "0x0", + "type" : "0x2", + "accessList": [ + {"address": "0x000000000000000000000000000000000000aaaa", + "storageKeys": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "v": "0x0", + "r": "0x0", + "s": "0x0", + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + }, + { + "gas": "0x4ef00", + "gasPrice": "0x12A05F200", + "chainId": "0x1", + "input": "0x", + "nonce": "0x1", + "to": "0x000000000000000000000000000000000000aaaa", + "value": "0x0", + "v": "0x0", + "r": "0x0", + "s": "0x0", + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + } +] diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 9954a023e5..b693e80518 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -299,10 +299,6 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H if header.GasLimit > cap { return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, cap) } - // Verify that the gasUsed is <= gasLimit - if header.GasUsed > header.GasLimit { - return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) - } // If all checks passed, validate any special fields for hard forks if err := misc.VerifyForkHashes(chain.Config(), header, false); err != nil { return err @@ -334,14 +330,21 @@ func (c *Clique) verifyCascadingFields(chain consensus.ChainHeaderReader, header if parent.Time+c.config.Period > header.Time { return errInvalidTimestamp } - // Verify that the gas limit remains within allowed bounds - diff := int64(parent.GasLimit) - int64(header.GasLimit) - if diff < 0 { - diff *= -1 + // Verify that the gasUsed is <= gasLimit + if header.GasUsed > header.GasLimit { + return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) } - limit := parent.GasLimit / params.GasLimitBoundDivisor - if uint64(diff) >= limit || header.GasLimit < params.MinGasLimit { - return fmt.Errorf("invalid gas limit: have %d, want %d += %d", header.GasLimit, parent.GasLimit, limit) + if !chain.Config().IsLondon(header.Number) { + // Verify BaseFee not present before EIP-1559 fork. + if header.BaseFee != nil { + return fmt.Errorf("invalid baseFee before fork: have %d, want ", header.BaseFee) + } + if err := misc.VerifyGaslimit(parent.GasLimit, header.GasLimit); err != nil { + return err + } + } else if err := misc.VerifyEip1559Header(chain.Config(), parent, header); err != nil { + // Verify the header's EIP-1559 attributes. + return err } // Retrieve the snapshot needed to verify this header and cache it snap, err := c.snapshot(chain, number-1, header.ParentHash, parents) @@ -725,7 +728,7 @@ func CliqueRLP(header *types.Header) []byte { } func encodeSigHeader(w io.Writer, header *types.Header) { - err := rlp.Encode(w, []interface{}{ + enc := []interface{}{ header.ParentHash, header.UncleHash, header.Coinbase, @@ -741,8 +744,11 @@ func encodeSigHeader(w io.Writer, header *types.Header) { header.Extra[:len(header.Extra)-crypto.SignatureLength], // Yes, this will panic if extra is too short header.MixDigest, header.Nonce, - }) - if err != nil { + } + if header.BaseFee != nil { + enc = append(enc, header.BaseFee) + } + if err := rlp.Encode(w, enc); err != nil { panic("can't encode: " + err.Error()) } } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index b04cb24fb4..9b9657e190 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -284,16 +284,18 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa if header.GasUsed > header.GasLimit { return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) } - - // Verify that the gas limit remains within allowed bounds - diff := int64(parent.GasLimit) - int64(header.GasLimit) - if diff < 0 { - diff *= -1 - } - limit := parent.GasLimit / params.GasLimitBoundDivisor - - if uint64(diff) >= limit || header.GasLimit < params.MinGasLimit { - return fmt.Errorf("invalid gas limit: have %d, want %d += %d", header.GasLimit, parent.GasLimit, limit) + // Verify the block's gas usage and (if applicable) verify the base fee. + if !chain.Config().IsLondon(header.Number) { + // Verify BaseFee not present before EIP-1559 fork. + if header.BaseFee != nil { + return fmt.Errorf("invalid baseFee before fork: have %d, expected 'nil'", header.BaseFee) + } + if err := misc.VerifyGaslimit(parent.GasLimit, header.GasLimit); err != nil { + return err + } + } else if err := misc.VerifyEip1559Header(chain.Config(), parent, header); err != nil { + // Verify the header's EIP-1559 attributes. + return err } // Verify that the block number is parent's +1 if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { @@ -604,7 +606,7 @@ func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { hasher := sha3.NewLegacyKeccak256() - rlp.Encode(hasher, []interface{}{ + enc := []interface{}{ header.ParentHash, header.UncleHash, header.Coinbase, @@ -618,7 +620,11 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { header.GasUsed, header.Time, header.Extra, - }) + } + if header.BaseFee != nil { + enc = append(enc, header.BaseFee) + } + rlp.Encode(hasher, enc) hasher.Sum(hash[:0]) return hash } diff --git a/consensus/misc/eip1559.go b/consensus/misc/eip1559.go new file mode 100644 index 0000000000..8fca0fdc70 --- /dev/null +++ b/consensus/misc/eip1559.go @@ -0,0 +1,93 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package misc + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +// VerifyEip1559Header verifies some header attributes which were changed in EIP-1559, +// - gas limit check +// - basefee check +func VerifyEip1559Header(config *params.ChainConfig, parent, header *types.Header) error { + // Verify that the gas limit remains within allowed bounds + parentGasLimit := parent.GasLimit + if !config.IsLondon(parent.Number) { + parentGasLimit = parent.GasLimit * params.ElasticityMultiplier + } + if err := VerifyGaslimit(parentGasLimit, header.GasLimit); err != nil { + return err + } + // Verify the header is not malformed + if header.BaseFee == nil { + return fmt.Errorf("header is missing baseFee") + } + // Verify the baseFee is correct based on the parent header. + expectedBaseFee := CalcBaseFee(config, parent) + if header.BaseFee.Cmp(expectedBaseFee) != 0 { + return fmt.Errorf("invalid baseFee: have %s, want %s, parentBaseFee %s, parentGasUsed %d", + expectedBaseFee, header.BaseFee, parent.BaseFee, parent.GasUsed) + } + return nil +} + +// CalcBaseFee calculates the basefee of the header. +func CalcBaseFee(config *params.ChainConfig, parent *types.Header) *big.Int { + // If the current block is the first EIP-1559 block, return the InitialBaseFee. + if !config.IsLondon(parent.Number) { + return new(big.Int).SetUint64(params.InitialBaseFee) + } + + var ( + parentGasTarget = parent.GasLimit / params.ElasticityMultiplier + parentGasTargetBig = new(big.Int).SetUint64(parentGasTarget) + baseFeeChangeDenominator = new(big.Int).SetUint64(params.BaseFeeChangeDenominator) + ) + // If the parent gasUsed is the same as the target, the baseFee remains unchanged. + if parent.GasUsed == parentGasTarget { + return new(big.Int).Set(parent.BaseFee) + } + if parent.GasUsed > parentGasTarget { + // If the parent block used more gas than its target, the baseFee should increase. + gasUsedDelta := new(big.Int).SetUint64(parent.GasUsed - parentGasTarget) + x := new(big.Int).Mul(parent.BaseFee, gasUsedDelta) + y := x.Div(x, parentGasTargetBig) + baseFeeDelta := math.BigMax( + x.Div(y, baseFeeChangeDenominator), + common.Big1, + ) + + return x.Add(parent.BaseFee, baseFeeDelta) + } else { + // Otherwise if the parent block used less gas than its target, the baseFee should decrease. + gasUsedDelta := new(big.Int).SetUint64(parentGasTarget - parent.GasUsed) + x := new(big.Int).Mul(parent.BaseFee, gasUsedDelta) + y := x.Div(x, parentGasTargetBig) + baseFeeDelta := x.Div(y, baseFeeChangeDenominator) + + return math.BigMax( + x.Sub(parent.BaseFee, baseFeeDelta), + common.Big0, + ) + } +} diff --git a/consensus/misc/eip1559_test.go b/consensus/misc/eip1559_test.go new file mode 100644 index 0000000000..333411db51 --- /dev/null +++ b/consensus/misc/eip1559_test.go @@ -0,0 +1,133 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package misc + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +// copyConfig does a _shallow_ copy of a given config. Safe to set new values, but +// do not use e.g. SetInt() on the numbers. For testing only +func copyConfig(original *params.ChainConfig) *params.ChainConfig { + return ¶ms.ChainConfig{ + ChainID: original.ChainID, + HomesteadBlock: original.HomesteadBlock, + DAOForkBlock: original.DAOForkBlock, + DAOForkSupport: original.DAOForkSupport, + EIP150Block: original.EIP150Block, + EIP150Hash: original.EIP150Hash, + EIP155Block: original.EIP155Block, + EIP158Block: original.EIP158Block, + ByzantiumBlock: original.ByzantiumBlock, + ConstantinopleBlock: original.ConstantinopleBlock, + PetersburgBlock: original.PetersburgBlock, + IstanbulBlock: original.IstanbulBlock, + MuirGlacierBlock: original.MuirGlacierBlock, + BerlinBlock: original.BerlinBlock, + LondonBlock: original.LondonBlock, + EWASMBlock: original.EWASMBlock, + CatalystBlock: original.CatalystBlock, + Ethash: original.Ethash, + Clique: original.Clique, + } +} + +func config() *params.ChainConfig { + config := copyConfig(params.TestChainConfig) + config.LondonBlock = big.NewInt(5) + return config +} + +// TestBlockGasLimits tests the gasLimit checks for blocks both across +// the EIP-1559 boundary and post-1559 blocks +func TestBlockGasLimits(t *testing.T) { + initial := new(big.Int).SetUint64(params.InitialBaseFee) + + for i, tc := range []struct { + pGasLimit uint64 + pNum int64 + gasLimit uint64 + ok bool + }{ + // Transitions from non-london to london + {10000000, 4, 20000000, true}, // No change + {10000000, 4, 20019530, true}, // Upper limit + {10000000, 4, 20019531, false}, // Upper +1 + {10000000, 4, 19980470, true}, // Lower limit + {10000000, 4, 19980469, false}, // Lower limit -1 + // London to London + {20000000, 5, 20000000, true}, + {20000000, 5, 20019530, true}, // Upper limit + {20000000, 5, 20019531, false}, // Upper limit +1 + {20000000, 5, 19980470, true}, // Lower limit + {20000000, 5, 19980469, false}, // Lower limit -1 + {40000000, 5, 40039061, true}, // Upper limit + {40000000, 5, 40039062, false}, // Upper limit +1 + {40000000, 5, 39960939, true}, // lower limit + {40000000, 5, 39960938, false}, // Lower limit -1 + } { + parent := &types.Header{ + GasUsed: tc.pGasLimit / 2, + GasLimit: tc.pGasLimit, + BaseFee: initial, + Number: big.NewInt(tc.pNum), + } + header := &types.Header{ + GasUsed: tc.gasLimit / 2, + GasLimit: tc.gasLimit, + BaseFee: initial, + Number: big.NewInt(tc.pNum + 1), + } + err := VerifyEip1559Header(config(), parent, header) + if tc.ok && err != nil { + t.Errorf("test %d: Expected valid header: %s", i, err) + } + if !tc.ok && err == nil { + t.Errorf("test %d: Expected invalid header", i) + } + } +} + +// TestCalcBaseFee assumes all blocks are 1559-blocks +func TestCalcBaseFee(t *testing.T) { + tests := []struct { + parentBaseFee int64 + parentGasLimit uint64 + parentGasUsed uint64 + expectedBaseFee int64 + }{ + {params.InitialBaseFee, 20000000, 10000000, params.InitialBaseFee}, // usage == target + {params.InitialBaseFee, 20000000, 9000000, 987500000}, // usage below target + {params.InitialBaseFee, 20000000, 11000000, 1012500000}, // usage above target + } + for i, test := range tests { + parent := &types.Header{ + Number: common.Big32, + GasLimit: test.parentGasLimit, + GasUsed: test.parentGasUsed, + BaseFee: big.NewInt(test.parentBaseFee), + } + if have, want := CalcBaseFee(config(), parent), big.NewInt(test.expectedBaseFee); have.Cmp(want) != 0 { + t.Errorf("test %d: have %d want %d, ", i, have, want) + } + } +} diff --git a/consensus/misc/gaslimit.go b/consensus/misc/gaslimit.go new file mode 100644 index 0000000000..25f35300b9 --- /dev/null +++ b/consensus/misc/gaslimit.go @@ -0,0 +1,42 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package misc + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/params" +) + +// VerifyGaslimit verifies the header gas limit according increase/decrease +// in relation to the parent gas limit. +func VerifyGaslimit(parentGasLimit, headerGasLimit uint64) error { + // Verify that the gas limit remains within allowed bounds + diff := int64(parentGasLimit) - int64(headerGasLimit) + if diff < 0 { + diff *= -1 + } + limit := parentGasLimit / params.GasLimitBoundDivisor + if uint64(diff) >= limit { + return fmt.Errorf("invalid gas limit: have %d, want %d +-= %d", headerGasLimit, parentGasLimit, limit-1) + } + if headerGasLimit < params.MinGasLimit { + return errors.New("invalid gas limit below 5000") + } + return nil +} diff --git a/core/blockchain_test.go b/core/blockchain_test.go index a8ce4c7b9a..8ace9f6a6d 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -73,6 +73,10 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B return db, blockchain, err } +func newGwei(n int64) *big.Int { + return new(big.Int).Mul(big.NewInt(n), big.NewInt(params.GWei)) +} + // Test fork of length N starting from block i func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db @@ -3109,3 +3113,156 @@ func TestEIP2718Transition(t *testing.T) { } } + +// TestEIP1559Transition tests the following: +// +// 1. A tranaction whose feeCap is greater than the baseFee is valid. +// 2. Gas accounting for access lists on EIP-1559 transactions is correct. +// 3. Only the transaction's tip will be received by the coinbase. +// 4. The transaction sender pays for both the tip and baseFee. +// 5. The coinbase receives only the partially realized tip when +// feeCap - tip < baseFee. +// 6. Legacy transaction behave as expected (e.g. gasPrice = feeCap = tip). +func TestEIP1559Transition(t *testing.T) { + var ( + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") + + // Generate a canonical chain to act as the main dataset + engine = ethash.NewFaker() + db = rawdb.NewMemoryDatabase() + + // A sender who makes transactions, has some funds + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + funds = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + gspec = &Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: GenesisAlloc{ + addr1: {Balance: funds}, + addr2: {Balance: funds}, + // The address 0xAAAA sloads 0x00 and 0x01 + aa: { + Code: []byte{ + byte(vm.PC), + byte(vm.PC), + byte(vm.SLOAD), + byte(vm.SLOAD), + }, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + ) + + gspec.Config.BerlinBlock = common.Big0 + gspec.Config.LondonBlock = common.Big0 + genesis := gspec.MustCommit(db) + signer := types.LatestSigner(gspec.Config) + + blocks, _ := GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{1}) + + // One transaction to 0xAAAA + accesses := types.AccessList{types.AccessTuple{ + Address: aa, + StorageKeys: []common.Hash{{0}}, + }} + + txdata := &types.DynamicFeeTx{ + ChainID: gspec.Config.ChainID, + Nonce: 0, + To: &aa, + Gas: 30000, + FeeCap: newGwei(5), + Tip: big.NewInt(2), + AccessList: accesses, + Data: []byte{}, + } + tx := types.NewTx(txdata) + tx, _ = types.SignTx(tx, signer, key1) + + b.AddTx(tx) + }) + + diskdb := rawdb.NewMemoryDatabase() + gspec.MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + + block := chain.GetBlockByNumber(1) + + // 1+2: Ensure EIP-1559 access lists are accounted for via gas usage. + expectedGas := params.TxGas + params.TxAccessListAddressGas + params.TxAccessListStorageKeyGas + + vm.GasQuickStep*2 + params.WarmStorageReadCostEIP2929 + params.ColdSloadCostEIP2929 + if block.GasUsed() != expectedGas { + t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expectedGas, block.GasUsed()) + } + + state, _ := chain.State() + + // 3: Ensure that miner received only the tx's tip. + actual := state.GetBalance(block.Coinbase()) + expected := new(big.Int).Add( + new(big.Int).SetUint64(block.GasUsed()*block.Transactions()[0].Tip().Uint64()), + ethash.ConstantinopleBlockReward, + ) + if actual.Cmp(expected) != 0 { + t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) + } + + // 4: Ensure the tx sender paid for the gasUsed * (tip + block baseFee). + actual = new(big.Int).Sub(funds, state.GetBalance(addr1)) + expected = new(big.Int).SetUint64(block.GasUsed() * (block.Transactions()[0].Tip().Uint64() + block.BaseFee().Uint64())) + if actual.Cmp(expected) != 0 { + t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) + } + + blocks, _ = GenerateChain(gspec.Config, block, engine, db, 1, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{2}) + + txdata := &types.LegacyTx{ + Nonce: 0, + To: &aa, + Gas: 30000, + GasPrice: newGwei(5), + } + tx := types.NewTx(txdata) + tx, _ = types.SignTx(tx, signer, key2) + + b.AddTx(tx) + }) + + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + + block = chain.GetBlockByNumber(2) + state, _ = chain.State() + effectiveTip := block.Transactions()[0].Tip().Uint64() - block.BaseFee().Uint64() + + // 6+5: Ensure that miner received only the tx's effective tip. + actual = state.GetBalance(block.Coinbase()) + expected = new(big.Int).Add( + new(big.Int).SetUint64(block.GasUsed()*effectiveTip), + ethash.ConstantinopleBlockReward, + ) + if actual.Cmp(expected) != 0 { + t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) + } + + // 4: Ensure the tx sender paid for the gasUsed * (effectiveTip + block baseFee). + actual = new(big.Int).Sub(funds, state.GetBalance(addr2)) + expected = new(big.Int).SetUint64(block.GasUsed() * (effectiveTip + block.BaseFee().Uint64())) + if actual.Cmp(expected) != 0 { + t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) + } +} diff --git a/core/chain_makers.go b/core/chain_makers.go index e058e5a78e..b1b7dc3591 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -253,7 +253,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S time = parent.Time() + 10 // block time is fixed at 10 seconds } - return &types.Header{ + header := &types.Header{ Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), ParentHash: parent.Hash(), Coinbase: parent.Coinbase(), @@ -267,6 +267,12 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S Number: new(big.Int).Add(parent.Number(), common.Big1), Time: time, } + + if chain.Config().IsLondon(parent.Number()) { + header.BaseFee = misc.CalcBaseFee(chain.Config(), parent.Header()) + } + + return header } // makeHeaderChain creates a deterministic chain of headers rooted at parent. diff --git a/core/error.go b/core/error.go index 197dd81567..3d62cb9bcf 100644 --- a/core/error.go +++ b/core/error.go @@ -71,4 +71,8 @@ var ( // ErrTxTypeNotSupported is returned if a transaction is not supported in the // current network configuration. ErrTxTypeNotSupported = types.ErrTxTypeNotSupported + + // ErrFeeCapTooLow is returned if the transaction fee cap is less than the + // the base fee of the block. + ErrFeeCapTooLow = errors.New("fee cap less than block base fee") ) diff --git a/core/evm.go b/core/evm.go index 8f69d51499..6c67fc4376 100644 --- a/core/evm.go +++ b/core/evm.go @@ -37,13 +37,20 @@ type ChainContext interface { // NewEVMBlockContext creates a new context for use in the EVM. func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common.Address) vm.BlockContext { + var ( + beneficiary common.Address + baseFee *big.Int + ) + // If we don't have an explicit author (i.e. not mining), extract from the header - var beneficiary common.Address if author == nil { beneficiary, _ = chain.Engine().Author(header) // Ignore error, we're past header validation } else { beneficiary = *author } + if header.BaseFee != nil { + baseFee = new(big.Int).Set(header.BaseFee) + } return vm.BlockContext{ CanTransfer: CanTransfer, Transfer: Transfer, @@ -52,6 +59,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common BlockNumber: new(big.Int).Set(header.Number), Time: new(big.Int).SetUint64(header.Time), Difficulty: new(big.Int).Set(header.Difficulty), + BaseFee: baseFee, GasLimit: header.GasLimit, } } diff --git a/core/genesis.go b/core/genesis.go index d7d08e0909..b68ae4ef56 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -291,6 +291,9 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if g.Difficulty == nil { head.Difficulty = params.GenesisDifficulty } + if g.Config != nil && g.Config.IsLondon(common.Big0) { + head.BaseFee = new(big.Int).SetUint64(params.InitialBaseFee) + } statedb.Commit(false) statedb.Database().TrieDB().Commit(root, true, nil) diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 05394321f7..ecdfa67f00 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -63,7 +63,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c return } // Convert the transaction into an executable message and pre-cache its sender - msg, err := tx.AsMessage(signer) + msg, err := tx.AsMessage(signer, header.BaseFee) if err != nil { return // Also invalid block, bail out } diff --git a/core/state_processor.go b/core/state_processor.go index 40a953f0d4..6f6bc1879b 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -71,9 +71,9 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg) // Iterate over and process the individual transactions for i, tx := range block.Transactions() { - msg, err := tx.AsMessage(types.MakeSigner(p.config, header.Number)) + msg, err := tx.AsMessage(types.MakeSigner(p.config, header.Number), header.BaseFee) if err != nil { - return nil, nil, 0, err + return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.Prepare(tx.Hash(), block.Hash(), i) receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv) @@ -139,7 +139,7 @@ func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainCon // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { - msg, err := tx.AsMessage(types.MakeSigner(config, header.Number)) + msg, err := tx.AsMessage(types.MakeSigner(config, header.Number), header.BaseFee) if err != nil { return nil, err } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 5976ecc3d4..28baf6e7d2 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -38,75 +39,157 @@ import ( // contain invalid transactions func TestStateProcessorErrors(t *testing.T) { var ( - signer = types.HomesteadSigner{} + signer = types.LatestSigner(params.TestChainConfig) testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - db = rawdb.NewMemoryDatabase() - gspec = &Genesis{ - Config: params.TestChainConfig, - } - genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) ) - defer blockchain.Stop() var makeTx = func(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *types.Transaction { tx, _ := types.SignTx(types.NewTransaction(nonce, to, amount, gasLimit, gasPrice, data), signer, testKey) return tx } - for i, tt := range []struct { - txs []*types.Transaction - want string - }{ - { - txs: []*types.Transaction{ - makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, nil, nil), - makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, nil, nil), + var mkDynamicTx = func(nonce uint64, to common.Address, gasLimit uint64, tip, feeCap *big.Int) *types.Transaction { + tx, _ := types.SignTx(types.NewTx(&types.DynamicFeeTx{ + Nonce: nonce, + Tip: tip, + FeeCap: feeCap, + Gas: 0, + To: &to, + Value: big.NewInt(0), + }), signer, testKey) + return tx + } + { // Tests against a 'recent' chain definition + var ( + db = rawdb.NewMemoryDatabase() + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Balance: big.NewInt(1000000000000000000), // 1 ether + Nonce: 0, + }, + }, + } + genesis = gspec.MustCommit(db) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ) + defer blockchain.Stop() + + for i, tt := range []struct { + txs []*types.Transaction + want string + }{ + { // ErrNonceTooLow + txs: []*types.Transaction{ + makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), + makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), + }, + want: "could not apply tx 1 [0x0026256b3939ed97e2c4a6f3fce8ecf83bdcfa6d507c47838c308a1fb0436f62]: nonce too low: address 0x71562b71999873DB5b286dF957af199Ec94617F7, tx: 0 state: 1", + }, + { // ErrNonceTooHigh + txs: []*types.Transaction{ + makeTx(100, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), + }, + want: "could not apply tx 0 [0xdebad714ca7f363bd0d8121c4518ad48fa469ca81b0a081be3d10c17460f751b]: nonce too high: address 0x71562b71999873DB5b286dF957af199Ec94617F7, tx: 100 state: 0", + }, + { // ErrGasLimitReached + txs: []*types.Transaction{ + makeTx(0, common.Address{}, big.NewInt(0), 21000000, big.NewInt(875000000), nil), + }, + want: "could not apply tx 0 [0xbd49d8dadfd47fb846986695f7d4da3f7b2c48c8da82dbc211a26eb124883de9]: gas limit reached", }, - want: "could not apply tx 1 [0x36bfa6d14f1cd35a1be8cc2322982a595fabc0e799f09c1de3bad7bd5b1f7626]: nonce too low: address 0x71562b71999873DB5b286dF957af199Ec94617F7, tx: 0 state: 1", - }, - { - txs: []*types.Transaction{ - makeTx(100, common.Address{}, big.NewInt(0), params.TxGas, nil, nil), + { // ErrInsufficientFundsForTransfer + txs: []*types.Transaction{ + makeTx(0, common.Address{}, big.NewInt(1000000000000000000), params.TxGas, big.NewInt(875000000), nil), + }, + want: "could not apply tx 0 [0x98c796b470f7fcab40aaef5c965a602b0238e1034cce6fb73823042dd0638d74]: insufficient funds for transfer: address 0x71562b71999873DB5b286dF957af199Ec94617F7", }, - want: "could not apply tx 0 [0x51cd272d41ef6011d8138e18bf4043797aca9b713c7d39a97563f9bbe6bdbe6f]: nonce too high: address 0x71562b71999873DB5b286dF957af199Ec94617F7, tx: 100 state: 0", - }, - { - txs: []*types.Transaction{ - makeTx(0, common.Address{}, big.NewInt(0), 21000000, nil, nil), + { // ErrInsufficientFunds + txs: []*types.Transaction{ + makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(900000000000000000), nil), + }, + want: "could not apply tx 0 [0x4a69690c4b0cd85e64d0d9ea06302455b01e10a83db964d60281739752003440]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 have 1000000000000000000 want 18900000000000000000000", }, - want: "could not apply tx 0 [0x54c58b530824b0bb84b7a98183f08913b5d74e1cebc368515ef3c65edf8eb56a]: gas limit reached", - }, - { - txs: []*types.Transaction{ - makeTx(0, common.Address{}, big.NewInt(1), params.TxGas, nil, nil), + // ErrGasUintOverflow + // One missing 'core' error is ErrGasUintOverflow: "gas uint64 overflow", + // In order to trigger that one, we'd have to allocate a _huge_ chunk of data, such that the + // multiplication len(data) +gas_per_byte overflows uint64. Not testable at the moment + { // ErrIntrinsicGas + txs: []*types.Transaction{ + makeTx(0, common.Address{}, big.NewInt(0), params.TxGas-1000, big.NewInt(875000000), nil), + }, + want: "could not apply tx 0 [0xcf3b049a0b516cb4f9274b3e2a264359e2ba53b2fb64b7bda2c634d5c9d01fca]: intrinsic gas too low: have 20000, want 21000", }, - want: "could not apply tx 0 [0x3094b17498940d92b13baccf356ce8bfd6f221e926abc903d642fa1466c5b50e]: insufficient funds for transfer: address 0x71562b71999873DB5b286dF957af199Ec94617F7", - }, - { - txs: []*types.Transaction{ - makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(0xffffff), nil), + { // ErrGasLimitReached + txs: []*types.Transaction{ + makeTx(0, common.Address{}, big.NewInt(0), params.TxGas*1000, big.NewInt(875000000), nil), + }, + want: "could not apply tx 0 [0xbd49d8dadfd47fb846986695f7d4da3f7b2c48c8da82dbc211a26eb124883de9]: gas limit reached", }, - want: "could not apply tx 0 [0xaa3f7d86802b1f364576d9071bf231e31d61b392d306831ac9cf706ff5371ce0]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 have 0 want 352321515000", - }, - { - txs: []*types.Transaction{ - makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, nil, nil), - makeTx(1, common.Address{}, big.NewInt(0), params.TxGas, nil, nil), - makeTx(2, common.Address{}, big.NewInt(0), params.TxGas, nil, nil), - makeTx(3, common.Address{}, big.NewInt(0), params.TxGas-1000, big.NewInt(0), nil), + { // ErrFeeCapTooLow + txs: []*types.Transaction{ + mkDynamicTx(0, common.Address{}, params.TxGas-1000, big.NewInt(0), big.NewInt(0)), + }, + want: "could not apply tx 0 [0x21e9b9015150fc7f6bd5059890a5e1727f2452df285e8a84f4ca61a74c159ded]: fee cap less than block base fee: address 0x71562b71999873DB5b286dF957af199Ec94617F7, feeCap: 0 baseFee: 875000000", }, - want: "could not apply tx 3 [0x836fab5882205362680e49b311a20646de03b630920f18ec6ee3b111a2cf6835]: intrinsic gas too low: have 20000, want 21000", - }, - // The last 'core' error is ErrGasUintOverflow: "gas uint64 overflow", but in order to - // trigger that one, we'd have to allocate a _huge_ chunk of data, such that the - // multiplication len(data) +gas_per_byte overflows uint64. Not testable at the moment - } { - block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs) - _, err := blockchain.InsertChain(types.Blocks{block}) - if err == nil { - t.Fatal("block imported without errors") + } { + block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs, gspec.Config) + _, err := blockchain.InsertChain(types.Blocks{block}) + if err == nil { + t.Fatal("block imported without errors") + } + if have, want := err.Error(), tt.want; have != want { + t.Errorf("test %d:\nhave \"%v\"\nwant \"%v\"\n", i, have, want) + } } - if have, want := err.Error(), tt.want; have != want { - t.Errorf("test %d:\nhave \"%v\"\nwant \"%v\"\n", i, have, want) + } + + // One final error is ErrTxTypeNotSupported. For this, we need an older chain + { + var ( + db = rawdb.NewMemoryDatabase() + gspec = &Genesis{ + Config: ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + }, + Alloc: GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Balance: big.NewInt(1000000000000000000), // 1 ether + Nonce: 0, + }, + }, + } + genesis = gspec.MustCommit(db) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ) + defer blockchain.Stop() + for i, tt := range []struct { + txs []*types.Transaction + want string + }{ + { // ErrTxTypeNotSupported + txs: []*types.Transaction{ + mkDynamicTx(0, common.Address{}, params.TxGas-1000, big.NewInt(0), big.NewInt(0)), + }, + want: "could not apply tx 0 [0x21e9b9015150fc7f6bd5059890a5e1727f2452df285e8a84f4ca61a74c159ded]: transaction type not supported", + }, + } { + block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs, gspec.Config) + _, err := blockchain.InsertChain(types.Blocks{block}) + if err == nil { + t.Fatal("block imported without errors") + } + if have, want := err.Error(), tt.want; have != want { + t.Errorf("test %d:\nhave \"%v\"\nwant \"%v\"\n", i, have, want) + } } } } @@ -115,11 +198,11 @@ func TestStateProcessorErrors(t *testing.T) { // valid, and no proper post-state can be made. But from the perspective of the blockchain, the block is sufficiently // valid to be considered for import: // - valid pow (fake), ancestry, difficulty, gaslimit etc -func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Transactions) *types.Block { +func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Transactions, config *params.ChainConfig) *types.Block { header := &types.Header{ ParentHash: parent.Hash(), Coinbase: parent.Coinbase(), - Difficulty: engine.CalcDifficulty(&fakeChainReader{params.TestChainConfig}, parent.Time()+10, &types.Header{ + Difficulty: engine.CalcDifficulty(&fakeChainReader{config}, parent.Time()+10, &types.Header{ Number: parent.Number(), Time: parent.Time(), Difficulty: parent.Difficulty(), @@ -130,8 +213,10 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr Time: parent.Time() + 10, UncleHash: types.EmptyUncleHash, } + if config.IsLondon(header.Number) { + header.BaseFee = misc.CalcBaseFee(config, parent.Header()) + } var receipts []*types.Receipt - // The post-state result doesn't need to be correct (this is a bad block), but we do need something there // Preferably something unique. So let's use a combo of blocknum + txhash hasher := sha3.NewLegacyKeccak256() diff --git a/core/state_transition.go b/core/state_transition.go index 05becd9a00..881d34f4bc 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -22,6 +22,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" @@ -49,6 +50,8 @@ type StateTransition struct { msg Message gas uint64 gasPrice *big.Int + feeCap *big.Int + tip *big.Int initialGas uint64 value *big.Int data []byte @@ -62,6 +65,8 @@ type Message interface { To() *common.Address GasPrice() *big.Int + FeeCap() *big.Int + Tip() *big.Int Gas() uint64 Value() *big.Int @@ -154,6 +159,8 @@ func NewStateTransition(evm *vm.EVM, msg Message, gp *GasPool) *StateTransition evm: evm, msg: msg, gasPrice: msg.GasPrice(), + feeCap: msg.FeeCap(), + tip: msg.Tip(), value: msg.Value(), data: msg.Data(), state: evm.StateDB, @@ -206,6 +213,15 @@ func (st *StateTransition) preCheck() error { st.msg.From().Hex(), msgNonce, stNonce) } } + // Make sure that transaction feeCap is greater than the baseFee (post london) + if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + // This will panic if baseFee is nil, but basefee presence is verified + // as part of header validation. + if st.feeCap.Cmp(st.evm.Context.BaseFee) < 0 { + return fmt.Errorf("%w: address %v, feeCap: %s baseFee: %s", ErrFeeCapTooLow, + st.msg.From().Hex(), st.feeCap, st.evm.Context.BaseFee) + } + } return st.buyGas() } @@ -281,7 +297,11 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { // After EIP-3529: refunds are capped to gasUsed / 5 st.refundGas(params.RefundQuotientEIP3529) } - st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.gasPrice)) + effectiveTip := st.gasPrice + if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + effectiveTip = cmath.BigMin(st.tip, new(big.Int).Sub(st.feeCap, st.evm.Context.BaseFee)) + } + st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip)) return &ExecutionResult{ UsedGas: st.gasUsed(), diff --git a/core/types/access_list_tx.go b/core/types/access_list_tx.go index 65ee95adf6..48102a1d40 100644 --- a/core/types/access_list_tx.go +++ b/core/types/access_list_tx.go @@ -94,7 +94,6 @@ func (tx *AccessListTx) copy() TxData { } // accessors for innerTx. - func (tx *AccessListTx) txType() byte { return AccessListTxType } func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } func (tx *AccessListTx) protected() bool { return true } @@ -102,6 +101,8 @@ func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } func (tx *AccessListTx) data() []byte { return tx.Data } func (tx *AccessListTx) gas() uint64 { return tx.Gas } func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) tip() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) feeCap() *big.Int { return tx.GasPrice } func (tx *AccessListTx) value() *big.Int { return tx.Value } func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } func (tx *AccessListTx) to() *common.Address { return tx.To } diff --git a/core/types/block.go b/core/types/block.go index a3318f8779..5f3dbb957b 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -82,6 +82,9 @@ type Header struct { Extra []byte `json:"extraData" gencodec:"required"` MixDigest common.Hash `json:"mixHash"` Nonce BlockNonce `json:"nonce"` + + // BaseFee was added by EIP-1559 and is ignored in legacy headers. + BaseFee *big.Int `json:"baseFee" rlp:"optional"` } // field type overrides for gencodec @@ -92,6 +95,7 @@ type headerMarshaling struct { GasUsed hexutil.Uint64 Time hexutil.Uint64 Extra hexutil.Bytes + BaseFee *hexutil.Big Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON } @@ -229,6 +233,9 @@ func CopyHeader(h *Header) *Header { if cpy.Number = new(big.Int); h.Number != nil { cpy.Number.Set(h.Number) } + if h.BaseFee != nil { + cpy.BaseFee = new(big.Int).Set(h.BaseFee) + } if len(h.Extra) > 0 { cpy.Extra = make([]byte, len(h.Extra)) copy(cpy.Extra, h.Extra) @@ -289,6 +296,13 @@ func (b *Block) ReceiptHash() common.Hash { return b.header.ReceiptHash } func (b *Block) UncleHash() common.Hash { return b.header.UncleHash } func (b *Block) Extra() []byte { return common.CopyBytes(b.header.Extra) } +func (b *Block) BaseFee() *big.Int { + if b.header.BaseFee == nil { + return nil + } + return new(big.Int).Set(b.header.BaseFee) +} + func (b *Block) Header() *Header { return CopyHeader(b.header) } // Body returns the non-header content of the block. diff --git a/core/types/block_test.go b/core/types/block_test.go index 63904f882c..9ecb1a4d8f 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -68,6 +68,71 @@ func TestBlockEncoding(t *testing.T) { } } +func TestEIP1559BlockEncoding(t *testing.T) { + blockEnc := common.FromHex("f9030bf901fea083cafc574e1f51ba9dc0568fc617a08ea2429fb384059c972f13b19fa1c8dd55a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a05fe50b260da6308036625b850b5d6ced6d0a9f814c0688bc91ffb7b7a3a54b67a0bc37d79753ad738a6dac4921e57392f145d8887476de3f783dfa7edae9283e52b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd8825208845506eb0780a0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4843b9aca00f90106f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1b8a302f8a0018080843b9aca008301e24194095e7baea6a6c7c4c2dfeb977efac326af552d878080f838f7940000000000000000000000000000000000000001e1a0000000000000000000000000000000000000000000000000000000000000000080a0fe38ca4e44a30002ac54af7cf922a6ac2ba11b7d22f548e8ecb3f51f41cb31b0a06de6a5cbae13c0c856e33acf021b51819636cfc009d39eafb9f606d546e305a8c0") + var block Block + if err := rlp.DecodeBytes(blockEnc, &block); err != nil { + t.Fatal("decode error: ", err) + } + + check := func(f string, got, want interface{}) { + if !reflect.DeepEqual(got, want) { + t.Errorf("%s mismatch: got %v, want %v", f, got, want) + } + } + + check("Difficulty", block.Difficulty(), big.NewInt(131072)) + check("GasLimit", block.GasLimit(), uint64(3141592)) + check("GasUsed", block.GasUsed(), uint64(21000)) + check("Coinbase", block.Coinbase(), common.HexToAddress("8888f1f195afa192cfee860698584c030f4c9db1")) + check("MixDigest", block.MixDigest(), common.HexToHash("bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff498")) + check("Root", block.Root(), common.HexToHash("ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017")) + check("Hash", block.Hash(), common.HexToHash("c7252048cd273fe0dac09650027d07f0e3da4ee0675ebbb26627cea92729c372")) + check("Nonce", block.Nonce(), uint64(0xa13a5a8c8f2bb1c4)) + check("Time", block.Time(), uint64(1426516743)) + check("Size", block.Size(), common.StorageSize(len(blockEnc))) + check("BaseFee", block.BaseFee(), new(big.Int).SetUint64(params.InitialBaseFee)) + + tx1 := NewTransaction(0, common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87"), big.NewInt(10), 50000, big.NewInt(10), nil) + tx1, _ = tx1.WithSignature(HomesteadSigner{}, common.Hex2Bytes("9bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094f8a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b100")) + + addr := common.HexToAddress("0x0000000000000000000000000000000000000001") + accesses := AccessList{AccessTuple{ + Address: addr, + StorageKeys: []common.Hash{ + {0}, + }, + }} + to := common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87") + txdata := &DynamicFeeTx{ + ChainID: big.NewInt(1), + Nonce: 0, + To: &to, + Gas: 123457, + FeeCap: new(big.Int).Set(block.BaseFee()), + Tip: big.NewInt(0), + AccessList: accesses, + Data: []byte{}, + } + tx2 := NewTx(txdata) + tx2, err := tx2.WithSignature(LatestSignerForChainID(big.NewInt(1)), common.Hex2Bytes("fe38ca4e44a30002ac54af7cf922a6ac2ba11b7d22f548e8ecb3f51f41cb31b06de6a5cbae13c0c856e33acf021b51819636cfc009d39eafb9f606d546e305a800")) + if err != nil { + t.Fatal("invalid signature error: ", err) + } + + check("len(Transactions)", len(block.Transactions()), 2) + check("Transactions[0].Hash", block.Transactions()[0].Hash(), tx1.Hash()) + check("Transactions[1].Hash", block.Transactions()[1].Hash(), tx2.Hash()) + check("Transactions[1].Type", block.Transactions()[1].Type(), tx2.Type()) + ourBlockEnc, err := rlp.EncodeToBytes(&block) + if err != nil { + t.Fatal("encode error: ", err) + } + if !bytes.Equal(ourBlockEnc, blockEnc) { + t.Errorf("encoded block mismatch:\ngot: %x\nwant: %x", ourBlockEnc, blockEnc) + } +} + func TestEIP2718BlockEncoding(t *testing.T) { blockEnc := common.FromHex("f90319f90211a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a0e6e49996c7ec59f7a23d22b83239a60151512c65613bf84a0d7da336399ebc4aa0cafe75574d59780665a97fbfd11365c7545aa8f1abf4e5e12e8243334ef7286bb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000820200832fefd882a410845506eb0796636f6f6c65737420626c6f636b206f6e20636861696ea0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4f90101f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1b89e01f89b01800a8301e24194095e7baea6a6c7c4c2dfeb977efac326af552d878080f838f7940000000000000000000000000000000000000001e1a0000000000000000000000000000000000000000000000000000000000000000001a03dbacc8d0259f2508625e97fdfc57cd85fdd16e5821bc2c10bdd1a52649e8335a0476e10695b183a87b0aa292a7f4b78ef0c3fbe62aa2c42c84e1d9c3da159ef14c0") var block Block diff --git a/core/types/dynamic_fee_tx.go b/core/types/dynamic_fee_tx.go new file mode 100644 index 0000000000..777babe027 --- /dev/null +++ b/core/types/dynamic_fee_tx.go @@ -0,0 +1,104 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +type DynamicFeeTx struct { + ChainID *big.Int + Nonce uint64 + Tip *big.Int + FeeCap *big.Int + Gas uint64 + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int + Data []byte + AccessList AccessList + + // Signature values + V *big.Int `json:"v" gencodec:"required"` + R *big.Int `json:"r" gencodec:"required"` + S *big.Int `json:"s" gencodec:"required"` +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *DynamicFeeTx) copy() TxData { + cpy := &DynamicFeeTx{ + Nonce: tx.Nonce, + To: tx.To, // TODO: copy pointed-to address + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are copied below. + AccessList: make(AccessList, len(tx.AccessList)), + Value: new(big.Int), + ChainID: new(big.Int), + Tip: new(big.Int), + FeeCap: new(big.Int), + V: new(big.Int), + R: new(big.Int), + S: new(big.Int), + } + copy(cpy.AccessList, tx.AccessList) + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.ChainID != nil { + cpy.ChainID.Set(tx.ChainID) + } + if tx.Tip != nil { + cpy.Tip.Set(tx.Tip) + } + if tx.FeeCap != nil { + cpy.FeeCap.Set(tx.FeeCap) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. +func (tx *DynamicFeeTx) txType() byte { return DynamicFeeTxType } +func (tx *DynamicFeeTx) chainID() *big.Int { return tx.ChainID } +func (tx *DynamicFeeTx) protected() bool { return true } +func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } +func (tx *DynamicFeeTx) data() []byte { return tx.Data } +func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } +func (tx *DynamicFeeTx) feeCap() *big.Int { return tx.FeeCap } +func (tx *DynamicFeeTx) tip() *big.Int { return tx.Tip } +func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.FeeCap } +func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } +func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } +func (tx *DynamicFeeTx) to() *common.Address { return tx.To } + +func (tx *DynamicFeeTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V, tx.R, tx.S +} + +func (tx *DynamicFeeTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s +} diff --git a/core/types/legacy_tx.go b/core/types/legacy_tx.go index 41ad44f379..f5e0f70783 100644 --- a/core/types/legacy_tx.go +++ b/core/types/legacy_tx.go @@ -91,13 +91,14 @@ func (tx *LegacyTx) copy() TxData { } // accessors for innerTx. - func (tx *LegacyTx) txType() byte { return LegacyTxType } func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } func (tx *LegacyTx) accessList() AccessList { return nil } func (tx *LegacyTx) data() []byte { return tx.Data } func (tx *LegacyTx) gas() uint64 { return tx.Gas } func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) tip() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) feeCap() *big.Int { return tx.GasPrice } func (tx *LegacyTx) value() *big.Int { return tx.Value } func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } func (tx *LegacyTx) to() *common.Address { return tx.To } diff --git a/core/types/receipt.go b/core/types/receipt.go index 6b519a79d2..b949bd2bd5 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -120,10 +120,6 @@ func (r *Receipt) EncodeRLP(w io.Writer) error { if r.Type == LegacyTxType { return rlp.Encode(w, data) } - // It's an EIP-2718 typed TX receipt. - if r.Type != AccessListTxType { - return ErrTxTypeNotSupported - } buf := encodeBufferPool.Get().(*bytes.Buffer) defer encodeBufferPool.Put(buf) buf.Reset() @@ -159,7 +155,7 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error { return errEmptyTypedReceipt } r.Type = b[0] - if r.Type == AccessListTxType { + if r.Type == AccessListTxType || r.Type == DynamicFeeTxType { var dec receiptRLP if err := rlp.DecodeBytes(b[1:], &dec); err != nil { return err @@ -263,6 +259,9 @@ func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) { case AccessListTxType: w.WriteByte(AccessListTxType) rlp.Encode(w, data) + case DynamicFeeTxType: + w.WriteByte(DynamicFeeTxType) + rlp.Encode(w, data) default: // For unsupported types, write nothing. Since this is for // DeriveSha, the error will be caught matching the derived hash diff --git a/core/types/transaction.go b/core/types/transaction.go index a35e07a5a3..ace1843e93 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -26,6 +26,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -42,6 +43,7 @@ var ( const ( LegacyTxType = iota AccessListTxType + DynamicFeeTxType ) // Transaction is an Ethereum transaction. @@ -74,6 +76,8 @@ type TxData interface { data() []byte gas() uint64 gasPrice() *big.Int + tip() *big.Int + feeCap() *big.Int value() *big.Int nonce() uint64 to() *common.Address @@ -177,6 +181,10 @@ func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { var inner AccessListTx err := rlp.DecodeBytes(b[1:], &inner) return &inner, err + case DynamicFeeTxType: + var inner DynamicFeeTx + err := rlp.DecodeBytes(b[1:], &inner) + return &inner, err default: return nil, ErrTxTypeNotSupported } @@ -260,6 +268,12 @@ func (tx *Transaction) Gas() uint64 { return tx.inner.gas() } // GasPrice returns the gas price of the transaction. func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.inner.gasPrice()) } +// Tip returns the tip per gas of the transaction. +func (tx *Transaction) Tip() *big.Int { return new(big.Int).Set(tx.inner.tip()) } + +// FeeCap returns the fee cap per gas of the transaction. +func (tx *Transaction) FeeCap() *big.Int { return new(big.Int).Set(tx.inner.feeCap()) } + // Value returns the ether amount of the transaction. func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) } @@ -486,12 +500,14 @@ type Message struct { amount *big.Int gasLimit uint64 gasPrice *big.Int + feeCap *big.Int + tip *big.Int data []byte accessList AccessList checkNonce bool } -func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte, accessList AccessList, checkNonce bool) Message { +func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice, feeCap, tip *big.Int, data []byte, accessList AccessList, checkNonce bool) Message { return Message{ from: from, to: to, @@ -499,6 +515,8 @@ func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *b amount: amount, gasLimit: gasLimit, gasPrice: gasPrice, + feeCap: feeCap, + tip: tip, data: data, accessList: accessList, checkNonce: checkNonce, @@ -506,11 +524,13 @@ func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *b } // AsMessage returns the transaction as a core.Message. -func (tx *Transaction) AsMessage(s Signer) (Message, error) { +func (tx *Transaction) AsMessage(s Signer, baseFee *big.Int) (Message, error) { msg := Message{ nonce: tx.Nonce(), gasLimit: tx.Gas(), gasPrice: new(big.Int).Set(tx.GasPrice()), + feeCap: new(big.Int).Set(tx.FeeCap()), + tip: new(big.Int).Set(tx.Tip()), to: tx.To(), amount: tx.Value(), data: tx.Data(), @@ -518,6 +538,11 @@ func (tx *Transaction) AsMessage(s Signer) (Message, error) { checkNonce: true, } + // If baseFee provided, set gasPrice to effectiveGasPrice. + if baseFee != nil { + msg.gasPrice = math.BigMin(msg.gasPrice.Add(msg.tip, baseFee), msg.feeCap) + } + var err error msg.from, err = Sender(s, tx) return msg, err @@ -526,6 +551,8 @@ func (tx *Transaction) AsMessage(s Signer) (Message, error) { func (m Message) From() common.Address { return m.from } func (m Message) To() *common.Address { return m.to } func (m Message) GasPrice() *big.Int { return m.gasPrice } +func (m Message) FeeCap() *big.Int { return m.feeCap } +func (m Message) Tip() *big.Int { return m.tip } func (m Message) Value() *big.Int { return m.amount } func (m Message) Gas() uint64 { return m.gasLimit } func (m Message) Nonce() uint64 { return m.nonce } diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index e561485556..ecdbd70afa 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -30,15 +30,19 @@ type txJSON struct { Type hexutil.Uint64 `json:"type"` // Common transaction fields: - Nonce *hexutil.Uint64 `json:"nonce"` - GasPrice *hexutil.Big `json:"gasPrice"` - Gas *hexutil.Uint64 `json:"gas"` - Value *hexutil.Big `json:"value"` - Data *hexutil.Bytes `json:"input"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` - To *common.Address `json:"to"` + Nonce *hexutil.Uint64 `json:"nonce"` + GasPrice *hexutil.Big `json:"gasPrice"` + FeeCap *hexutil.Big `json:"feeCap"` + Tip *hexutil.Big `json:"tip"` + MaxPriorityFeePerGas *hexutil.Big `json:"maxPriorityFeePerGas"` + MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"` + Gas *hexutil.Uint64 `json:"gas"` + Value *hexutil.Big `json:"value"` + Data *hexutil.Bytes `json:"input"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + To *common.Address `json:"to"` // Access list transaction fields: ChainID *hexutil.Big `json:"chainId,omitempty"` @@ -79,6 +83,19 @@ func (t *Transaction) MarshalJSON() ([]byte, error) { enc.V = (*hexutil.Big)(tx.V) enc.R = (*hexutil.Big)(tx.R) enc.S = (*hexutil.Big)(tx.S) + case *DynamicFeeTx: + enc.ChainID = (*hexutil.Big)(tx.ChainID) + enc.AccessList = &tx.AccessList + enc.Nonce = (*hexutil.Uint64)(&tx.Nonce) + enc.Gas = (*hexutil.Uint64)(&tx.Gas) + enc.FeeCap = (*hexutil.Big)(tx.FeeCap) + enc.Tip = (*hexutil.Big)(tx.Tip) + enc.Value = (*hexutil.Big)(tx.Value) + enc.Data = (*hexutil.Bytes)(&tx.Data) + enc.To = t.To() + enc.V = (*hexutil.Big)(tx.V) + enc.R = (*hexutil.Big)(tx.R) + enc.S = (*hexutil.Big)(tx.S) } return json.Marshal(&enc) } @@ -191,6 +208,75 @@ func (t *Transaction) UnmarshalJSON(input []byte) error { } } + case DynamicFeeTxType: + var itx DynamicFeeTx + inner = &itx + // Access list is optional for now. + if dec.AccessList != nil { + itx.AccessList = *dec.AccessList + } + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' in transaction") + } + itx.ChainID = (*big.Int)(dec.ChainID) + if dec.To != nil { + itx.To = dec.To + } + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + switch { + case dec.Tip == nil && dec.MaxPriorityFeePerGas == nil: + return errors.New("at least one of 'tip' or 'maxPriorityFeePerGas' must be defined") + case dec.Tip != nil && dec.MaxPriorityFeePerGas != nil: + return errors.New("only one of 'tip' or 'maxPriorityFeePerGas' may be defined") + case dec.Tip != nil && dec.MaxPriorityFeePerGas == nil: + itx.Tip = (*big.Int)(dec.Tip) + case dec.Tip == nil && dec.MaxPriorityFeePerGas != nil: + itx.Tip = (*big.Int)(dec.MaxPriorityFeePerGas) + } + switch { + case dec.FeeCap == nil && dec.MaxFeePerGas == nil: + return errors.New("at least one of 'feeCap' or 'maxFeePerGas' must be defined") + case dec.FeeCap != nil && dec.MaxFeePerGas != nil: + return errors.New("only one of 'feeCap' or 'maxFeePerGas' may be defined") + case dec.FeeCap != nil && dec.MaxFeePerGas == nil: + itx.FeeCap = (*big.Int)(dec.FeeCap) + case dec.FeeCap == nil && dec.MaxFeePerGas != nil: + itx.FeeCap = (*big.Int)(dec.MaxFeePerGas) + } + if dec.Gas == nil { + return errors.New("missing required field 'gas' for txdata") + } + itx.Gas = uint64(*dec.Gas) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = (*big.Int)(dec.Value) + if dec.Data == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Data + if dec.V == nil { + return errors.New("missing required field 'v' in transaction") + } + itx.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S = (*big.Int)(dec.S) + withSignature := itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 + if withSignature { + if err := sanityCheckSignature(itx.V, itx.R, itx.S, false); err != nil { + return err + } + } + default: return ErrTxTypeNotSupported } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 5d94b26b36..e553e6af96 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -40,6 +40,8 @@ type sigCache struct { func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { var signer Signer switch { + case config.IsLondon(blockNumber): + signer = NewLondonSigner(config.ChainID) case config.IsBerlin(blockNumber): signer = NewEIP2930Signer(config.ChainID) case config.IsEIP155(blockNumber): @@ -61,6 +63,9 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { // have the current block number available, use MakeSigner instead. func LatestSigner(config *params.ChainConfig) Signer { if config.ChainID != nil { + if config.LondonBlock != nil { + return NewLondonSigner(config.ChainID) + } if config.BerlinBlock != nil { return NewEIP2930Signer(config.ChainID) } @@ -82,7 +87,7 @@ func LatestSignerForChainID(chainID *big.Int) Signer { if chainID == nil { return HomesteadSigner{} } - return NewEIP2930Signer(chainID) + return NewLondonSigner(chainID) } // SignTx signs the transaction using the given signer and private key. @@ -165,6 +170,72 @@ type Signer interface { Equal(Signer) bool } +type londonSigner struct{ eip2930Signer } + +// NewLondonSigner returns a signer that accepts +// - EIP-1559 dynamic fee transactions +// - EIP-2930 access list transactions, +// - EIP-155 replay protected transactions, and +// - legacy Homestead transactions. +func NewLondonSigner(chainId *big.Int) Signer { + return londonSigner{eip2930Signer{NewEIP155Signer(chainId)}} +} + +func (s londonSigner) Sender(tx *Transaction) (common.Address, error) { + if tx.Type() != DynamicFeeTxType { + return s.eip2930Signer.Sender(tx) + } + V, R, S := tx.RawSignatureValues() + // DynamicFee txs are defined to use 0 and 1 as their recovery + // id, add 27 to become equivalent to unprotected Homestead signatures. + V = new(big.Int).Add(V, big.NewInt(27)) + if tx.ChainId().Cmp(s.chainId) != 0 { + return common.Address{}, ErrInvalidChainId + } + return recoverPlain(s.Hash(tx), R, S, V, true) +} + +func (s londonSigner) Equal(s2 Signer) bool { + x, ok := s2.(londonSigner) + return ok && x.chainId.Cmp(s.chainId) == 0 +} + +func (s londonSigner) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { + txdata, ok := tx.inner.(*DynamicFeeTx) + if !ok { + return s.eip2930Signer.SignatureValues(tx, sig) + } + // Check that chain ID of tx matches the signer. We also accept ID zero here, + // because it indicates that the chain ID was not specified in the tx. + if txdata.ChainID.Sign() != 0 && txdata.ChainID.Cmp(s.chainId) != 0 { + return nil, nil, nil, ErrInvalidChainId + } + R, S, _ = decodeSignature(sig) + V = big.NewInt(int64(sig[64])) + return R, S, V, nil +} + +// Hash returns the hash to be signed by the sender. +// It does not uniquely identify the transaction. +func (s londonSigner) Hash(tx *Transaction) common.Hash { + if tx.Type() != DynamicFeeTxType { + return s.eip2930Signer.Hash(tx) + } + return prefixedRlpHash( + tx.Type(), + []interface{}{ + s.chainId, + tx.Nonce(), + tx.Tip(), + tx.FeeCap(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + tx.AccessList(), + }) +} + type eip2930Signer struct{ EIP155Signer } // NewEIP2930Signer returns a signer that accepts EIP-2930 access list transactions, @@ -192,8 +263,8 @@ func (s eip2930Signer) Sender(tx *Transaction) (common.Address, error) { V = new(big.Int).Sub(V, s.chainIdMul) V.Sub(V, big8) case AccessListTxType: - // ACL txs are defined to use 0 and 1 as their recovery id, add - // 27 to become equivalent to unprotected Homestead signatures. + // AL txs are defined to use 0 and 1 as their recovery + // id, add 27 to become equivalent to unprotected Homestead signatures. V = new(big.Int).Add(V, big.NewInt(27)) default: return common.Address{}, ErrTxTypeNotSupported diff --git a/core/vm/eips.go b/core/vm/eips.go index 025502760b..4070a2db53 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -26,6 +26,7 @@ import ( var activators = map[int]func(*JumpTable){ 3529: enable3529, + 3198: enable3198, 2929: enable2929, 2200: enable2200, 1884: enable1884, @@ -154,3 +155,22 @@ func enable3529(jt *JumpTable) { jt[SSTORE].dynamicGas = gasSStoreEIP3529 jt[SELFDESTRUCT].dynamicGas = gasSelfdestructEIP3529 } + +// enable3198 applies EIP-3198 (BASEFEE Opcode) +// - Adds an opcode that returns the current block's base fee. +func enable3198(jt *JumpTable) { + // New opcode + jt[BASEFEE] = &operation{ + execute: opBaseFee, + constantGas: GasQuickStep, + minStack: minStack(0, 1), + maxStack: maxStack(0, 1), + } +} + +// opBaseFee implements BASEFEE opcode +func opBaseFee(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + baseFee, _ := uint256.FromBig(interpreter.evm.Context.BaseFee) + scope.Stack.push(baseFee) + return nil, nil +} diff --git a/core/vm/evm.go b/core/vm/evm.go index 8e3c9fe00f..980dc6d201 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -93,6 +93,7 @@ type BlockContext struct { BlockNumber *big.Int // Provides information for NUMBER Time *big.Int // Provides information for TIME Difficulty *big.Int // Provides information for DIFFICULTY + BaseFee *big.Int // Provides information for BASEFEE } // TxContext provides the EVM with information about a transaction. diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index a0609a0d78..329ad77cbf 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -68,6 +68,7 @@ type JumpTable [256]*operation func newLondonInstructionSet() JumpTable { instructionSet := newBerlinInstructionSet() enable3529(&instructionSet) // EIP-3529: Reduction in refunds https://eips.ethereum.org/EIPS/eip-3529 + enable3198(&instructionSet) // Base fee opcode https://eips.ethereum.org/EIPS/eip-3198 return instructionSet } diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index b0adf37d0c..286307ae91 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -103,6 +103,7 @@ const ( GASLIMIT CHAINID OpCode = 0x46 SELFBALANCE OpCode = 0x47 + BASEFEE OpCode = 0x48 ) // 0x50 range - 'storage' and execution. @@ -280,6 +281,7 @@ var opCodeToString = map[OpCode]string{ GASLIMIT: "GASLIMIT", CHAINID: "CHAINID", SELFBALANCE: "SELFBALANCE", + BASEFEE: "BASEFEE", // 0x50 range - 'storage' and execution. POP: "POP", @@ -432,6 +434,7 @@ var stringToOp = map[string]OpCode{ "CALLDATASIZE": CALLDATASIZE, "CALLDATACOPY": CALLDATACOPY, "CHAINID": CHAINID, + "BASEFEE": BASEFEE, "DELEGATECALL": DELEGATECALL, "STATICCALL": STATICCALL, "CODESIZE": CODESIZE, diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 84cfaf4d73..8d53739721 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -162,7 +162,7 @@ func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec signer := types.MakeSigner(eth.blockchain.Config(), block.Number()) for idx, tx := range block.Transactions() { // Assemble the transaction call message and return if the requested offset - msg, _ := tx.AsMessage(signer) + msg, _ := tx.AsMessage(signer, block.BaseFee()) txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) if idx == txIndex { diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 1c727f1366..172054e9b8 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -271,7 +271,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(localctx), nil) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { - msg, _ := tx.AsMessage(signer) + msg, _ := tx.AsMessage(signer, task.block.BaseFee()) txctx := &txTraceContext{ index: i, hash: tx.Hash(), @@ -523,7 +523,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac defer pend.Done() // Fetch and execute the next transaction trace tasks for task := range jobs { - msg, _ := txs[task.index].AsMessage(signer) + msg, _ := txs[task.index].AsMessage(signer, block.BaseFee()) txctx := &txTraceContext{ index: task.index, hash: txs[task.index].Hash(), @@ -545,7 +545,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac jobs <- &txTraceTask{statedb: statedb.Copy(), index: i} // Generate the next state snapshot fast without tracing - msg, _ := tx.AsMessage(signer) + msg, _ := tx.AsMessage(signer, block.BaseFee()) statedb.Prepare(tx.Hash(), block.Hash(), i) vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { @@ -630,7 +630,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block for i, tx := range block.Transactions() { // Prepare the trasaction for un-traced execution var ( - msg, _ = tx.AsMessage(signer) + msg, _ = tx.AsMessage(signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) vmConf vm.Config dump *os.File diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 4c0240cd2c..24bce320cf 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -161,7 +161,7 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block // Recompute transactions up to the target index. signer := types.MakeSigner(b.chainConfig, block.Number()) for idx, tx := range block.Transactions() { - msg, _ := tx.AsMessage(signer) + msg, _ := tx.AsMessage(signer, block.BaseFee()) txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), b.chain, nil) if idx == txIndex { diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 9dc4c69631..8b01edd7b4 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -179,7 +179,7 @@ func TestPrestateTracerCreate2(t *testing.T) { } evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Debug: true, Tracer: tracer}) - msg, err := tx.AsMessage(signer) + msg, err := tx.AsMessage(signer, nil) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } @@ -254,7 +254,7 @@ func TestCallTracer(t *testing.T) { } evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Debug: true, Tracer: tracer}) - msg, err := tx.AsMessage(signer) + msg, err := tx.AsMessage(signer, nil) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } diff --git a/interfaces.go b/interfaces.go index afcdc17e58..857d309be9 100644 --- a/interfaces.go +++ b/interfaces.go @@ -120,6 +120,9 @@ type CallMsg struct { Value *big.Int // amount of wei sent along with the call Data []byte // input data, usually an ABI-encoded contract method invocation + FeeCap *big.Int // EIP-1559 fee cap per gas. + Tip *big.Int // EIP-1559 tip per gas. + AccessList types.AccessList // EIP-2930 access list. } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index fe3f80c038..7bc0477bd2 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -799,7 +799,7 @@ func (args *CallArgs) ToMessage(globalGasCap uint64) types.Message { accessList = *args.AccessList } - msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, data, accessList, false) + msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, nil, nil, data, accessList, false) return msg } @@ -1271,7 +1271,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) result.TransactionIndex = (*hexutil.Uint64)(&index) } - if tx.Type() == types.AccessListTxType { + if tx.Type() != types.LegacyTxType { al := tx.AccessList() result.Accesses = &al result.ChainID = (*hexutil.Big)(tx.ChainId()) @@ -1393,7 +1393,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH } // Copy the original db so we don't modify it statedb := db.Copy() - msg := types.NewMessage(args.From, args.To, uint64(*args.Nonce), args.Value.ToInt(), uint64(*args.Gas), args.GasPrice.ToInt(), input, accessList, false) + msg := types.NewMessage(args.From, args.To, uint64(*args.Nonce), args.Value.ToInt(), uint64(*args.Gas), args.GasPrice.ToInt(), nil, nil, input, accessList, false) // Apply the transaction with the access list tracer tracer := vm.NewAccessListTracer(accessList, args.From, to, precompiles) diff --git a/les/odr_test.go b/les/odr_test.go index 0c75014d49..5fb881b5c4 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -135,7 +135,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai from := statedb.GetOrNewStateObject(bankAddr) from.SetBalance(math.MaxBig256) - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, new(big.Int), nil, nil, data, nil, false)} context := core.NewEVMBlockContext(header, bc, nil) txContext := core.NewEVMTxContext(msg) @@ -150,7 +150,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai header := lc.GetHeaderByHash(bhash) state := light.NewState(ctx, header, lc.Odr()) state.SetBalance(bankAddr, math.MaxBig256) - msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, new(big.Int), nil, nil, data, nil, false)} context := core.NewEVMBlockContext(header, lc, nil) txContext := core.NewEVMTxContext(msg) vmenv := vm.NewEVM(context, txContext, state, config, vm.Config{}) diff --git a/les/state_accessor.go b/les/state_accessor.go index af5df36508..e276b06dc7 100644 --- a/les/state_accessor.go +++ b/les/state_accessor.go @@ -55,7 +55,7 @@ func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types. signer := types.MakeSigner(leth.blockchain.Config(), block.Number()) for idx, tx := range block.Transactions() { // Assemble the transaction call message and return if the requested offset - msg, _ := tx.AsMessage(signer) + msg, _ := tx.AsMessage(signer, block.BaseFee()) txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), leth.blockchain, nil) statedb.Prepare(tx.Hash(), block.Hash(), idx) diff --git a/light/odr_test.go b/light/odr_test.go index 0fc45b8734..bb47c69eb1 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -194,7 +194,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain // Perform read-only call. st.SetBalance(testBankAddress, math.MaxBig256) - msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, new(big.Int), nil, nil, data, nil, false)} txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, chain, nil) vmenv := vm.NewEVM(context, txContext, st, config, vm.Config{}) diff --git a/params/bootnodes.go b/params/bootnodes.go index 20bf0b7cbf..99068750fc 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -69,9 +69,8 @@ var GoerliBootnodes = []string{ // BaikalBootnodes are the enode URLs of the P2P bootstrap nodes running on the // Baikal ephemeral test network. -// TODO: Set Baikal bootnodes var BaikalBootnodes = []string{ - "", + "enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303", } var V5Bootnodes = []string{ diff --git a/params/config.go b/params/config.go index 2cafa0e449..c777172381 100644 --- a/params/config.go +++ b/params/config.go @@ -231,7 +231,7 @@ var ( IstanbulBlock: big.NewInt(0), MuirGlacierBlock: nil, BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), + LondonBlock: big.NewInt(500), Clique: &CliqueConfig{ Period: 30, Epoch: 30000, diff --git a/params/protocol_params.go b/params/protocol_params.go index 22b4c0651c..a49c4489f1 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -118,6 +118,10 @@ const ( // Introduced in Tangerine Whistle (Eip 150) CreateBySelfdestructGas uint64 = 25000 + BaseFeeChangeDenominator = 8 // Bounds the amount the base fee can change between blocks. + ElasticityMultiplier = 2 // Bounds the maximum gas limit an EIP-1559 block may have. + InitialBaseFee = 1000000000 // Initial base fee for EIP-1559 blocks. + MaxCodeSize = 24576 // Maximum bytecode to permit for a contract // Precompiled contract gas prices diff --git a/tests/init.go b/tests/init.go index 240b7159d5..b0a38e68b0 100644 --- a/tests/init.go +++ b/tests/init.go @@ -179,6 +179,19 @@ var Forks = map[string]*params.ChainConfig{ BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(0), }, + "Aleut": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + }, } // Returns the set of defined fork names diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 46834de6da..9778f058fe 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -297,7 +297,7 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { if tx.AccessLists != nil && tx.AccessLists[ps.Indexes.Data] != nil { accessList = *tx.AccessLists[ps.Indexes.Data] } - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, accessList, true) + msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, nil, nil, data, accessList, true) return msg, nil } From 67e7f61af7fffbc47aadf48777e2dd5da39796ff Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 18 May 2021 01:10:28 +0200 Subject: [PATCH 366/709] core: fix failing tests (#22888) This PR fixes two errors that regressed when EIP-1559 was merged. --- core/blockchain_test.go | 4 ++-- core/state_processor_test.go | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 8ace9f6a6d..1869b0f183 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1357,8 +1357,8 @@ func TestEIP155Transition(t *testing.T) { } }) _, err := blockchain.InsertChain(blocks) - if err != types.ErrInvalidChainId { - t.Error("expected error:", types.ErrInvalidChainId) + if have, want := err, types.ErrInvalidChainId; !errors.Is(have, want) { + t.Errorf("have %v, want %v", have, want) } } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 28baf6e7d2..c15d3d276d 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -39,7 +39,22 @@ import ( // contain invalid transactions func TestStateProcessorErrors(t *testing.T) { var ( - signer = types.LatestSigner(params.TestChainConfig) + config = ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), + } + signer = types.LatestSigner(config) testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") ) var makeTx = func(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *types.Transaction { @@ -61,7 +76,7 @@ func TestStateProcessorErrors(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() gspec = &Genesis{ - Config: params.TestChainConfig, + Config: config, Alloc: GenesisAlloc{ common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ Balance: big.NewInt(1000000000000000000), // 1 ether From bb9f9ccf4fdc1256a31abe634ab882baa6f1c888 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 18 May 2021 01:30:01 +0200 Subject: [PATCH 367/709] core/rawdb: wait for background freezing to exit when closing freezer (#22878) --- core/rawdb/database.go | 6 +++++- core/rawdb/freezer.go | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 3a0a26c61d..698f1ced84 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -197,7 +197,11 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st } // Freezer is consistent with the key-value database, permit combining the two if !frdb.readonly { - go frdb.freeze(db) + frdb.wg.Add(1) + go func() { + frdb.freeze(db) + frdb.wg.Done() + }() } return &freezerdb{ KeyValueStore: db, diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 94b99a64eb..ff8919b59e 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -84,6 +84,7 @@ type freezer struct { trigger chan chan struct{} // Manual blocking freeze trigger, test determinism quit chan struct{} + wg sync.WaitGroup closeOnce sync.Once } @@ -145,6 +146,8 @@ func (f *freezer) Close() error { var errs []error f.closeOnce.Do(func() { close(f.quit) + // Wait for any background freezing to stop + f.wg.Wait() for _, table := range f.tables { if err := table.Close(); err != nil { errs = append(errs, err) From b7a91663ab93abe360ae1b4bc68b75b5944625c4 Mon Sep 17 00:00:00 2001 From: Evolution404 <35091674+Evolution404@users.noreply.github.com> Date: Tue, 18 May 2021 16:22:58 +0800 Subject: [PATCH 368/709] core/asm: fix the bug of "00" prefix number (#22883) --- core/asm/lex_test.go | 4 ++++ core/asm/lexer.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/asm/lex_test.go b/core/asm/lex_test.go index 6b8bd3d740..53e05fbbba 100644 --- a/core/asm/lex_test.go +++ b/core/asm/lex_test.go @@ -60,6 +60,10 @@ func TestLexer(t *testing.T) { input: "0123abc", tokens: []token{{typ: lineStart}, {typ: number, text: "0123"}, {typ: element, text: "abc"}, {typ: eof}}, }, + { + input: "00123abc", + tokens: []token{{typ: lineStart}, {typ: number, text: "00123"}, {typ: element, text: "abc"}, {typ: eof}}, + }, { input: "@foo", tokens: []token{{typ: lineStart}, {typ: label, text: "foo"}, {typ: eof}}, diff --git a/core/asm/lexer.go b/core/asm/lexer.go index 9eb8f914ac..21cc8c4658 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -254,7 +254,7 @@ func lexInsideString(l *lexer) stateFn { func lexNumber(l *lexer) stateFn { acceptance := Numbers - if l.accept("0") || l.accept("xX") { + if l.accept("xX") { acceptance = HexadecimalNumbers } l.acceptRun(acceptance) From 32c1ed8a9c963dbf75faa80116b37f8cf53d4a38 Mon Sep 17 00:00:00 2001 From: Shane Bammel Date: Tue, 18 May 2021 03:37:18 -0500 Subject: [PATCH 369/709] core/forkid: fix off-by-one bug (#22879) * forkid: added failing test * forkid: fixed off-by-one bug --- core/forkid/forkid.go | 2 +- core/forkid/forkid_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go index 1bf3406828..f56ce85fee 100644 --- a/core/forkid/forkid.go +++ b/core/forkid/forkid.go @@ -155,7 +155,7 @@ func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() ui for i, fork := range forks { // If our head is beyond this fork, continue to the next (we have a dummy // fork of maxuint64 as the last item to always fail this check eventually). - if head > fork { + if head >= fork { continue } // Found the first unpassed fork block, check if our current state matches diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index a20598fa9d..2a7938bd29 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -163,6 +163,10 @@ func TestValidation(t *testing.T) { // neither forks passed at neither nodes, they may mismatch, but we still connect for now. {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: math.MaxUint64}, nil}, + // Local is mainnet exactly on Petersburg, remote announces Byzantium + knowledge about Petersburg. Remote + // is simply out of sync, accept. + {7280000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}, nil}, + // Local is mainnet Petersburg, remote announces Byzantium + knowledge about Petersburg. Remote // is simply out of sync, accept. {7987396, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}, nil}, From 3e6f46caec51d82aef363632517eb5842eef6db6 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 18 May 2021 11:48:41 +0200 Subject: [PATCH 370/709] p2p/discover/v4wire: use optional RLP field for EIP-868 seq (#22842) This changes the definitions of Ping and Pong, adding an optional field for the sequence number. This field was previously encoded/decoded using the "tail" struct tag, but using "optional" is much nicer. --- p2p/discover/v4_udp.go | 9 +++------ p2p/discover/v4_udp_test.go | 8 ++++---- p2p/discover/v4wire/v4wire.go | 23 ++++++++--------------- p2p/discover/v4wire/v4wire_test.go | 24 ++---------------------- 4 files changed, 17 insertions(+), 47 deletions(-) diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index ad23eee6b4..2b3eb48391 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -34,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover/v4wire" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/netutil" - "github.com/ethereum/go-ethereum/rlp" ) // Errors @@ -217,7 +216,7 @@ func (t *UDPv4) Ping(n *enode.Node) error { func (t *UDPv4) ping(n *enode.Node) (seq uint64, err error) { rm := t.sendPing(n.ID(), &net.UDPAddr{IP: n.IP(), Port: n.UDP()}, nil) if err = <-rm.errc; err == nil { - seq = rm.reply.(*v4wire.Pong).ENRSeq() + seq = rm.reply.(*v4wire.Pong).ENRSeq } return seq, err } @@ -248,13 +247,12 @@ func (t *UDPv4) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) *r } func (t *UDPv4) makePing(toaddr *net.UDPAddr) *v4wire.Ping { - seq, _ := rlp.EncodeToBytes(t.localNode.Node().Seq()) return &v4wire.Ping{ Version: 4, From: t.ourEndpoint(), To: v4wire.NewEndpoint(toaddr, 0), Expiration: uint64(time.Now().Add(expiration).Unix()), - Rest: []rlp.RawValue{seq}, + ENRSeq: t.localNode.Node().Seq(), } } @@ -660,12 +658,11 @@ func (t *UDPv4) handlePing(h *packetHandlerV4, from *net.UDPAddr, fromID enode.I req := h.Packet.(*v4wire.Ping) // Reply. - seq, _ := rlp.EncodeToBytes(t.localNode.Node().Seq()) t.send(from, fromID, &v4wire.Pong{ To: v4wire.NewEndpoint(from, req.From.TCP), ReplyTok: mac, Expiration: uint64(time.Now().Add(expiration).Unix()), - Rest: []rlp.RawValue{seq}, + ENRSeq: t.localNode.Node().Seq(), }) // Ping back if our last pong on file is too far in the past. diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 262e3f0ba3..e36912f010 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -470,13 +470,13 @@ func TestUDPv4_EIP868(t *testing.T) { // Perform endpoint proof and check for sequence number in packet tail. test.packetIn(nil, &v4wire.Ping{Expiration: futureExp}) test.waitPacketOut(func(p *v4wire.Pong, addr *net.UDPAddr, hash []byte) { - if p.ENRSeq() != wantNode.Seq() { - t.Errorf("wrong sequence number in pong: %d, want %d", p.ENRSeq(), wantNode.Seq()) + if p.ENRSeq != wantNode.Seq() { + t.Errorf("wrong sequence number in pong: %d, want %d", p.ENRSeq, wantNode.Seq()) } }) test.waitPacketOut(func(p *v4wire.Ping, addr *net.UDPAddr, hash []byte) { - if p.ENRSeq() != wantNode.Seq() { - t.Errorf("wrong sequence number in ping: %d, want %d", p.ENRSeq(), wantNode.Seq()) + if p.ENRSeq != wantNode.Seq() { + t.Errorf("wrong sequence number in ping: %d, want %d", p.ENRSeq, wantNode.Seq()) } test.packetIn(nil, &v4wire.Pong{Expiration: futureExp, ReplyTok: hash}) }) diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index b5dcb6e517..23e7134414 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -50,6 +50,8 @@ type ( Version uint From, To Endpoint Expiration uint64 + ENRSeq uint64 `rlp:"optional"` // Sequence number of local record, added by EIP-868. + // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } @@ -62,6 +64,8 @@ type ( To Endpoint ReplyTok []byte // This contains the hash of the ping packet. Expiration uint64 // Absolute timestamp at which the packet becomes invalid. + ENRSeq uint64 `rlp:"optional"` // Sequence number of local record, added by EIP-868. + // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } @@ -162,13 +166,11 @@ type Packet interface { Kind() byte } -func (req *Ping) Name() string { return "PING/v4" } -func (req *Ping) Kind() byte { return PingPacket } -func (req *Ping) ENRSeq() uint64 { return seqFromTail(req.Rest) } +func (req *Ping) Name() string { return "PING/v4" } +func (req *Ping) Kind() byte { return PingPacket } -func (req *Pong) Name() string { return "PONG/v4" } -func (req *Pong) Kind() byte { return PongPacket } -func (req *Pong) ENRSeq() uint64 { return seqFromTail(req.Rest) } +func (req *Pong) Name() string { return "PONG/v4" } +func (req *Pong) Kind() byte { return PongPacket } func (req *Findnode) Name() string { return "FINDNODE/v4" } func (req *Findnode) Kind() byte { return FindnodePacket } @@ -187,15 +189,6 @@ func Expired(ts uint64) bool { return time.Unix(int64(ts), 0).Before(time.Now()) } -func seqFromTail(tail []rlp.RawValue) uint64 { - if len(tail) == 0 { - return 0 - } - var seq uint64 - rlp.DecodeBytes(tail[0], &seq) - return seq -} - // Encoder/decoder. const ( diff --git a/p2p/discover/v4wire/v4wire_test.go b/p2p/discover/v4wire/v4wire_test.go index 4dddeadd20..3b4161998d 100644 --- a/p2p/discover/v4wire/v4wire_test.go +++ b/p2p/discover/v4wire/v4wire_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/davecgh/go-spew/spew" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -40,7 +39,6 @@ var testPackets = []struct { From: Endpoint{net.ParseIP("127.0.0.1").To4(), 3322, 5544}, To: Endpoint{net.ParseIP("::1"), 2222, 3333}, Expiration: 1136239445, - Rest: []rlp.RawValue{}, }, }, { @@ -50,26 +48,8 @@ var testPackets = []struct { From: Endpoint{net.ParseIP("127.0.0.1").To4(), 3322, 5544}, To: Endpoint{net.ParseIP("::1"), 2222, 3333}, Expiration: 1136239445, - Rest: []rlp.RawValue{{0x01}, {0x02}}, - }, - }, - { - input: "577be4349c4dd26768081f58de4c6f375a7a22f3f7adda654d1428637412c3d7fe917cadc56d4e5e7ffae1dbe3efffb9849feb71b262de37977e7c7a44e677295680e9e38ab26bee2fcbae207fba3ff3d74069a50b902a82c9903ed37cc993c50001f83e82022bd79020010db83c4d001500000000abcdef12820cfa8215a8d79020010db885a308d313198a2e037073488208ae82823a8443b9a355c5010203040531b9019afde696e582a78fa8d95ea13ce3297d4afb8ba6433e4154caa5ac6431af1b80ba76023fa4090c408f6b4bc3701562c031041d4702971d102c9ab7fa5eed4cd6bab8f7af956f7d565ee1917084a95398b6a21eac920fe3dd1345ec0a7ef39367ee69ddf092cbfe5b93e5e568ebc491983c09c76d922dc3", - wantPacket: &Ping{ - Version: 555, - From: Endpoint{net.ParseIP("2001:db8:3c4d:15::abcd:ef12"), 3322, 5544}, - To: Endpoint{net.ParseIP("2001:db8:85a3:8d3:1319:8a2e:370:7348"), 2222, 33338}, - Expiration: 1136239445, - Rest: []rlp.RawValue{{0xC5, 0x01, 0x02, 0x03, 0x04, 0x05}}, - }, - }, - { - input: "09b2428d83348d27cdf7064ad9024f526cebc19e4958f0fdad87c15eb598dd61d08423e0bf66b2069869e1724125f820d851c136684082774f870e614d95a2855d000f05d1648b2d5945470bc187c2d2216fbe870f43ed0909009882e176a46b0102f846d79020010db885a308d313198a2e037073488208ae82823aa0fbc914b16819237dcd8801d7e53f69e9719adecb3cc0e790c57e91ca4461c9548443b9a355c6010203c2040506a0c969a58f6f9095004c0177a6b47f451530cab38966a25cca5cb58f055542124e", - wantPacket: &Pong{ - To: Endpoint{net.ParseIP("2001:db8:85a3:8d3:1319:8a2e:370:7348"), 2222, 33338}, - ReplyTok: common.Hex2Bytes("fbc914b16819237dcd8801d7e53f69e9719adecb3cc0e790c57e91ca4461c954"), - Expiration: 1136239445, - Rest: []rlp.RawValue{{0xC6, 0x01, 0x02, 0x03, 0xC2, 0x04, 0x05}, {0x06}}, + ENRSeq: 1, + Rest: []rlp.RawValue{{0x02}}, }, }, { From 088da24ebfb17a50652d4f9c3657670abcf055c8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 18 May 2021 12:10:27 +0200 Subject: [PATCH 371/709] rlp: improve decoder stream implementation (#22858) This commit makes various cleanup changes to rlp.Stream. * rlp: shrink Stream struct This removes a lot of unused padding space in Stream by reordering the fields. The size of Stream changes from 120 bytes to 88 bytes. Stream instances are internally cached and reused using sync.Pool, so this does not improve performance. * rlp: simplify list stack The list stack kept track of the size of the current list context as well as the current offset into it. The size had to be stored in the stack in order to subtract it from the remaining bytes of any enclosing list in ListEnd. It seems that this can be implemented in a simpler way: just subtract the size from the enclosing list context in List instead. --- rlp/decode.go | 175 +++++++++++++++++++++++++------------------------- 1 file changed, 86 insertions(+), 89 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index b340aa029e..9767809717 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -518,7 +518,7 @@ func decodeDecoder(s *Stream, val reflect.Value) error { } // Kind represents the kind of value contained in an RLP stream. -type Kind int +type Kind int8 const ( Byte Kind = iota @@ -561,22 +561,16 @@ type ByteReader interface { type Stream struct { r ByteReader - // number of bytes remaining to be read from r. - remaining uint64 - limited bool - - // auxiliary buffer for integer decoding - uintbuf []byte - - kind Kind // kind of value ahead - size uint64 // size of value ahead - byteval byte // value of single byte in type tag - kinderr error // error from last readKind - stack []listpos + remaining uint64 // number of bytes remaining to be read from r + size uint64 // size of value ahead + kinderr error // error from last readKind + stack []uint64 // list sizes + uintbuf [8]byte // auxiliary buffer for integer decoding + kind Kind // kind of value ahead + byteval byte // value of single byte in type tag + limited bool // true if input limit is in effect } -type listpos struct{ pos, size uint64 } - // NewStream creates a new decoding stream reading from r. // // If r implements the ByteReader interface, Stream will @@ -646,8 +640,8 @@ func (s *Stream) Raw() ([]byte, error) { s.kind = -1 // rearm Kind return []byte{s.byteval}, nil } - // the original header has already been read and is no longer - // available. read content and put a new header in front of it. + // The original header has already been read and is no longer + // available. Read content and put a new header in front of it. start := headsize(size) buf := make([]byte, uint64(start)+size) if err := s.readFull(buf[start:]); err != nil { @@ -730,7 +724,14 @@ func (s *Stream) List() (size uint64, err error) { if kind != List { return 0, ErrExpectedList } - s.stack = append(s.stack, listpos{0, size}) + + // Remove size of inner list from outer list before pushing the new size + // onto the stack. This ensures that the remaining outer list size will + // be correct after the matching call to ListEnd. + if inList, limit := s.listLimit(); inList { + s.stack[len(s.stack)-1] = limit - size + } + s.stack = append(s.stack, size) s.kind = -1 s.size = 0 return size, nil @@ -739,17 +740,13 @@ func (s *Stream) List() (size uint64, err error) { // ListEnd returns to the enclosing list. // The input reader must be positioned at the end of a list. func (s *Stream) ListEnd() error { - if len(s.stack) == 0 { + // Ensure that no more data is remaining in the current list. + if inList, listLimit := s.listLimit(); !inList { return errNotInList - } - tos := s.stack[len(s.stack)-1] - if tos.pos != tos.size { + } else if listLimit > 0 { return errNotAtEOL } s.stack = s.stack[:len(s.stack)-1] // pop - if len(s.stack) > 0 { - s.stack[len(s.stack)-1].pos += tos.size - } s.kind = -1 s.size = 0 return nil @@ -777,7 +774,7 @@ func (s *Stream) Decode(val interface{}) error { err = decoder(s, rval.Elem()) if decErr, ok := err.(*decodeError); ok && len(decErr.ctx) > 0 { - // add decode target type to error so context has more meaning + // Add decode target type to error so context has more meaning. decErr.ctx = append(decErr.ctx, fmt.Sprint("(", rtyp.Elem(), ")")) } return err @@ -800,6 +797,9 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { case *bytes.Reader: s.remaining = uint64(br.Len()) s.limited = true + case *bytes.Buffer: + s.remaining = uint64(br.Len()) + s.limited = true case *strings.Reader: s.remaining = uint64(br.Len()) s.limited = true @@ -818,10 +818,8 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { s.size = 0 s.kind = -1 s.kinderr = nil - if s.uintbuf == nil { - s.uintbuf = make([]byte, 8) - } s.byteval = 0 + s.uintbuf = [8]byte{} } // Kind returns the kind and size of the next value in the @@ -836,35 +834,29 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { // the value. Subsequent calls to Kind (until the value is decoded) // will not advance the input reader and return cached information. func (s *Stream) Kind() (kind Kind, size uint64, err error) { - var tos *listpos - if len(s.stack) > 0 { - tos = &s.stack[len(s.stack)-1] - } - if s.kind < 0 { - s.kinderr = nil - // Don't read further if we're at the end of the - // innermost list. - if tos != nil && tos.pos == tos.size { - return 0, 0, EOL - } - s.kind, s.size, s.kinderr = s.readKind() - if s.kinderr == nil { - if tos == nil { - // At toplevel, check that the value is smaller - // than the remaining input length. - if s.limited && s.size > s.remaining { - s.kinderr = ErrValueTooLarge - } - } else { - // Inside a list, check that the value doesn't overflow the list. - if s.size > tos.size-tos.pos { - s.kinderr = ErrElemTooLarge - } - } + if s.kind >= 0 { + return s.kind, s.size, s.kinderr + } + + // Check for end of list. This needs to be done here because readKind + // checks against the list size, and would return the wrong error. + inList, listLimit := s.listLimit() + if inList && listLimit == 0 { + return 0, 0, EOL + } + // Read the actual size tag. + s.kind, s.size, s.kinderr = s.readKind() + if s.kinderr == nil { + // Check the data size of the value ahead against input limits. This + // is done here because many decoders require allocating an input + // buffer matching the value size. Checking it here protects those + // decoders from inputs declaring very large value size. + if inList && s.size > listLimit { + s.kinderr = ErrElemTooLarge + } else if s.limited && s.size > s.remaining { + s.kinderr = ErrValueTooLarge } } - // Note: this might return a sticky error generated - // by an earlier call to readKind. return s.kind, s.size, s.kinderr } @@ -891,37 +883,35 @@ func (s *Stream) readKind() (kind Kind, size uint64, err error) { s.byteval = b return Byte, 0, nil case b < 0xB8: - // Otherwise, if a string is 0-55 bytes long, - // the RLP encoding consists of a single byte with value 0x80 plus the - // length of the string followed by the string. The range of the first - // byte is thus [0x80, 0xB7]. + // Otherwise, if a string is 0-55 bytes long, the RLP encoding consists + // of a single byte with value 0x80 plus the length of the string + // followed by the string. The range of the first byte is thus [0x80, 0xB7]. return String, uint64(b - 0x80), nil case b < 0xC0: - // If a string is more than 55 bytes long, the - // RLP encoding consists of a single byte with value 0xB7 plus the length - // of the length of the string in binary form, followed by the length of - // the string, followed by the string. For example, a length-1024 string - // would be encoded as 0xB90400 followed by the string. The range of - // the first byte is thus [0xB8, 0xBF]. + // If a string is more than 55 bytes long, the RLP encoding consists of a + // single byte with value 0xB7 plus the length of the length of the + // string in binary form, followed by the length of the string, followed + // by the string. For example, a length-1024 string would be encoded as + // 0xB90400 followed by the string. The range of the first byte is thus + // [0xB8, 0xBF]. size, err = s.readUint(b - 0xB7) if err == nil && size < 56 { err = ErrCanonSize } return String, size, err case b < 0xF8: - // If the total payload of a list - // (i.e. the combined length of all its items) is 0-55 bytes long, the - // RLP encoding consists of a single byte with value 0xC0 plus the length - // of the list followed by the concatenation of the RLP encodings of the - // items. The range of the first byte is thus [0xC0, 0xF7]. + // If the total payload of a list (i.e. the combined length of all its + // items) is 0-55 bytes long, the RLP encoding consists of a single byte + // with value 0xC0 plus the length of the list followed by the + // concatenation of the RLP encodings of the items. The range of the + // first byte is thus [0xC0, 0xF7]. return List, uint64(b - 0xC0), nil default: - // If the total payload of a list is more than 55 bytes long, - // the RLP encoding consists of a single byte with value 0xF7 - // plus the length of the length of the payload in binary - // form, followed by the length of the payload, followed by - // the concatenation of the RLP encodings of the items. The - // range of the first byte is thus [0xF8, 0xFF]. + // If the total payload of a list is more than 55 bytes long, the RLP + // encoding consists of a single byte with value 0xF7 plus the length of + // the length of the payload in binary form, followed by the length of + // the payload, followed by the concatenation of the RLP encodings of + // the items. The range of the first byte is thus [0xF8, 0xFF]. size, err = s.readUint(b - 0xF7) if err == nil && size < 56 { err = ErrCanonSize @@ -940,22 +930,20 @@ func (s *Stream) readUint(size byte) (uint64, error) { return uint64(b), err default: start := int(8 - size) - for i := 0; i < start; i++ { - s.uintbuf[i] = 0 - } + s.uintbuf = [8]byte{} if err := s.readFull(s.uintbuf[start:]); err != nil { return 0, err } if s.uintbuf[start] == 0 { - // Note: readUint is also used to decode integer - // values. The error needs to be adjusted to become - // ErrCanonInt in this case. + // Note: readUint is also used to decode integer values. + // The error needs to be adjusted to become ErrCanonInt in this case. return 0, ErrCanonSize } - return binary.BigEndian.Uint64(s.uintbuf), nil + return binary.BigEndian.Uint64(s.uintbuf[:]), nil } } +// readFull reads into buf from the underlying stream. func (s *Stream) readFull(buf []byte) (err error) { if err := s.willRead(uint64(len(buf))); err != nil { return err @@ -977,6 +965,7 @@ func (s *Stream) readFull(buf []byte) (err error) { return err } +// readByte reads a single byte from the underlying stream. func (s *Stream) readByte() (byte, error) { if err := s.willRead(1); err != nil { return 0, err @@ -988,16 +977,16 @@ func (s *Stream) readByte() (byte, error) { return b, err } +// willRead is called before any read from the underlying stream. It checks +// n against size limits, and updates the limits if n doesn't overflow them. func (s *Stream) willRead(n uint64) error { s.kind = -1 // rearm Kind - if len(s.stack) > 0 { - // check list overflow - tos := s.stack[len(s.stack)-1] - if n > tos.size-tos.pos { + if inList, limit := s.listLimit(); inList { + if n > limit { return ErrElemTooLarge } - s.stack[len(s.stack)-1].pos += n + s.stack[len(s.stack)-1] = limit - n } if s.limited { if n > s.remaining { @@ -1007,3 +996,11 @@ func (s *Stream) willRead(n uint64) error { } return nil } + +// listLimit returns the amount of data remaining in the innermost list. +func (s *Stream) listLimit() (inList bool, limit uint64) { + if len(s.stack) == 0 { + return false, 0 + } + return true, s.stack[len(s.stack)-1] +} From b3a1fda6509c8dd64b5f5916f62a6602bcdc7a9d Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 18 May 2021 11:54:10 -0600 Subject: [PATCH 372/709] cmd/utils: expand tilde in --jspath (#22900) --- cmd/utils/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d3fb3f2cbd..1ae07108e4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -675,10 +675,10 @@ var ( } // ATM the url is left to the user and deployment to - JSpathFlag = cli.StringFlag{ + JSpathFlag = DirectoryFlag{ Name: "jspath", Usage: "JavaScript root path for `loadScript`", - Value: ".", + Value: DirectoryString("."), } // Gas price oracle settings From 3e795881ea6d68c32da5da3c95f0d458a64e35c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 19 May 2021 15:09:03 +0300 Subject: [PATCH 373/709] eth, p2p/msgrate: move peer QoS tracking to its own package and use it for snap (#22876) This change extracts the peer QoS tracking logic from eth/downloader, moving it into the new package p2p/msgrate. The job of msgrate.Tracker is determining suitable timeout values and request sizes per peer. The snap sync scheduler now uses msgrate.Tracker instead of the hard-coded 15s timeout. This should make the sync work better on network links with high latency. --- eth/downloader/downloader.go | 119 +-------- eth/downloader/peer.go | 171 ++++-------- eth/downloader/peer_test.go | 53 ---- eth/downloader/statesync.go | 4 +- eth/protocols/snap/sync.go | 331 +++++++++++++++-------- eth/protocols/snap/sync_test.go | 18 +- p2p/msgrate/msgrate.go | 458 ++++++++++++++++++++++++++++++++ 7 files changed, 745 insertions(+), 409 deletions(-) delete mode 100644 eth/downloader/peer_test.go create mode 100644 p2p/msgrate/msgrate.go diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 6f59b29a5e..e8a4a76ca2 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -47,16 +47,6 @@ var ( MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request MaxStateFetch = 384 // Amount of node state values to allow fetching per request - rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests - rttMaxEstimate = 20 * time.Second // Maximum round-trip time to target for download requests - rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value - ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion - ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts - - qosTuningPeers = 5 // Number of peers to tune based on (best peers) - qosConfidenceCap = 10 // Number of peers above which not to modify RTT confidence - qosTuningImpact = 0.25 // Impact that a new tuning target has on the previous value - maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) maxHeadersProcess = 2048 // Number of header download results to import at once into the chain maxResultsProcess = 2048 // Number of content download results to import at once into the chain @@ -96,13 +86,6 @@ var ( ) type Downloader struct { - // WARNING: The `rttEstimate` and `rttConfidence` fields are accessed atomically. - // On 32 bit platforms, only 64-bit aligned fields can be atomic. The struct is - // guaranteed to be so aligned, so take advantage of that. For more information, - // see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - rttEstimate uint64 // Round trip time to target for download requests - rttConfidence uint64 // Confidence in the estimated RTT (unit: millionths to allow atomic ops) - mode uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode mux *event.TypeMux // Event multiplexer to announce sync operation events @@ -232,8 +215,6 @@ func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, checkpoint: checkpoint, queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), peers: newPeerSet(), - rttEstimate: uint64(rttMaxEstimate), - rttConfidence: uint64(1000000), blockchain: chain, lightchain: lightchain, dropPeer: dropPeer, @@ -252,7 +233,6 @@ func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, }, trackStateReq: make(chan *stateReq), } - go dl.qosTuner() go dl.stateFetcher() return dl } @@ -310,8 +290,6 @@ func (d *Downloader) RegisterPeer(id string, version uint, peer Peer) error { logger.Error("Failed to register sync peer", "err", err) return err } - d.qosReduceConfidence() - return nil } @@ -670,7 +648,7 @@ func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *ty } go p.peer.RequestHeadersByHash(latest, fetch, fsMinFullBlocks-1, true) - ttl := d.requestTTL() + ttl := d.peers.rates.TargetTimeout() timeout := time.After(ttl) for { select { @@ -853,7 +831,7 @@ func (d *Downloader) findAncestorSpanSearch(p *peerConnection, mode SyncMode, re // Wait for the remote response to the head fetch number, hash := uint64(0), common.Hash{} - ttl := d.requestTTL() + ttl := d.peers.rates.TargetTimeout() timeout := time.After(ttl) for finished := false; !finished; { @@ -942,7 +920,7 @@ func (d *Downloader) findAncestorBinarySearch(p *peerConnection, mode SyncMode, // Split our chain interval in two, and request the hash to cross check check := (start + end) / 2 - ttl := d.requestTTL() + ttl := d.peers.rates.TargetTimeout() timeout := time.After(ttl) go p.peer.RequestHeadersByNumber(check, 1, 0, false) @@ -1035,7 +1013,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error { getHeaders := func(from uint64) { request = time.Now() - ttl = d.requestTTL() + ttl = d.peers.rates.TargetTimeout() timeout.Reset(ttl) if skeleton { @@ -1050,7 +1028,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error { pivoting = true request = time.Now() - ttl = d.requestTTL() + ttl = d.peers.rates.TargetTimeout() timeout.Reset(ttl) d.pivotLock.RLock() @@ -1262,12 +1240,12 @@ func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ( pack := packet.(*headerPack) return d.queue.DeliverHeaders(pack.peerID, pack.headers, d.headerProcCh) } - expire = func() map[string]int { return d.queue.ExpireHeaders(d.requestTTL()) } + expire = func() map[string]int { return d.queue.ExpireHeaders(d.peers.rates.TargetTimeout()) } reserve = func(p *peerConnection, count int) (*fetchRequest, bool, bool) { return d.queue.ReserveHeaders(p, count), false, false } fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchHeaders(req.From, MaxHeaderFetch) } - capacity = func(p *peerConnection) int { return p.HeaderCapacity(d.requestRTT()) } + capacity = func(p *peerConnection) int { return p.HeaderCapacity(d.peers.rates.TargetRoundTrip()) } setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { p.SetHeadersIdle(accepted, deliveryTime) } @@ -1293,9 +1271,9 @@ func (d *Downloader) fetchBodies(from uint64) error { pack := packet.(*bodyPack) return d.queue.DeliverBodies(pack.peerID, pack.transactions, pack.uncles) } - expire = func() map[string]int { return d.queue.ExpireBodies(d.requestTTL()) } + expire = func() map[string]int { return d.queue.ExpireBodies(d.peers.rates.TargetTimeout()) } fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchBodies(req) } - capacity = func(p *peerConnection) int { return p.BlockCapacity(d.requestRTT()) } + capacity = func(p *peerConnection) int { return p.BlockCapacity(d.peers.rates.TargetRoundTrip()) } setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { p.SetBodiesIdle(accepted, deliveryTime) } ) err := d.fetchParts(d.bodyCh, deliver, d.bodyWakeCh, expire, @@ -1317,9 +1295,9 @@ func (d *Downloader) fetchReceipts(from uint64) error { pack := packet.(*receiptPack) return d.queue.DeliverReceipts(pack.peerID, pack.receipts) } - expire = func() map[string]int { return d.queue.ExpireReceipts(d.requestTTL()) } + expire = func() map[string]int { return d.queue.ExpireReceipts(d.peers.rates.TargetTimeout()) } fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchReceipts(req) } - capacity = func(p *peerConnection) int { return p.ReceiptCapacity(d.requestRTT()) } + capacity = func(p *peerConnection) int { return p.ReceiptCapacity(d.peers.rates.TargetRoundTrip()) } setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { p.SetReceiptsIdle(accepted, deliveryTime) } @@ -2031,78 +2009,3 @@ func (d *Downloader) deliver(destCh chan dataPack, packet dataPack, inMeter, dro return errNoSyncActive } } - -// qosTuner is the quality of service tuning loop that occasionally gathers the -// peer latency statistics and updates the estimated request round trip time. -func (d *Downloader) qosTuner() { - for { - // Retrieve the current median RTT and integrate into the previoust target RTT - rtt := time.Duration((1-qosTuningImpact)*float64(atomic.LoadUint64(&d.rttEstimate)) + qosTuningImpact*float64(d.peers.medianRTT())) - atomic.StoreUint64(&d.rttEstimate, uint64(rtt)) - - // A new RTT cycle passed, increase our confidence in the estimated RTT - conf := atomic.LoadUint64(&d.rttConfidence) - conf = conf + (1000000-conf)/2 - atomic.StoreUint64(&d.rttConfidence, conf) - - // Log the new QoS values and sleep until the next RTT - log.Debug("Recalculated downloader QoS values", "rtt", rtt, "confidence", float64(conf)/1000000.0, "ttl", d.requestTTL()) - select { - case <-d.quitCh: - return - case <-time.After(rtt): - } - } -} - -// qosReduceConfidence is meant to be called when a new peer joins the downloader's -// peer set, needing to reduce the confidence we have in out QoS estimates. -func (d *Downloader) qosReduceConfidence() { - // If we have a single peer, confidence is always 1 - peers := uint64(d.peers.Len()) - if peers == 0 { - // Ensure peer connectivity races don't catch us off guard - return - } - if peers == 1 { - atomic.StoreUint64(&d.rttConfidence, 1000000) - return - } - // If we have a ton of peers, don't drop confidence) - if peers >= uint64(qosConfidenceCap) { - return - } - // Otherwise drop the confidence factor - conf := atomic.LoadUint64(&d.rttConfidence) * (peers - 1) / peers - if float64(conf)/1000000 < rttMinConfidence { - conf = uint64(rttMinConfidence * 1000000) - } - atomic.StoreUint64(&d.rttConfidence, conf) - - rtt := time.Duration(atomic.LoadUint64(&d.rttEstimate)) - log.Debug("Relaxed downloader QoS values", "rtt", rtt, "confidence", float64(conf)/1000000.0, "ttl", d.requestTTL()) -} - -// requestRTT returns the current target round trip time for a download request -// to complete in. -// -// Note, the returned RTT is .9 of the actually estimated RTT. The reason is that -// the downloader tries to adapt queries to the RTT, so multiple RTT values can -// be adapted to, but smaller ones are preferred (stabler download stream). -func (d *Downloader) requestRTT() time.Duration { - return time.Duration(atomic.LoadUint64(&d.rttEstimate)) * 9 / 10 -} - -// requestTTL returns the current timeout allowance for a single download request -// to finish under. -func (d *Downloader) requestTTL() time.Duration { - var ( - rtt = time.Duration(atomic.LoadUint64(&d.rttEstimate)) - conf = float64(atomic.LoadUint64(&d.rttConfidence)) / 1000000.0 - ) - ttl := time.Duration(ttlScaling) * time.Duration(float64(rtt)/conf) - if ttl > ttlLimit { - ttl = ttlLimit - } - return ttl -} diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index b3b6cc95a0..b9c7716941 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -32,11 +32,11 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/msgrate" ) const ( - maxLackingHashes = 4096 // Maximum number of entries allowed on the list or lacking items - measurementImpact = 0.1 // The impact a single measurement has on a peer's final throughput value. + maxLackingHashes = 4096 // Maximum number of entries allowed on the list or lacking items ) var ( @@ -54,18 +54,12 @@ type peerConnection struct { receiptIdle int32 // Current receipt activity state of the peer (idle = 0, active = 1) stateIdle int32 // Current node data activity state of the peer (idle = 0, active = 1) - headerThroughput float64 // Number of headers measured to be retrievable per second - blockThroughput float64 // Number of blocks (bodies) measured to be retrievable per second - receiptThroughput float64 // Number of receipts measured to be retrievable per second - stateThroughput float64 // Number of node data pieces measured to be retrievable per second - - rtt time.Duration // Request round trip time to track responsiveness (QoS) - headerStarted time.Time // Time instance when the last header fetch was started blockStarted time.Time // Time instance when the last block (body) fetch was started receiptStarted time.Time // Time instance when the last receipt fetch was started stateStarted time.Time // Time instance when the last node data fetch was started + rates *msgrate.Tracker // Tracker to hone in on the number of items retrievable per second lacking map[common.Hash]struct{} // Set of hashes not to request (didn't have previously) peer Peer @@ -133,11 +127,6 @@ func (p *peerConnection) Reset() { atomic.StoreInt32(&p.receiptIdle, 0) atomic.StoreInt32(&p.stateIdle, 0) - p.headerThroughput = 0 - p.blockThroughput = 0 - p.receiptThroughput = 0 - p.stateThroughput = 0 - p.lacking = make(map[common.Hash]struct{}) } @@ -212,93 +201,72 @@ func (p *peerConnection) FetchNodeData(hashes []common.Hash) error { // requests. Its estimated header retrieval throughput is updated with that measured // just now. func (p *peerConnection) SetHeadersIdle(delivered int, deliveryTime time.Time) { - p.setIdle(deliveryTime.Sub(p.headerStarted), delivered, &p.headerThroughput, &p.headerIdle) + p.rates.Update(eth.BlockHeadersMsg, deliveryTime.Sub(p.headerStarted), delivered) + atomic.StoreInt32(&p.headerIdle, 0) } // SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval // requests. Its estimated body retrieval throughput is updated with that measured // just now. func (p *peerConnection) SetBodiesIdle(delivered int, deliveryTime time.Time) { - p.setIdle(deliveryTime.Sub(p.blockStarted), delivered, &p.blockThroughput, &p.blockIdle) + p.rates.Update(eth.BlockBodiesMsg, deliveryTime.Sub(p.blockStarted), delivered) + atomic.StoreInt32(&p.blockIdle, 0) } // SetReceiptsIdle sets the peer to idle, allowing it to execute new receipt // retrieval requests. Its estimated receipt retrieval throughput is updated // with that measured just now. func (p *peerConnection) SetReceiptsIdle(delivered int, deliveryTime time.Time) { - p.setIdle(deliveryTime.Sub(p.receiptStarted), delivered, &p.receiptThroughput, &p.receiptIdle) + p.rates.Update(eth.ReceiptsMsg, deliveryTime.Sub(p.receiptStarted), delivered) + atomic.StoreInt32(&p.receiptIdle, 0) } // SetNodeDataIdle sets the peer to idle, allowing it to execute new state trie // data retrieval requests. Its estimated state retrieval throughput is updated // with that measured just now. func (p *peerConnection) SetNodeDataIdle(delivered int, deliveryTime time.Time) { - p.setIdle(deliveryTime.Sub(p.stateStarted), delivered, &p.stateThroughput, &p.stateIdle) -} - -// setIdle sets the peer to idle, allowing it to execute new retrieval requests. -// Its estimated retrieval throughput is updated with that measured just now. -func (p *peerConnection) setIdle(elapsed time.Duration, delivered int, throughput *float64, idle *int32) { - // Irrelevant of the scaling, make sure the peer ends up idle - defer atomic.StoreInt32(idle, 0) - - p.lock.Lock() - defer p.lock.Unlock() - - // If nothing was delivered (hard timeout / unavailable data), reduce throughput to minimum - if delivered == 0 { - *throughput = 0 - return - } - // Otherwise update the throughput with a new measurement - if elapsed <= 0 { - elapsed = 1 // +1 (ns) to ensure non-zero divisor - } - measured := float64(delivered) / (float64(elapsed) / float64(time.Second)) - - *throughput = (1-measurementImpact)*(*throughput) + measurementImpact*measured - p.rtt = time.Duration((1-measurementImpact)*float64(p.rtt) + measurementImpact*float64(elapsed)) - - p.log.Trace("Peer throughput measurements updated", - "hps", p.headerThroughput, "bps", p.blockThroughput, - "rps", p.receiptThroughput, "sps", p.stateThroughput, - "miss", len(p.lacking), "rtt", p.rtt) + p.rates.Update(eth.NodeDataMsg, deliveryTime.Sub(p.stateStarted), delivered) + atomic.StoreInt32(&p.stateIdle, 0) } // HeaderCapacity retrieves the peers header download allowance based on its // previously discovered throughput. func (p *peerConnection) HeaderCapacity(targetRTT time.Duration) int { - p.lock.RLock() - defer p.lock.RUnlock() - - return int(math.Min(1+math.Max(1, p.headerThroughput*float64(targetRTT)/float64(time.Second)), float64(MaxHeaderFetch))) + cap := int(math.Ceil(p.rates.Capacity(eth.BlockHeadersMsg, targetRTT))) + if cap > MaxHeaderFetch { + cap = MaxHeaderFetch + } + return cap } // BlockCapacity retrieves the peers block download allowance based on its // previously discovered throughput. func (p *peerConnection) BlockCapacity(targetRTT time.Duration) int { - p.lock.RLock() - defer p.lock.RUnlock() - - return int(math.Min(1+math.Max(1, p.blockThroughput*float64(targetRTT)/float64(time.Second)), float64(MaxBlockFetch))) + cap := int(math.Ceil(p.rates.Capacity(eth.BlockBodiesMsg, targetRTT))) + if cap > MaxBlockFetch { + cap = MaxBlockFetch + } + return cap } // ReceiptCapacity retrieves the peers receipt download allowance based on its // previously discovered throughput. func (p *peerConnection) ReceiptCapacity(targetRTT time.Duration) int { - p.lock.RLock() - defer p.lock.RUnlock() - - return int(math.Min(1+math.Max(1, p.receiptThroughput*float64(targetRTT)/float64(time.Second)), float64(MaxReceiptFetch))) + cap := int(math.Ceil(p.rates.Capacity(eth.ReceiptsMsg, targetRTT))) + if cap > MaxReceiptFetch { + cap = MaxReceiptFetch + } + return cap } // NodeDataCapacity retrieves the peers state download allowance based on its // previously discovered throughput. func (p *peerConnection) NodeDataCapacity(targetRTT time.Duration) int { - p.lock.RLock() - defer p.lock.RUnlock() - - return int(math.Min(1+math.Max(1, p.stateThroughput*float64(targetRTT)/float64(time.Second)), float64(MaxStateFetch))) + cap := int(math.Ceil(p.rates.Capacity(eth.NodeDataMsg, targetRTT))) + if cap > MaxStateFetch { + cap = MaxStateFetch + } + return cap } // MarkLacking appends a new entity to the set of items (blocks, receipts, states) @@ -330,16 +298,20 @@ func (p *peerConnection) Lacks(hash common.Hash) bool { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peerConnection + peers map[string]*peerConnection + rates *msgrate.Trackers // Set of rate trackers to give the sync a common beat + newPeerFeed event.Feed peerDropFeed event.Feed - lock sync.RWMutex + + lock sync.RWMutex } // newPeerSet creates a new peer set top track the active download sources. func newPeerSet() *peerSet { return &peerSet{ peers: make(map[string]*peerConnection), + rates: msgrate.NewTrackers(log.New("proto", "eth")), } } @@ -371,30 +343,15 @@ func (ps *peerSet) Reset() { // average of all existing peers, to give it a realistic chance of being used // for data retrievals. func (ps *peerSet) Register(p *peerConnection) error { - // Retrieve the current median RTT as a sane default - p.rtt = ps.medianRTT() - // Register the new peer with some meaningful defaults ps.lock.Lock() if _, ok := ps.peers[p.id]; ok { ps.lock.Unlock() return errAlreadyRegistered } - if len(ps.peers) > 0 { - p.headerThroughput, p.blockThroughput, p.receiptThroughput, p.stateThroughput = 0, 0, 0, 0 - - for _, peer := range ps.peers { - peer.lock.RLock() - p.headerThroughput += peer.headerThroughput - p.blockThroughput += peer.blockThroughput - p.receiptThroughput += peer.receiptThroughput - p.stateThroughput += peer.stateThroughput - peer.lock.RUnlock() - } - p.headerThroughput /= float64(len(ps.peers)) - p.blockThroughput /= float64(len(ps.peers)) - p.receiptThroughput /= float64(len(ps.peers)) - p.stateThroughput /= float64(len(ps.peers)) + p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) + if err := ps.rates.Track(p.id, p.rates); err != nil { + return err } ps.peers[p.id] = p ps.lock.Unlock() @@ -413,6 +370,7 @@ func (ps *peerSet) Unregister(id string) error { return errNotRegistered } delete(ps.peers, id) + ps.rates.Untrack(id) ps.lock.Unlock() ps.peerDropFeed.Send(p) @@ -454,9 +412,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { return atomic.LoadInt32(&p.headerIdle) == 0 } throughput := func(p *peerConnection) float64 { - p.lock.RLock() - defer p.lock.RUnlock() - return p.headerThroughput + return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } @@ -468,9 +424,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { return atomic.LoadInt32(&p.blockIdle) == 0 } throughput := func(p *peerConnection) float64 { - p.lock.RLock() - defer p.lock.RUnlock() - return p.blockThroughput + return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } @@ -482,9 +436,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { return atomic.LoadInt32(&p.receiptIdle) == 0 } throughput := func(p *peerConnection) float64 { - p.lock.RLock() - defer p.lock.RUnlock() - return p.receiptThroughput + return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } @@ -496,9 +448,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { return atomic.LoadInt32(&p.stateIdle) == 0 } throughput := func(p *peerConnection) float64 { - p.lock.RLock() - defer p.lock.RUnlock() - return p.stateThroughput + return p.rates.Capacity(eth.NodeDataMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } @@ -527,37 +477,6 @@ func (ps *peerSet) idlePeers(minProtocol, maxProtocol uint, idleCheck func(*peer return sortPeers.p, total } -// medianRTT returns the median RTT of the peerset, considering only the tuning -// peers if there are more peers available. -func (ps *peerSet) medianRTT() time.Duration { - // Gather all the currently measured round trip times - ps.lock.RLock() - defer ps.lock.RUnlock() - - rtts := make([]float64, 0, len(ps.peers)) - for _, p := range ps.peers { - p.lock.RLock() - rtts = append(rtts, float64(p.rtt)) - p.lock.RUnlock() - } - sort.Float64s(rtts) - - median := rttMaxEstimate - if qosTuningPeers <= len(rtts) { - median = time.Duration(rtts[qosTuningPeers/2]) // Median of our tuning peers - } else if len(rtts) > 0 { - median = time.Duration(rtts[len(rtts)/2]) // Median of our connected peers (maintain even like this some baseline qos) - } - // Restrict the RTT into some QoS defaults, irrelevant of true RTT - if median < rttMinEstimate { - median = rttMinEstimate - } - if median > rttMaxEstimate { - median = rttMaxEstimate - } - return median -} - // peerThroughputSort implements the Sort interface, and allows for // sorting a set of peers by their throughput // The sorted data is with the _highest_ throughput first diff --git a/eth/downloader/peer_test.go b/eth/downloader/peer_test.go deleted file mode 100644 index 4bf0e200bb..0000000000 --- a/eth/downloader/peer_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package downloader - -import ( - "sort" - "testing" -) - -func TestPeerThroughputSorting(t *testing.T) { - a := &peerConnection{ - id: "a", - headerThroughput: 1.25, - } - b := &peerConnection{ - id: "b", - headerThroughput: 1.21, - } - c := &peerConnection{ - id: "c", - headerThroughput: 1.23, - } - - peers := []*peerConnection{a, b, c} - tps := []float64{a.headerThroughput, - b.headerThroughput, c.headerThroughput} - sortPeers := &peerThroughputSort{peers, tps} - sort.Sort(sortPeers) - if got, exp := sortPeers.p[0].id, "a"; got != exp { - t.Errorf("sort fail, got %v exp %v", got, exp) - } - if got, exp := sortPeers.p[1].id, "c"; got != exp { - t.Errorf("sort fail, got %v exp %v", got, exp) - } - if got, exp := sortPeers.p[2].id, "b"; got != exp { - t.Errorf("sort fail, got %v exp %v", got, exp) - } - -} diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index ff84a3a8f0..6c53e5577a 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -433,8 +433,8 @@ func (s *stateSync) assignTasks() { peers, _ := s.d.peers.NodeDataIdlePeers() for _, p := range peers { // Assign a batch of fetches proportional to the estimated latency/bandwidth - cap := p.NodeDataCapacity(s.d.requestRTT()) - req := &stateReq{peer: p, timeout: s.d.requestTTL()} + cap := p.NodeDataCapacity(s.d.peers.rates.TargetRoundTrip()) + req := &stateReq{peer: p, timeout: s.d.peers.rates.TargetTimeout()} nodes, _, codes := s.fillTasks(cap, req) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index e283473207..c57fcd71f6 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/msgrate" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" @@ -51,14 +52,15 @@ var ( ) const ( - // maxRequestSize is the maximum number of bytes to request from a remote peer. - maxRequestSize = 128 * 1024 + // minRequestSize is the minimum number of bytes to request from a remote peer. + // This number is used as the low cap for account and storage range requests. + // Bytecode and trienode are limited inherently by item count (1). + minRequestSize = 64 * 1024 - // maxStorageSetRequestCount is the maximum number of contracts to request the - // storage of in a single query. If this number is too low, we're not filling - // responses fully and waste round trip times. If it's too high, we're capping - // responses and waste bandwidth. - maxStorageSetRequestCount = maxRequestSize / 1024 + // maxRequestSize is the maximum number of bytes to request from a remote peer. + // This number is used as the high cap for account and storage range requests. + // Bytecode and trienode are limited more explicitly by the caps below. + maxRequestSize = 512 * 1024 // maxCodeRequestCount is the maximum number of bytecode blobs to request in a // single query. If this number is too low, we're not filling responses fully @@ -74,7 +76,7 @@ const ( // a single query. If this number is too low, we're not filling responses fully // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. - maxTrieRequestCount = 256 + maxTrieRequestCount = maxRequestSize / 512 ) var ( @@ -85,10 +87,6 @@ var ( // storageConcurrency is the number of chunks to split the a large contract // storage trie into to allow concurrent retrievals. storageConcurrency = 16 - - // requestTimeout is the maximum time a peer is allowed to spend on serving - // a single network request. - requestTimeout = 15 * time.Second // TODO(karalabe): Make it dynamic ala fast-sync? ) // ErrCancelled is returned from snap syncing if the operation was prematurely @@ -105,8 +103,9 @@ var ErrCancelled = errors.New("sync cancelled") // is only included to allow the runloop to match a response to the task being // synced without having yet another set of maps. type accountRequest struct { - peer string // Peer to which this request is assigned - id uint64 // Request ID of this request + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + time time.Time // Timestamp when the request was sent deliver chan *accountResponse // Channel to deliver successful response on revert chan *accountRequest // Channel to deliver request failure on @@ -142,8 +141,9 @@ type accountResponse struct { // is only included to allow the runloop to match a response to the task being // synced without having yet another set of maps. type bytecodeRequest struct { - peer string // Peer to which this request is assigned - id uint64 // Request ID of this request + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + time time.Time // Timestamp when the request was sent deliver chan *bytecodeResponse // Channel to deliver successful response on revert chan *bytecodeRequest // Channel to deliver request failure on @@ -173,8 +173,9 @@ type bytecodeResponse struct { // is only included to allow the runloop to match a response to the task being // synced without having yet another set of maps. type storageRequest struct { - peer string // Peer to which this request is assigned - id uint64 // Request ID of this request + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + time time.Time // Timestamp when the request was sent deliver chan *storageResponse // Channel to deliver successful response on revert chan *storageRequest // Channel to deliver request failure on @@ -218,8 +219,9 @@ type storageResponse struct { // is only included to allow the runloop to match a response to the task being // synced without having yet another set of maps. type trienodeHealRequest struct { - peer string // Peer to which this request is assigned - id uint64 // Request ID of this request + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + time time.Time // Timestamp when the request was sent deliver chan *trienodeHealResponse // Channel to deliver successful response on revert chan *trienodeHealRequest // Channel to deliver request failure on @@ -252,8 +254,9 @@ type trienodeHealResponse struct { // is only included to allow the runloop to match a response to the task being // synced without having yet another set of maps. type bytecodeHealRequest struct { - peer string // Peer to which this request is assigned - id uint64 // Request ID of this request + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + time time.Time // Timestamp when the request was sent deliver chan *bytecodeHealResponse // Channel to deliver successful response on revert chan *bytecodeHealRequest // Channel to deliver request failure on @@ -396,6 +399,7 @@ type Syncer struct { peers map[string]SyncPeer // Currently active peers to download from peerJoin *event.Feed // Event feed to react to peers joining peerDrop *event.Feed // Event feed to react to peers dropping + rates *msgrate.Trackers // Message throughput rates for peers // Request tracking during syncing phase statelessPeers map[string]struct{} // Peers that failed to deliver state data @@ -452,6 +456,7 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { peers: make(map[string]SyncPeer), peerJoin: new(event.Feed), peerDrop: new(event.Feed), + rates: msgrate.NewTrackers(log.New("proto", "snap")), update: make(chan struct{}, 1), accountIdlers: make(map[string]struct{}), @@ -484,6 +489,7 @@ func (s *Syncer) Register(peer SyncPeer) error { return errors.New("already registered") } s.peers[id] = peer + s.rates.Track(id, msgrate.NewTracker(s.rates.MeanCapacities(), s.rates.MedianRoundTrip())) // Mark the peer as idle, even if no sync is running s.accountIdlers[id] = struct{}{} @@ -509,6 +515,7 @@ func (s *Syncer) Unregister(id string) error { return errors.New("not registered") } delete(s.peers, id) + s.rates.Untrack(id) // Remove status markers, even if no sync is running delete(s.statelessPeers, id) @@ -851,10 +858,24 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac s.lock.Lock() defer s.lock.Unlock() - // If there are no idle peers, short circuit assignment - if len(s.accountIdlers) == 0 { + // Sort the peers by download capacity to use faster ones if many available + idlers := &capacitySort{ + ids: make([]string, 0, len(s.accountIdlers)), + caps: make([]float64, 0, len(s.accountIdlers)), + } + targetTTL := s.rates.TargetTimeout() + for id := range s.accountIdlers { + if _, ok := s.statelessPeers[id]; ok { + continue + } + idlers.ids = append(idlers.ids, id) + idlers.caps = append(idlers.caps, s.rates.Capacity(id, AccountRangeMsg, targetTTL)) + } + if len(idlers.ids) == 0 { return } + sort.Sort(sort.Reverse(idlers)) + // Iterate over all the tasks and try to find a pending one for _, task := range s.tasks { // Skip any tasks already filling @@ -864,20 +885,15 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac // Task pending retrieval, try to find an idle peer. If no such peer // exists, we probably assigned tasks for all (or they are stateless). // Abort the entire assignment mechanism. - var idle string - for id := range s.accountIdlers { - // If the peer rejected a query in this sync cycle, don't bother asking - // again for anything, it's either out of sync or already pruned - if _, ok := s.statelessPeers[id]; ok { - continue - } - idle = id - break - } - if idle == "" { + if len(idlers.ids) == 0 { return } - peer := s.peers[idle] + var ( + idle = idlers.ids[0] + peer = s.peers[idle] + cap = idlers.caps[0] + ) + idlers.ids, idlers.caps = idlers.ids[1:], idlers.caps[1:] // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 @@ -895,6 +911,7 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac req := &accountRequest{ peer: idle, id: reqid, + time: time.Now(), deliver: success, revert: fail, cancel: cancel, @@ -903,8 +920,9 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac limit: task.Last, task: task, } - req.timeout = time.AfterFunc(requestTimeout, func() { + req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { peer.Log().Debug("Account range request timed out", "reqid", reqid) + s.rates.Update(idle, AccountRangeMsg, 0, 0) s.scheduleRevertAccountRequest(req) }) s.accountReqs[reqid] = req @@ -915,7 +933,13 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac defer s.pend.Done() // Attempt to send the remote request and revert if it fails - if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, maxRequestSize); err != nil { + if cap > maxRequestSize { + cap = maxRequestSize + } + if cap < minRequestSize { // Don't bother with peers below a bare minimum performance + cap = minRequestSize + } + if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, uint64(cap)); err != nil { peer.Log().Debug("Failed to request account range", "err", err) s.scheduleRevertAccountRequest(req) } @@ -931,10 +955,24 @@ func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan * s.lock.Lock() defer s.lock.Unlock() - // If there are no idle peers, short circuit assignment - if len(s.bytecodeIdlers) == 0 { + // Sort the peers by download capacity to use faster ones if many available + idlers := &capacitySort{ + ids: make([]string, 0, len(s.bytecodeIdlers)), + caps: make([]float64, 0, len(s.bytecodeIdlers)), + } + targetTTL := s.rates.TargetTimeout() + for id := range s.bytecodeIdlers { + if _, ok := s.statelessPeers[id]; ok { + continue + } + idlers.ids = append(idlers.ids, id) + idlers.caps = append(idlers.caps, s.rates.Capacity(id, ByteCodesMsg, targetTTL)) + } + if len(idlers.ids) == 0 { return } + sort.Sort(sort.Reverse(idlers)) + // Iterate over all the tasks and try to find a pending one for _, task := range s.tasks { // Skip any tasks not in the bytecode retrieval phase @@ -948,20 +986,15 @@ func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan * // Task pending retrieval, try to find an idle peer. If no such peer // exists, we probably assigned tasks for all (or they are stateless). // Abort the entire assignment mechanism. - var idle string - for id := range s.bytecodeIdlers { - // If the peer rejected a query in this sync cycle, don't bother asking - // again for anything, it's either out of sync or already pruned - if _, ok := s.statelessPeers[id]; ok { - continue - } - idle = id - break - } - if idle == "" { + if len(idlers.ids) == 0 { return } - peer := s.peers[idle] + var ( + idle = idlers.ids[0] + peer = s.peers[idle] + cap = idlers.caps[0] + ) + idlers.ids, idlers.caps = idlers.ids[1:], idlers.caps[1:] // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 @@ -976,17 +1009,21 @@ func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan * break } // Generate the network query and send it to the peer - hashes := make([]common.Hash, 0, maxCodeRequestCount) + if cap > maxCodeRequestCount { + cap = maxCodeRequestCount + } + hashes := make([]common.Hash, 0, int(cap)) for hash := range task.codeTasks { delete(task.codeTasks, hash) hashes = append(hashes, hash) - if len(hashes) >= maxCodeRequestCount { + if len(hashes) >= int(cap) { break } } req := &bytecodeRequest{ peer: idle, id: reqid, + time: time.Now(), deliver: success, revert: fail, cancel: cancel, @@ -994,8 +1031,9 @@ func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan * hashes: hashes, task: task, } - req.timeout = time.AfterFunc(requestTimeout, func() { + req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { peer.Log().Debug("Bytecode request timed out", "reqid", reqid) + s.rates.Update(idle, ByteCodesMsg, 0, 0) s.scheduleRevertBytecodeRequest(req) }) s.bytecodeReqs[reqid] = req @@ -1020,10 +1058,24 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st s.lock.Lock() defer s.lock.Unlock() - // If there are no idle peers, short circuit assignment - if len(s.storageIdlers) == 0 { + // Sort the peers by download capacity to use faster ones if many available + idlers := &capacitySort{ + ids: make([]string, 0, len(s.storageIdlers)), + caps: make([]float64, 0, len(s.storageIdlers)), + } + targetTTL := s.rates.TargetTimeout() + for id := range s.storageIdlers { + if _, ok := s.statelessPeers[id]; ok { + continue + } + idlers.ids = append(idlers.ids, id) + idlers.caps = append(idlers.caps, s.rates.Capacity(id, StorageRangesMsg, targetTTL)) + } + if len(idlers.ids) == 0 { return } + sort.Sort(sort.Reverse(idlers)) + // Iterate over all the tasks and try to find a pending one for _, task := range s.tasks { // Skip any tasks not in the storage retrieval phase @@ -1037,20 +1089,15 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st // Task pending retrieval, try to find an idle peer. If no such peer // exists, we probably assigned tasks for all (or they are stateless). // Abort the entire assignment mechanism. - var idle string - for id := range s.storageIdlers { - // If the peer rejected a query in this sync cycle, don't bother asking - // again for anything, it's either out of sync or already pruned - if _, ok := s.statelessPeers[id]; ok { - continue - } - idle = id - break - } - if idle == "" { + if len(idlers.ids) == 0 { return } - peer := s.peers[idle] + var ( + idle = idlers.ids[0] + peer = s.peers[idle] + cap = idlers.caps[0] + ) + idlers.ids, idlers.caps = idlers.ids[1:], idlers.caps[1:] // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 @@ -1067,9 +1114,17 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st // Generate the network query and send it to the peer. If there are // large contract tasks pending, complete those before diving into // even more new contracts. + if cap > maxRequestSize { + cap = maxRequestSize + } + if cap < minRequestSize { // Don't bother with peers below a bare minimum performance + cap = minRequestSize + } + storageSets := int(cap / 1024) + var ( - accounts = make([]common.Hash, 0, maxStorageSetRequestCount) - roots = make([]common.Hash, 0, maxStorageSetRequestCount) + accounts = make([]common.Hash, 0, storageSets) + roots = make([]common.Hash, 0, storageSets) subtask *storageTask ) for account, subtasks := range task.SubTasks { @@ -1096,7 +1151,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st accounts = append(accounts, acccount) roots = append(roots, root) - if len(accounts) >= maxStorageSetRequestCount { + if len(accounts) >= storageSets { break } } @@ -1109,6 +1164,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st req := &storageRequest{ peer: idle, id: reqid, + time: time.Now(), deliver: success, revert: fail, cancel: cancel, @@ -1122,8 +1178,9 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st req.origin = subtask.Next req.limit = subtask.Last } - req.timeout = time.AfterFunc(requestTimeout, func() { + req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { peer.Log().Debug("Storage request timed out", "reqid", reqid) + s.rates.Update(idle, StorageRangesMsg, 0, 0) s.scheduleRevertStorageRequest(req) }) s.storageReqs[reqid] = req @@ -1138,7 +1195,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st if subtask != nil { origin, limit = req.origin[:], req.limit[:] } - if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, maxRequestSize); err != nil { + if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, uint64(cap)); err != nil { log.Debug("Failed to request storage", "err", err) s.scheduleRevertStorageRequest(req) } @@ -1157,10 +1214,24 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai s.lock.Lock() defer s.lock.Unlock() - // If there are no idle peers, short circuit assignment - if len(s.trienodeHealIdlers) == 0 { + // Sort the peers by download capacity to use faster ones if many available + idlers := &capacitySort{ + ids: make([]string, 0, len(s.trienodeHealIdlers)), + caps: make([]float64, 0, len(s.trienodeHealIdlers)), + } + targetTTL := s.rates.TargetTimeout() + for id := range s.trienodeHealIdlers { + if _, ok := s.statelessPeers[id]; ok { + continue + } + idlers.ids = append(idlers.ids, id) + idlers.caps = append(idlers.caps, s.rates.Capacity(id, TrieNodesMsg, targetTTL)) + } + if len(idlers.ids) == 0 { return } + sort.Sort(sort.Reverse(idlers)) + // Iterate over pending tasks and try to find a peer to retrieve with for len(s.healer.trieTasks) > 0 || s.healer.scheduler.Pending() > 0 { // If there are not enough trie tasks queued to fully assign, fill the @@ -1186,20 +1257,15 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai // Task pending retrieval, try to find an idle peer. If no such peer // exists, we probably assigned tasks for all (or they are stateless). // Abort the entire assignment mechanism. - var idle string - for id := range s.trienodeHealIdlers { - // If the peer rejected a query in this sync cycle, don't bother asking - // again for anything, it's either out of sync or already pruned - if _, ok := s.statelessPeers[id]; ok { - continue - } - idle = id - break - } - if idle == "" { + if len(idlers.ids) == 0 { return } - peer := s.peers[idle] + var ( + idle = idlers.ids[0] + peer = s.peers[idle] + cap = idlers.caps[0] + ) + idlers.ids, idlers.caps = idlers.ids[1:], idlers.caps[1:] // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 @@ -1214,10 +1280,13 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai break } // Generate the network query and send it to the peer + if cap > maxTrieRequestCount { + cap = maxTrieRequestCount + } var ( - hashes = make([]common.Hash, 0, maxTrieRequestCount) - paths = make([]trie.SyncPath, 0, maxTrieRequestCount) - pathsets = make([]TrieNodePathSet, 0, maxTrieRequestCount) + hashes = make([]common.Hash, 0, int(cap)) + paths = make([]trie.SyncPath, 0, int(cap)) + pathsets = make([]TrieNodePathSet, 0, int(cap)) ) for hash, pathset := range s.healer.trieTasks { delete(s.healer.trieTasks, hash) @@ -1226,13 +1295,14 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai paths = append(paths, pathset) pathsets = append(pathsets, [][]byte(pathset)) // TODO(karalabe): group requests by account hash - if len(hashes) >= maxTrieRequestCount { + if len(hashes) >= int(cap) { break } } req := &trienodeHealRequest{ peer: idle, id: reqid, + time: time.Now(), deliver: success, revert: fail, cancel: cancel, @@ -1241,8 +1311,9 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai paths: paths, task: s.healer, } - req.timeout = time.AfterFunc(requestTimeout, func() { + req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { peer.Log().Debug("Trienode heal request timed out", "reqid", reqid) + s.rates.Update(idle, TrieNodesMsg, 0, 0) s.scheduleRevertTrienodeHealRequest(req) }) s.trienodeHealReqs[reqid] = req @@ -1267,10 +1338,24 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai s.lock.Lock() defer s.lock.Unlock() - // If there are no idle peers, short circuit assignment - if len(s.bytecodeHealIdlers) == 0 { + // Sort the peers by download capacity to use faster ones if many available + idlers := &capacitySort{ + ids: make([]string, 0, len(s.bytecodeHealIdlers)), + caps: make([]float64, 0, len(s.bytecodeHealIdlers)), + } + targetTTL := s.rates.TargetTimeout() + for id := range s.bytecodeHealIdlers { + if _, ok := s.statelessPeers[id]; ok { + continue + } + idlers.ids = append(idlers.ids, id) + idlers.caps = append(idlers.caps, s.rates.Capacity(id, ByteCodesMsg, targetTTL)) + } + if len(idlers.ids) == 0 { return } + sort.Sort(sort.Reverse(idlers)) + // Iterate over pending tasks and try to find a peer to retrieve with for len(s.healer.codeTasks) > 0 || s.healer.scheduler.Pending() > 0 { // If there are not enough trie tasks queued to fully assign, fill the @@ -1296,20 +1381,15 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai // Task pending retrieval, try to find an idle peer. If no such peer // exists, we probably assigned tasks for all (or they are stateless). // Abort the entire assignment mechanism. - var idle string - for id := range s.bytecodeHealIdlers { - // If the peer rejected a query in this sync cycle, don't bother asking - // again for anything, it's either out of sync or already pruned - if _, ok := s.statelessPeers[id]; ok { - continue - } - idle = id - break - } - if idle == "" { + if len(idlers.ids) == 0 { return } - peer := s.peers[idle] + var ( + idle = idlers.ids[0] + peer = s.peers[idle] + cap = idlers.caps[0] + ) + idlers.ids, idlers.caps = idlers.ids[1:], idlers.caps[1:] // Matched a pending task to an idle peer, allocate a unique request id var reqid uint64 @@ -1324,18 +1404,22 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai break } // Generate the network query and send it to the peer - hashes := make([]common.Hash, 0, maxCodeRequestCount) + if cap > maxCodeRequestCount { + cap = maxCodeRequestCount + } + hashes := make([]common.Hash, 0, int(cap)) for hash := range s.healer.codeTasks { delete(s.healer.codeTasks, hash) hashes = append(hashes, hash) - if len(hashes) >= maxCodeRequestCount { + if len(hashes) >= int(cap) { break } } req := &bytecodeHealRequest{ peer: idle, id: reqid, + time: time.Now(), deliver: success, revert: fail, cancel: cancel, @@ -1343,8 +1427,9 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai hashes: hashes, task: s.healer, } - req.timeout = time.AfterFunc(requestTimeout, func() { + req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { peer.Log().Debug("Bytecode heal request timed out", "reqid", reqid) + s.rates.Update(idle, ByteCodesMsg, 0, 0) s.scheduleRevertBytecodeHealRequest(req) }) s.bytecodeHealReqs[reqid] = req @@ -2142,6 +2227,7 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco return nil } delete(s.accountReqs, id) + s.rates.Update(peer.ID(), AccountRangeMsg, time.Since(req.time), int(size)) // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content @@ -2253,6 +2339,7 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error return nil } delete(s.bytecodeReqs, id) + s.rates.Update(peer.ID(), ByteCodesMsg, time.Since(req.time), len(bytecodes)) // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content @@ -2361,6 +2448,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo return nil } delete(s.storageReqs, id) + s.rates.Update(peer.ID(), StorageRangesMsg, time.Since(req.time), int(size)) // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content @@ -2487,6 +2575,7 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error return nil } delete(s.trienodeHealReqs, id) + s.rates.Update(peer.ID(), TrieNodesMsg, time.Since(req.time), len(trienodes)) // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content @@ -2581,6 +2670,7 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e return nil } delete(s.bytecodeHealReqs, id) + s.rates.Update(peer.ID(), ByteCodesMsg, time.Since(req.time), len(bytecodes)) // Clean up the request timeout timer, we'll see how to proceed further based // on the actual delivered content @@ -2756,3 +2846,24 @@ func estimateRemainingSlots(hashes int, last common.Hash) (uint64, error) { } return space.Uint64() - uint64(hashes), nil } + +// capacitySort implements the Sort interface, allowing sorting by peer message +// throughput. Note, callers should use sort.Reverse to get the desired effect +// of highest capacity being at the front. +type capacitySort struct { + ids []string + caps []float64 +} + +func (s *capacitySort) Len() int { + return len(s.ids) +} + +func (s *capacitySort) Less(i, j int) bool { + return s.caps[i] < s.caps[j] +} + +func (s *capacitySort) Swap(i, j int) { + s.ids[i], s.ids[j] = s.ids[j], s.ids[i] + s.caps[i], s.caps[j] = s.caps[j], s.caps[i] +} diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index a1cc3581a8..023fc8ee00 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -796,12 +796,6 @@ func TestMultiSyncManyUseless(t *testing.T) { // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { - // We're setting the timeout to very low, to increase the chance of the timeout - // being triggered. This was previously a cause of panic, when a response - // arrived simultaneously as a timeout was triggered. - defer func(old time.Duration) { requestTimeout = old }(requestTimeout) - requestTimeout = time.Millisecond - var ( once sync.Once cancel = make(chan struct{}) @@ -838,6 +832,11 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { mkSource("noStorage", true, false, true), mkSource("noTrie", true, true, false), ) + // We're setting the timeout to very low, to increase the chance of the timeout + // being triggered. This was previously a cause of panic, when a response + // arrived simultaneously as a timeout was triggered. + syncer.rates.OverrideTTLLimit = time.Millisecond + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) @@ -848,10 +847,6 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { // TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all func TestMultiSyncManyUnresponsive(t *testing.T) { - // We're setting the timeout to very low, to make the test run a bit faster - defer func(old time.Duration) { requestTimeout = old }(requestTimeout) - requestTimeout = time.Millisecond - var ( once sync.Once cancel = make(chan struct{}) @@ -888,6 +883,9 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { mkSource("noStorage", true, false, true), mkSource("noTrie", true, true, false), ) + // We're setting the timeout to very low, to make the test run a bit faster + syncer.rates.OverrideTTLLimit = time.Millisecond + done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go new file mode 100644 index 0000000000..7cd172c566 --- /dev/null +++ b/p2p/msgrate/msgrate.go @@ -0,0 +1,458 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package msgrate allows estimating the throughput of peers for more balanced syncs. +package msgrate + +import ( + "errors" + "sort" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +// measurementImpact is the impact a single measurement has on a peer's final +// capacity value. A value closer to 0 reacts slower to sudden network changes, +// but it is also more stable against temporary hiccups. 0.1 worked well for +// most of Ethereum's existence, so might as well go with it. +const measurementImpact = 0.1 + +// capacityOverestimation is the ratio of items to over-estimate when retrieving +// a peer's capacity to avoid locking into a lower value due to never attempting +// to fetch more than some local stable value. +const capacityOverestimation = 1.01 + +// qosTuningPeers is the number of best peers to tune round trip times based on. +// An Ethereum node doesn't need hundreds of connections to operate correctly, +// so instead of lowering our download speed to the median of potentially many +// bad nodes, we can target a smaller set of vey good nodes. At worse this will +// result in less nodes to sync from, but that's still better than some hogging +// the pipeline. +const qosTuningPeers = 5 + +// rttMinEstimate is the minimal round trip time to target requests for. Since +// every request entails a 2 way latency + bandwidth + serving database lookups, +// it should be generous enough to permit meaningful work to be done on top of +// the transmission costs. +const rttMinEstimate = 2 * time.Second + +// rttMaxEstimate is the maximal round trip time to target requests for. Although +// the expectation is that a well connected node will never reach this, certain +// special connectivity ones might experience significant delays (e.g. satellite +// uplink with 3s RTT). This value should be low enough to forbid stalling the +// pipeline too long, but large enough to cover the worst of the worst links. +const rttMaxEstimate = 20 * time.Second + +// rttPushdownFactor is a multiplier to attempt forcing quicker requests than +// what the message rate tracker estimates. The reason is that message rate +// tracking adapts queries to the RTT, but multiple RTT values can be perfectly +// valid, they just result in higher packet sizes. Since smaller packets almost +// always result in stabler download streams, this factor hones in on the lowest +// RTT from all the functional ones. +const rttPushdownFactor = 0.9 + +// rttMinConfidence is the minimum value the roundtrip confidence factor may drop +// to. Since the target timeouts are based on how confident the tracker is in the +// true roundtrip, it's important to not allow too huge fluctuations. +const rttMinConfidence = 0.1 + +// ttlScaling is the multiplier that converts the estimated roundtrip time to a +// timeout cap for network requests. The expectation is that peers' response time +// will fluctuate around the estimated roundtrip, but depending in their load at +// request time, it might be higher than anticipated. This scaling factor ensures +// that we allow remote connections some slack but at the same time do enforce a +// behavior similar to our median peers. +const ttlScaling = 3 + +// ttlLimit is the maximum timeout allowance to prevent reaching crazy numbers +// if some unforeseen network events shappen. As much as we try to hone in on +// the most optimal values, it doesn't make any sense to go above a threshold, +// even if everything is slow and screwy. +const ttlLimit = time.Minute + +// tuningConfidenceCap is the number of active peers above which to stop detuning +// the confidence number. The idea here is that once we hone in on the capacity +// of a meaningful number of peers, adding one more should ot have a significant +// impact on things, so just ron with the originals. +const tuningConfidenceCap = 10 + +// tuningImpact is the influence that a new tuning target has on the previously +// cached value. This number is mostly just an out-of-the-blue heuristic that +// prevents the estimates from jumping around. There's no particular reason for +// the current value. +const tuningImpact = 0.25 + +// Tracker estimates the throughput capacity of a peer with regard to each data +// type it can deliver. The goal is to dynamically adjust request sizes to max +// out network throughput without overloading either the peer or th elocal node. +// +// By tracking in real time the latencies and bandiwdths peers exhibit for each +// packet type, it's possible to prevent overloading by detecting a slowdown on +// one type when another type is pushed too hard. +// +// Similarly, real time measurements also help avoid overloading the local net +// connection if our peers would otherwise be capable to deliver more, but the +// local link is saturated. In that case, the live measurements will force us +// to reduce request sizes until the throughput gets stable. +// +// Lastly, message rate measurements allows us to detect if a peer is unsuaully +// slow compared to other peers, in which case we can decide to keep it around +// or free up the slot so someone closer. +// +// Since throughput tracking and estimation adapts dynamically to live network +// conditions, it's fine to have multiple trackers locally track the same peer +// in different subsystem. The throughput will simply be distributed across the +// two trackers if both are highly active. +type Tracker struct { + // capacity is the number of items retrievable per second of a given type. + // It is analogous to bandwidth, but we deliberately avoided using bytes + // as the unit, since serving nodes also spend a lot of time loading data + // from disk, which is linear in the number of items, but mostly constant + // in their sizes. + // + // Callers of course are free to use the item counter as a byte counter if + // or when their protocol of choise if capped by bytes instead of items. + // (eg. eth.getHeaders vs snap.getAccountRange). + capacity map[uint64]float64 + + // roundtrip is the latency a peer in general responds to data requests. + // This number is not used inside the tracker, but is exposed to compare + // peers to each other and filter out slow ones. Note however, it only + // makes sense to compare RTTs if the caller caters request sizes for + // each peer to target the same RTT. There's no need to make this number + // the real networking RTT, we just need a number to compare peers with. + roundtrip time.Duration + + lock sync.RWMutex +} + +// NewTracker creates a new message rate tracker for a specific peer. An initial +// RTT is needed to avoid a peer getting marked as an outlier compared to others +// right after joining. It's suggested to use the median rtt across all peers to +// init a new peer tracker. +func NewTracker(caps map[uint64]float64, rtt time.Duration) *Tracker { + if caps == nil { + caps = make(map[uint64]float64) + } + return &Tracker{ + capacity: caps, + roundtrip: rtt, + } +} + +// Capacity calculates the number of items the peer is estimated to be able to +// retrieve within the alloted time slot. The method will round up any division +// errors and will add an additional overestimation ratio on top. The reason for +// overshooting the capacity is because certain message types might not increase +// the load proportionally to the requested items, so fetching a bit more might +// still take the same RTT. By forcefully overshooting by a small amount, we can +// avoid locking into a lower-that-real capacity. +func (t *Tracker) Capacity(kind uint64, targetRTT time.Duration) float64 { + t.lock.RLock() + defer t.lock.RUnlock() + + // Calculate the actual measured throughput + throughput := t.capacity[kind] * float64(targetRTT) / float64(time.Second) + + // Return an overestimation to force the peer out of a stuck minima, adding + // +1 in case the item count is too low for the overestimator to dent + return 1 + capacityOverestimation*throughput +} + +// Update modifies the peer's capacity values for a specific data type with a new +// measurement. If the delivery is zero, the peer is assumed to have either timed +// out or to not have the requested data, resulting in a slash to 0 capacity. This +// avoids assigning the peer retrievals that it won't be able to honour. +func (t *Tracker) Update(kind uint64, elapsed time.Duration, items int) { + t.lock.Lock() + defer t.lock.Unlock() + + // If nothing was delivered (timeout / unavailable data), reduce throughput + // to minimum + if items == 0 { + t.capacity[kind] = 0 + return + } + // Otherwise update the throughput with a new measurement + if elapsed <= 0 { + elapsed = 1 // +1 (ns) to ensure non-zero divisor + } + measured := float64(items) / (float64(elapsed) / float64(time.Second)) + + t.capacity[kind] = (1-measurementImpact)*(t.capacity[kind]) + measurementImpact*measured + t.roundtrip = time.Duration((1-measurementImpact)*float64(t.roundtrip) + measurementImpact*float64(elapsed)) +} + +// Trackers is a set of message rate trackers across a number of peers with the +// goal of aggregating certain measurements across the entire set for outlier +// filtering and newly joining initialization. +type Trackers struct { + trackers map[string]*Tracker + + // roundtrip is the current best guess as to what is a stable round trip time + // across the entire collection of connected peers. This is derived from the + // various trackers added, but is used as a cache to avoid recomputing on each + // network request. The value is updated once every RTT to avoid fluctuations + // caused by hiccups or peer events. + roundtrip time.Duration + + // confidence represents the probability that the estimated roundtrip value + // is the real one across all our peers. The confidence value is used as an + // impact factor of new measurements on old estimates. As our connectivity + // stabilizes, this value gravitates towards 1, new measurements havinng + // almost no impact. If there's a large peer churn and few peers, then new + // measurements will impact it more. The confidence is increased with every + // packet and dropped with every new connection. + confidence float64 + + // tuned is the time instance the tracker recalculated its cached roundtrip + // value and confidence values. A cleaner way would be to have a heartbeat + // goroutine do it regularly, but that requires a lot of maintenance to just + // run every now and again. + tuned time.Time + + // The fields below can be used to override certain default values. Their + // purpose is to allow quicker tests. Don't use them in production. + OverrideTTLLimit time.Duration + + log log.Logger + lock sync.RWMutex +} + +// NewTrackers creates an empty set of trackers to be filled with peers. +func NewTrackers(log log.Logger) *Trackers { + return &Trackers{ + trackers: make(map[string]*Tracker), + roundtrip: rttMaxEstimate, + confidence: 1, + tuned: time.Now(), + OverrideTTLLimit: ttlLimit, + log: log, + } +} + +// Track inserts a new tracker into the set. +func (t *Trackers) Track(id string, tracker *Tracker) error { + t.lock.Lock() + defer t.lock.Unlock() + + if _, ok := t.trackers[id]; ok { + return errors.New("already tracking") + } + t.trackers[id] = tracker + t.detune() + + return nil +} + +// Untrack stops tracking a previously added peer. +func (t *Trackers) Untrack(id string) error { + t.lock.Lock() + defer t.lock.Unlock() + + if _, ok := t.trackers[id]; !ok { + return errors.New("not tracking") + } + delete(t.trackers, id) + return nil +} + +// MedianRoundTrip returns the median RTT across all known trackers. The purpose +// of the median RTT is to initialize a new peer with sane statistics that it will +// hopefully outperform. If it seriously underperforms, there's a risk of dropping +// the peer, but that is ok as we're aiming for a strong median. +func (t *Trackers) MedianRoundTrip() time.Duration { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.medianRoundTrip() +} + +// medianRoundTrip is the internal lockless version of MedianRoundTrip to be used +// by the QoS tuner. +func (t *Trackers) medianRoundTrip() time.Duration { + // Gather all the currently measured round trip times + rtts := make([]float64, 0, len(t.trackers)) + for _, tt := range t.trackers { + tt.lock.RLock() + rtts = append(rtts, float64(tt.roundtrip)) + tt.lock.RUnlock() + } + sort.Float64s(rtts) + + median := rttMaxEstimate + if qosTuningPeers <= len(rtts) { + median = time.Duration(rtts[qosTuningPeers/2]) // Median of our best few peers + } else if len(rtts) > 0 { + median = time.Duration(rtts[len(rtts)/2]) // Median of all out connected peers + } + // Restrict the RTT into some QoS defaults, irrelevant of true RTT + if median < rttMinEstimate { + median = rttMinEstimate + } + if median > rttMaxEstimate { + median = rttMaxEstimate + } + return median +} + +// MeanCapacities returns the capacities averaged across all the added trackers. +// The purpos of the mean capacities are to initialize a new peer with some sane +// starting values that it will hopefully outperform. If the mean overshoots, the +// peer will be cut back to minimal capacity and given another chance. +func (t *Trackers) MeanCapacities() map[uint64]float64 { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.meanCapacities() +} + +// meanCapacities is the internal lockless version of MeanCapacities used for +// debug logging. +func (t *Trackers) meanCapacities() map[uint64]float64 { + capacities := make(map[uint64]float64) + for _, tt := range t.trackers { + tt.lock.RLock() + for key, val := range tt.capacity { + capacities[key] += val + } + tt.lock.RUnlock() + } + for key, val := range capacities { + capacities[key] = val / float64(len(t.trackers)) + } + return capacities +} + +// TargetRoundTrip returns the current target round trip time for a request to +// complete in.The returned RTT is slightly under the estimated RTT. The reason +// is that message rate estimation is a 2 dimensional problem which is solvable +// for any RTT. The goal is to gravitate towards smaller RTTs instead of large +// messages, to result in a stabler download stream. +func (t *Trackers) TargetRoundTrip() time.Duration { + // Recalculate the internal caches if it's been a while + t.tune() + + // Caches surely recent, return target roundtrip + t.lock.RLock() + defer t.lock.RUnlock() + + return time.Duration(float64(t.roundtrip) * rttPushdownFactor) +} + +// TargetTimeout returns the timeout allowance for a single request to finish +// under. The timeout is proportional to the roundtrip, but also takes into +// consideration the tracker's confidence in said roundtrip and scales it +// accordingly. The final value is capped to avoid runaway requests. +func (t *Trackers) TargetTimeout() time.Duration { + // Recalculate the internal caches if it's been a while + t.tune() + + // Caches surely recent, return target timeout + t.lock.RLock() + defer t.lock.RUnlock() + + return t.targetTimeout() +} + +// targetTimeout is the internal lockless version of TargetTimeout to be used +// during QoS tuning. +func (t *Trackers) targetTimeout() time.Duration { + timeout := time.Duration(ttlScaling * float64(t.roundtrip) / t.confidence) + if timeout > t.OverrideTTLLimit { + timeout = t.OverrideTTLLimit + } + return timeout +} + +// tune gathers the individual tracker statistics and updates the estimated +// request round trip time. +func (t *Trackers) tune() { + // Tune may be called concurrently all over the place, but we only want to + // periodically update and even then only once. First check if it was updated + // recently and abort if so. + t.lock.RLock() + dirty := time.Since(t.tuned) > t.roundtrip + t.lock.RUnlock() + if !dirty { + return + } + // If an update is needed, obtain a write lock but make sure we don't update + // it on all concurrent threads one by one. + t.lock.Lock() + defer t.lock.Unlock() + + if dirty := time.Since(t.tuned) > t.roundtrip; !dirty { + return // A concurrent request beat us to the tuning + } + // First thread reaching the tuning point, update the estimates and return + t.roundtrip = time.Duration((1-tuningImpact)*float64(t.roundtrip) + tuningImpact*float64(t.medianRoundTrip())) + t.confidence = t.confidence + (1-t.confidence)/2 + + t.tuned = time.Now() + t.log.Debug("Recalculated msgrate QoS values", "rtt", t.roundtrip, "confidence", t.confidence, "ttl", t.targetTimeout(), "next", t.tuned.Add(t.roundtrip)) + t.log.Trace("Debug dump of mean capacities", "caps", log.Lazy{Fn: t.meanCapacities}) +} + +// detune reduces the tracker's confidence in order to make fresh measurements +// have a larger impact on the estimates. It is meant to be used during new peer +// connections so they can have a proper impact on the estimates. +func (t *Trackers) detune() { + // If we have a single peer, confidence is always 1 + if len(t.trackers) == 1 { + t.confidence = 1 + return + } + // If we have a ton of peers, don't drop the confidence since there's enough + // remaining to retain the same throughput + if len(t.trackers) >= tuningConfidenceCap { + return + } + // Otherwise drop the confidence factor + peers := float64(len(t.trackers)) + + t.confidence = t.confidence * (peers - 1) / peers + if t.confidence < rttMinConfidence { + t.confidence = rttMinConfidence + } + t.log.Debug("Relaxed msgrate QoS values", "rtt", t.roundtrip, "confidence", t.confidence, "ttl", t.targetTimeout()) +} + +// Capacity is a helper function to access a specific tracker without having to +// track it explicitly outside. +func (t *Trackers) Capacity(id string, kind uint64, targetRTT time.Duration) float64 { + t.lock.RLock() + defer t.lock.RUnlock() + + tracker := t.trackers[id] + if tracker == nil { + return 1 // Unregister race, don't return 0, it's a dangerous number + } + return tracker.Capacity(kind, targetRTT) +} + +// Update is a helper function to access a specific tracker without having to +// track it explicitly outside. +func (t *Trackers) Update(id string, kind uint64, elapsed time.Duration, items int) { + t.lock.RLock() + defer t.lock.RUnlock() + + if tracker := t.trackers[id]; tracker != nil { + tracker.Update(kind, elapsed, items) + } +} From 16bc57438bd3d28e947d12f6f295da62e4ca9e26 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 20 May 2021 09:24:41 +0200 Subject: [PATCH 374/709] p2p/dnsdisc: fix crash when iterator closed before first call to Next (#22906) --- p2p/dnsdisc/client.go | 6 ++++++ p2p/dnsdisc/client_test.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index f2a4bed4c6..096df06a54 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -298,6 +298,12 @@ func (it *randomIterator) pickTree() *clientTree { it.mu.Lock() defer it.mu.Unlock() + // First check if iterator was closed. + // Need to do this here to avoid nil map access in rebuildTrees. + if it.trees == nil { + return nil + } + // Rebuild the trees map if any links have changed. if it.lc.changed { it.rebuildTrees() diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index 741bee4230..9320dd667a 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -115,6 +115,21 @@ func TestIterator(t *testing.T) { checkIterator(t, it, nodes) } +func TestIteratorCloseWithoutNext(t *testing.T) { + tree1, url1 := makeTestTree("t1", nil, nil) + c := NewClient(Config{Resolver: newMapResolver(tree1.ToTXT("t1"))}) + it, err := c.NewIterator(url1) + if err != nil { + t.Fatal(err) + } + + it.Close() + ok := it.Next() + if ok { + t.Fatal("Next returned true after Close") + } +} + // This test checks if closing randomIterator races. func TestIteratorClose(t *testing.T) { nodes := testNodes(nodesSeed1, 500) From a6c462781f2ebac39b8bbcbbfeb01a6e70b46997 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 21 May 2021 09:59:26 +0200 Subject: [PATCH 375/709] EIP-1559: miner changes (#22896) * core/types, miner: create TxWithMinerFee wrapper, add EIP-1559 support to TransactionsByMinerFeeAndNonce miner: set base fee when creating a new header, handle gas limit, log miner fees * all: rename to NewTransactionsByPriceAndNonce * core/types, miner: rename to NewTransactionsByPriceAndNonce + EffectiveTip miner: activate 1559 for testGenerateBlockAndImport tests * core,miner: revert naming to TransactionsByPriceAndTime * core/types/transaction: update effective tip calculation logic * miner: update aleut to london * core/types/transaction_test: use correct signer for 1559 txs + add back sender check * miner/worker: calculate gas target from gas limit * core, miner: fix block gas limits for 1559 Co-authored-by: Ansgar Dietrichs Co-authored-by: lightclient@protonmail.com --- core/bench_test.go | 2 +- core/block_validator.go | 36 ++++++++++++--- core/block_validator_test.go | 33 ++++++++++++++ core/chain_makers.go | 2 +- core/state_processor_test.go | 2 +- core/types/transaction.go | 80 +++++++++++++++++++++++++--------- core/types/transaction_test.go | 63 ++++++++++++++++++++++---- eth/catalyst/api.go | 2 +- miner/worker.go | 28 ++++++++---- miner/worker_test.go | 6 ++- 10 files changed, 206 insertions(+), 48 deletions(-) diff --git a/core/bench_test.go b/core/bench_test.go index 0c49907e64..ce288d372e 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -112,7 +112,7 @@ func genTxRing(naccounts int) func(int, *BlockGen) { from := 0 return func(i int, gen *BlockGen) { block := gen.PrevBlock(i - 1) - gas := CalcGasLimit(block, block.GasLimit(), block.GasLimit()) + gas := block.GasLimit() for { gas -= params.TxGas if gas < params.TxGas { diff --git a/core/block_validator.go b/core/block_validator.go index 8dbd0f7552..d317d82ed4 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -106,12 +106,12 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD // to keep the baseline gas above the provided floor, and increase it towards the // ceil if the blocks are full. If the ceil is exceeded, it will always decrease // the gas allowance. -func CalcGasLimit(parent *types.Block, gasFloor, gasCeil uint64) uint64 { +func CalcGasLimit(parentGasUsed, parentGasLimit, gasFloor, gasCeil uint64) uint64 { // contrib = (parentGasUsed * 3 / 2) / 1024 - contrib := (parent.GasUsed() + parent.GasUsed()/2) / params.GasLimitBoundDivisor + contrib := (parentGasUsed + parentGasUsed/2) / params.GasLimitBoundDivisor // decay = parentGasLimit / 1024 -1 - decay := parent.GasLimit()/params.GasLimitBoundDivisor - 1 + decay := parentGasLimit/params.GasLimitBoundDivisor - 1 /* strategy: gasLimit of block-to-mine is set based on parent's @@ -120,21 +120,45 @@ func CalcGasLimit(parent *types.Block, gasFloor, gasCeil uint64) uint64 { at that usage) the amount increased/decreased depends on how far away from parentGasLimit * (2/3) parentGasUsed is. */ - limit := parent.GasLimit() - decay + contrib + limit := parentGasLimit - decay + contrib if limit < params.MinGasLimit { limit = params.MinGasLimit } // If we're outside our allowed gas range, we try to hone towards them if limit < gasFloor { - limit = parent.GasLimit() + decay + limit = parentGasLimit + decay if limit > gasFloor { limit = gasFloor } } else if limit > gasCeil { - limit = parent.GasLimit() - decay + limit = parentGasLimit - decay if limit < gasCeil { limit = gasCeil } } return limit } + +// CalcGasLimit1559 calculates the next block gas limit under 1559 rules. +func CalcGasLimit1559(parentGasLimit, desiredLimit uint64) uint64 { + delta := parentGasLimit/params.GasLimitBoundDivisor - 1 + limit := parentGasLimit + if desiredLimit < params.MinGasLimit { + desiredLimit = params.MinGasLimit + } + // If we're outside our allowed gas range, we try to hone towards them + if limit < desiredLimit { + limit = parentGasLimit + delta + if limit > desiredLimit { + limit = desiredLimit + } + return limit + } + if limit > desiredLimit { + limit = parentGasLimit - delta + if limit < desiredLimit { + limit = desiredLimit + } + } + return limit +} diff --git a/core/block_validator_test.go b/core/block_validator_test.go index dfb37b88cf..3b4de33789 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -197,3 +197,36 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { t.Errorf("verification count too large: have %d, want below %d", verified, 2*threads) } } + +func TestCalcGasLimit1559(t *testing.T) { + + for i, tc := range []struct { + pGasLimit uint64 + max uint64 + min uint64 + }{ + {20000000, 20019530, 19980470}, + {40000000, 40039061, 39960939}, + } { + // Increase + if have, want := CalcGasLimit1559(tc.pGasLimit, 2*tc.pGasLimit), tc.max; have != want { + t.Errorf("test %d: have %d want <%d", i, have, want) + } + // Decrease + if have, want := CalcGasLimit1559(tc.pGasLimit, 0), tc.min; have != want { + t.Errorf("test %d: have %d want >%d", i, have, want) + } + // Small decrease + if have, want := CalcGasLimit1559(tc.pGasLimit, tc.pGasLimit-1), tc.pGasLimit-1; have != want { + t.Errorf("test %d: have %d want %d", i, have, want) + } + // Small increase + if have, want := CalcGasLimit1559(tc.pGasLimit, tc.pGasLimit+1), tc.pGasLimit+1; have != want { + t.Errorf("test %d: have %d want %d", i, have, want) + } + // No change + if have, want := CalcGasLimit1559(tc.pGasLimit, tc.pGasLimit), tc.pGasLimit; have != want { + t.Errorf("test %d: have %d want %d", i, have, want) + } + } +} diff --git a/core/chain_makers.go b/core/chain_makers.go index b1b7dc3591..f7353ffce3 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -263,7 +263,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S Difficulty: parent.Difficulty(), UncleHash: parent.UncleHash(), }), - GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), + GasLimit: parent.GasLimit(), Number: new(big.Int).Add(parent.Number(), common.Big1), Time: time, } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index c15d3d276d..ab446eb0a2 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -223,7 +223,7 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr Difficulty: parent.Difficulty(), UncleHash: parent.UncleHash(), }), - GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), + GasLimit: parent.GasLimit(), Number: new(big.Int).Add(parent.Number(), common.Big1), Time: parent.Time() + 10, UncleHash: types.EmptyUncleHash, diff --git a/core/types/transaction.go b/core/types/transaction.go index ace1843e93..5347fdab8c 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -36,6 +36,7 @@ var ( ErrUnexpectedProtection = errors.New("transaction type does not supported EIP-155 protected signatures") ErrInvalidTxType = errors.New("transaction type not valid in this context") ErrTxTypeNotSupported = errors.New("transaction type not supported") + ErrFeeCapTooLow = errors.New("fee cap less than base fee") errEmptyTypedTx = errors.New("empty typed transaction bytes") ) @@ -299,6 +300,19 @@ func (tx *Transaction) Cost() *big.Int { return total } +// EffectiveTip returns the effective miner tip for the given base fee. +// Returns error in case of a negative effective miner tip. +func (tx *Transaction) EffectiveTip(baseFee *big.Int) (*big.Int, error) { + if baseFee == nil { + return tx.Tip(), nil + } + feeCap := tx.FeeCap() + if feeCap.Cmp(baseFee) == -1 { + return nil, ErrFeeCapTooLow + } + return math.BigMin(tx.Tip(), feeCap.Sub(feeCap, baseFee)), nil +} + // RawSignatureValues returns the V, R, S signature values of the transaction. // The return values should not be modified by the caller. func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) { @@ -400,24 +414,44 @@ func (s TxByNonce) Len() int { return len(s) } func (s TxByNonce) Less(i, j int) bool { return s[i].Nonce() < s[j].Nonce() } func (s TxByNonce) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +// TxWithMinerFee wraps a transaction with its gas price or effective miner tip +type TxWithMinerFee struct { + tx *Transaction + minerFee *big.Int +} + +// NewTxWithMinerFee creates a wrapped transaction, calculating the effective +// miner tip if a base fee is provided. +// Returns error in case of a negative effective miner tip. +func NewTxWithMinerFee(tx *Transaction, baseFee *big.Int) (*TxWithMinerFee, error) { + minerFee, err := tx.EffectiveTip(baseFee) + if err != nil { + return nil, err + } + return &TxWithMinerFee{ + tx: tx, + minerFee: minerFee, + }, nil +} + // TxByPriceAndTime implements both the sort and the heap interface, making it useful // for all at once sorting as well as individually adding and removing elements. -type TxByPriceAndTime Transactions +type TxByPriceAndTime []*TxWithMinerFee func (s TxByPriceAndTime) Len() int { return len(s) } func (s TxByPriceAndTime) Less(i, j int) bool { // If the prices are equal, use the time the transaction was first seen for // deterministic sorting - cmp := s[i].GasPrice().Cmp(s[j].GasPrice()) + cmp := s[i].minerFee.Cmp(s[j].minerFee) if cmp == 0 { - return s[i].time.Before(s[j].time) + return s[i].tx.time.Before(s[j].tx.time) } return cmp > 0 } func (s TxByPriceAndTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s *TxByPriceAndTime) Push(x interface{}) { - *s = append(*s, x.(*Transaction)) + *s = append(*s, x.(*TxWithMinerFee)) } func (s *TxByPriceAndTime) Pop() interface{} { @@ -432,9 +466,10 @@ func (s *TxByPriceAndTime) Pop() interface{} { // transactions in a profit-maximizing sorted order, while supporting removing // entire batches of transactions for non-executable accounts. type TransactionsByPriceAndNonce struct { - txs map[common.Address]Transactions // Per account nonce-sorted list of transactions - heads TxByPriceAndTime // Next transaction for each unique account (price heap) - signer Signer // Signer for the set of transactions + txs map[common.Address]Transactions // Per account nonce-sorted list of transactions + heads TxByPriceAndTime // Next transaction for each unique account (price heap) + signer Signer // Signer for the set of transactions + baseFee *big.Int // Current base fee } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -442,25 +477,28 @@ type TransactionsByPriceAndNonce struct { // // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. -func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions, baseFee *big.Int) *TransactionsByPriceAndNonce { // Initialize a price and received time based heap with the head transactions heads := make(TxByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { - // Ensure the sender address is from the signer - if acc, _ := Sender(signer, accTxs[0]); acc != from { + acc, _ := Sender(signer, accTxs[0]) + wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) + // Remove transaction if sender doesn't match from, or if wrapping fails. + if acc != from || err != nil { delete(txs, from) continue } - heads = append(heads, accTxs[0]) + heads = append(heads, wrapped) txs[from] = accTxs[1:] } heap.Init(&heads) // Assemble and return the transaction set return &TransactionsByPriceAndNonce{ - txs: txs, - heads: heads, - signer: signer, + txs: txs, + heads: heads, + signer: signer, + baseFee: baseFee, } } @@ -469,18 +507,20 @@ func (t *TransactionsByPriceAndNonce) Peek() *Transaction { if len(t.heads) == 0 { return nil } - return t.heads[0] + return t.heads[0].tx } // Shift replaces the current best head with the next one from the same account. func (t *TransactionsByPriceAndNonce) Shift() { - acc, _ := Sender(t.signer, t.heads[0]) + acc, _ := Sender(t.signer, t.heads[0].tx) if txs, ok := t.txs[acc]; ok && len(txs) > 0 { - t.heads[0], t.txs[acc] = txs[0], txs[1:] - heap.Fix(&t.heads, 0) - } else { - heap.Pop(&t.heads) + if wrapped, err := NewTxWithMinerFee(txs[0], t.baseFee); err == nil { + t.heads[0], t.txs[acc] = wrapped, txs[1:] + heap.Fix(&t.heads, 0) + return + } } + heap.Pop(&t.heads) } // Pop removes the best transaction, *not* replacing it with the next one from diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 3cece9c235..7c30834c02 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "math/big" + "math/rand" "reflect" "testing" "time" @@ -258,36 +259,77 @@ func TestRecipientNormal(t *testing.T) { } } +func TestTransactionPriceNonceSortLegacy(t *testing.T) { + testTransactionPriceNonceSort(t, nil) +} + +func TestTransactionPriceNonceSort1559(t *testing.T) { + testTransactionPriceNonceSort(t, big.NewInt(0)) + testTransactionPriceNonceSort(t, big.NewInt(5)) + testTransactionPriceNonceSort(t, big.NewInt(50)) +} + // Tests that transactions can be correctly sorted according to their price in // decreasing order, but at the same time with increasing nonces when issued by // the same account. -func TestTransactionPriceNonceSort(t *testing.T) { +func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { // Generate a batch of accounts to start with keys := make([]*ecdsa.PrivateKey, 25) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() } - signer := HomesteadSigner{} + signer := LatestSignerForChainID(common.Big1) // Generate a batch of transactions with overlapping values, but shifted nonces groups := map[common.Address]Transactions{} + expectedCount := 0 for start, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) + count := 25 for i := 0; i < 25; i++ { - tx, _ := SignTx(NewTransaction(uint64(start+i), common.Address{}, big.NewInt(100), 100, big.NewInt(int64(start+i)), nil), signer, key) + var tx *Transaction + feeCap := rand.Intn(50) + if baseFee == nil { + tx = NewTx(&LegacyTx{ + Nonce: uint64(start + i), + To: &common.Address{}, + Value: big.NewInt(100), + Gas: 100, + GasPrice: big.NewInt(int64(feeCap)), + Data: nil, + }) + } else { + tx = NewTx(&DynamicFeeTx{ + Nonce: uint64(start + i), + To: &common.Address{}, + Value: big.NewInt(100), + Gas: 100, + FeeCap: big.NewInt(int64(feeCap)), + Tip: big.NewInt(int64(rand.Intn(feeCap + 1))), + Data: nil, + }) + if count == 25 && int64(feeCap) < baseFee.Int64() { + count = i + } + } + tx, err := SignTx(tx, signer, key) + if err != nil { + t.Fatalf("failed to sign tx: %s", err) + } groups[addr] = append(groups[addr], tx) } + expectedCount += count } // Sort the transactions and cross check the nonce ordering - txset := NewTransactionsByPriceAndNonce(signer, groups) + txset := NewTransactionsByPriceAndNonce(signer, groups, baseFee) txs := Transactions{} for tx := txset.Peek(); tx != nil; tx = txset.Peek() { txs = append(txs, tx) txset.Shift() } - if len(txs) != 25*25 { - t.Errorf("expected %d transactions, found %d", 25*25, len(txs)) + if len(txs) != expectedCount { + t.Errorf("expected %d transactions, found %d", expectedCount, len(txs)) } for i, txi := range txs { fromi, _ := Sender(signer, txi) @@ -303,7 +345,12 @@ func TestTransactionPriceNonceSort(t *testing.T) { if i+1 < len(txs) { next := txs[i+1] fromNext, _ := Sender(signer, next) - if fromi != fromNext && txi.GasPrice().Cmp(next.GasPrice()) < 0 { + tip, err := txi.EffectiveTip(baseFee) + nextTip, nextErr := next.EffectiveTip(baseFee) + if err != nil || nextErr != nil { + t.Errorf("error calculating effective tip") + } + if fromi != fromNext && tip.Cmp(nextTip) < 0 { t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) < tx #%d (A=%x P=%v)", i, fromi[:4], txi.GasPrice(), i+1, fromNext[:4], next.GasPrice()) } } @@ -331,7 +378,7 @@ func TestTransactionTimeSort(t *testing.T) { groups[addr] = append(groups[addr], tx) } // Sort the transactions and cross check the nonce ordering - txset := NewTransactionsByPriceAndNonce(signer, groups) + txset := NewTransactionsByPriceAndNonce(signer, groups, nil) txs := Transactions{} for tx := txset.Peek(); tx != nil; tx = txset.Peek() { diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index d7e2af1c1a..d577e2a9ec 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -155,7 +155,7 @@ func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableD var ( signer = types.MakeSigner(bc.Config(), header.Number) - txHeap = types.NewTransactionsByPriceAndNonce(signer, pending) + txHeap = types.NewTransactionsByPriceAndNonce(signer, pending, nil) transactions []*types.Transaction ) for { diff --git a/miner/worker.go b/miner/worker.go index 2cee6af0c3..f9aae0fdc9 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -499,7 +499,7 @@ func (w *worker) mainLoop() { acc, _ := types.Sender(w.current.signer, tx) txs[acc] = append(txs[acc], tx) } - txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs) + txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) tcount := w.current.tcount w.commitTransactions(txset, coinbase, nil) // Only update the snapshot if any new transactons were added @@ -753,8 +753,9 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin return true } + gasLimit := w.current.header.GasLimit if w.current.gasPool == nil { - w.current.gasPool = new(core.GasPool).AddGas(w.current.header.GasLimit) + w.current.gasPool = new(core.GasPool).AddGas(gasLimit) } var coalescedLogs []*types.Log @@ -769,7 +770,7 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { // Notify resubmit loop to increase resubmitting interval due to too frequent commits. if atomic.LoadInt32(interrupt) == commitInterruptResubmit { - ratio := float64(w.current.header.GasLimit-w.current.gasPool.Gas()) / float64(w.current.header.GasLimit) + ratio := float64(gasLimit-w.current.gasPool.Gas()) / float64(gasLimit) if ratio < 0.1 { ratio = 0.1 } @@ -880,10 +881,20 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) header := &types.Header{ ParentHash: parent.Hash(), Number: num.Add(num, common.Big1), - GasLimit: core.CalcGasLimit(parent, w.config.GasFloor, w.config.GasCeil), + GasLimit: core.CalcGasLimit(parent.GasUsed(), parent.GasLimit(), w.config.GasFloor, w.config.GasCeil), Extra: w.extra, Time: uint64(timestamp), } + // Set baseFee and GasLimit if we are on an EIP-1559 chain + if w.chainConfig.IsLondon(header.Number) { + header.BaseFee = misc.CalcBaseFee(w.chainConfig, parent.Header()) + parentGasLimit := parent.GasLimit() + if !w.chainConfig.IsLondon(parent.Number()) { + // Bump by 2x + parentGasLimit = parent.GasLimit() * params.ElasticityMultiplier + } + header.GasLimit = core.CalcGasLimit1559(parentGasLimit, w.config.GasCeil) + } // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) if w.isRunning() { if w.coinbase == (common.Address{}) { @@ -973,13 +984,13 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) } } if len(localTxs) > 0 { - txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs) + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs, header.BaseFee) if w.commitTransactions(txs, w.coinbase, interrupt) { return } } if len(remoteTxs) > 0 { - txs := types.NewTransactionsByPriceAndNonce(w.current.signer, remoteTxs) + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, remoteTxs, header.BaseFee) if w.commitTransactions(txs, w.coinbase, interrupt) { return } @@ -1037,11 +1048,12 @@ func (w *worker) postSideBlock(event core.ChainSideEvent) { } } -// totalFees computes total consumed fees in ETH. Block transactions and receipts have to have the same order. +// totalFees computes total consumed miner fees in ETH. Block transactions and receipts have to have the same order. func totalFees(block *types.Block, receipts []*types.Receipt) *big.Float { feesWei := new(big.Int) for i, tx := range block.Transactions() { - feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), tx.GasPrice())) + minerFee, _ := tx.EffectiveTip(block.BaseFee()) + feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), minerFee)) } return new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) } diff --git a/miner/worker_test.go b/miner/worker_test.go index 0fe62316e1..0a1e55ff33 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -182,10 +182,11 @@ func (b *testWorkerBackend) newRandomUncle() *types.Block { func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { var tx *types.Transaction + gasPrice := big.NewInt(10 * params.InitialBaseFee) if creation { - tx, _ = types.SignTx(types.NewContractCreation(b.txPool.Nonce(testBankAddress), big.NewInt(0), testGas, nil, common.FromHex(testCode)), types.HomesteadSigner{}, testBankKey) + tx, _ = types.SignTx(types.NewContractCreation(b.txPool.Nonce(testBankAddress), big.NewInt(0), testGas, gasPrice, common.FromHex(testCode)), types.HomesteadSigner{}, testBankKey) } else { - tx, _ = types.SignTx(types.NewTransaction(b.txPool.Nonce(testBankAddress), testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + tx, _ = types.SignTx(types.NewTransaction(b.txPool.Nonce(testBankAddress), testUserAddress, big.NewInt(1000), params.TxGas, gasPrice, nil), types.HomesteadSigner{}, testBankKey) } return tx } @@ -221,6 +222,7 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool) { engine = ethash.NewFaker() } + chainConfig.LondonBlock = big.NewInt(0) w, b := newTestWorker(t, chainConfig, engine, db, 0) defer w.close() From 81662fe82788a7b66aa95de86d7afd2cb4567370 Mon Sep 17 00:00:00 2001 From: Evolution404 <35091674+Evolution404@users.noreply.github.com> Date: Fri, 21 May 2021 16:33:59 +0800 Subject: [PATCH 376/709] core/rawdb: handle prefix in table.compact method (#22911) --- core/rawdb/table.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 323ef6293c..d5ef60ae50 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -131,6 +131,8 @@ func (t *table) Compact(start []byte, limit []byte) error { // If no start was specified, use the table prefix as the first value if start == nil { start = []byte(t.prefix) + } else { + start = append([]byte(t.prefix), start...) } // If no limit was specified, use the first element not matching the prefix // as the limit @@ -147,6 +149,8 @@ func (t *table) Compact(start []byte, limit []byte) error { limit = nil } } + } else { + limit = append([]byte(t.prefix), limit...) } // Range correctly calculated based on table prefix, delegate down return t.db.Compact(start, limit) From 835fe06f1dba8ce2764d6d501f977c22e97eac9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 21 May 2021 12:36:04 +0300 Subject: [PATCH 377/709] les: generate random nums directly, not via strange conversions --- les/client_handler.go | 7 ++++--- les/fetcher.go | 2 +- les/odr.go | 5 +++-- les/retrieve.go | 9 --------- les/txrelay.go | 3 ++- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/les/client_handler.go b/les/client_handler.go index 73149975c3..e95996c51f 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -19,6 +19,7 @@ package les import ( "context" "math/big" + "math/rand" "sync" "sync/atomic" "time" @@ -388,7 +389,7 @@ func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, s return dp.(*serverPeer) == pc.peer }, request: func(dp distPeer) func() { - reqID := genReqID() + reqID := rand.Uint64() peer := dp.(*serverPeer) cost := peer.getRequestCost(GetBlockHeadersMsg, amount) peer.fcServer.QueuedRequest(reqID, cost) @@ -412,7 +413,7 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip return dp.(*serverPeer) == pc.peer }, request: func(dp distPeer) func() { - reqID := genReqID() + reqID := rand.Uint64() peer := dp.(*serverPeer) cost := peer.getRequestCost(GetBlockHeadersMsg, amount) peer.fcServer.QueuedRequest(reqID, cost) @@ -429,7 +430,7 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip // RetrieveSingleHeaderByNumber requests a single header by the specified block // number. This function will wait the response until it's timeout or delivered. func (pc *peerConnection) RetrieveSingleHeaderByNumber(context context.Context, number uint64) (*types.Header, error) { - reqID := genReqID() + reqID := rand.Uint64() rq := &distReq{ getCost: func(dp distPeer) uint64 { peer := dp.(*serverPeer) diff --git a/les/fetcher.go b/les/fetcher.go index fc4c5e386a..a6d1c93c4b 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -507,7 +507,7 @@ func (f *lightFetcher) requestHeaderByHash(peerid enode.ID) func(common.Hash) er getCost: func(dp distPeer) uint64 { return dp.(*serverPeer).getRequestCost(GetBlockHeadersMsg, 1) }, canSend: func(dp distPeer) bool { return dp.(*serverPeer).ID() == peerid }, request: func(dp distPeer) func() { - peer, id := dp.(*serverPeer), genReqID() + peer, id := dp.(*serverPeer), rand.Uint64() cost := peer.getRequestCost(GetBlockHeadersMsg, 1) peer.fcServer.QueuedRequest(id, cost) diff --git a/les/odr.go b/les/odr.go index d45c6a1a5d..10ff0854d3 100644 --- a/les/odr.go +++ b/les/odr.go @@ -18,6 +18,7 @@ package les import ( "context" + "math/rand" "sort" "time" @@ -156,7 +157,7 @@ func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequ var ( // Deep copy the request, so that the partial result won't be mixed. req = &TxStatusRequest{Hashes: req.Hashes} - id = genReqID() + id = rand.Uint64() distreq = &distReq{ getCost: func(dp distPeer) uint64 { return req.GetCost(dp.(*serverPeer)) }, canSend: func(dp distPeer) bool { return canSend[dp.(*serverPeer).id] }, @@ -200,7 +201,7 @@ func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequ func (odr *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err error) { lreq := LesRequest(req) - reqID := genReqID() + reqID := rand.Uint64() rq := &distReq{ getCost: func(dp distPeer) uint64 { return lreq.GetCost(dp.(*serverPeer)) diff --git a/les/retrieve.go b/les/retrieve.go index 3174d49878..307af04212 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -18,8 +18,6 @@ package les import ( "context" - "crypto/rand" - "encoding/binary" "fmt" "sync" "time" @@ -430,10 +428,3 @@ func (r *sentReq) stop(err error) { func (r *sentReq) getError() error { return r.err } - -// genReqID generates a new random request ID -func genReqID() uint64 { - var rnd [8]byte - rand.Read(rnd[:]) - return binary.BigEndian.Uint64(rnd[:]) -} diff --git a/les/txrelay.go b/les/txrelay.go index 9d29b2f234..40a51fb76f 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -18,6 +18,7 @@ package les import ( "context" + "math/rand" "sync" "github.com/ethereum/go-ethereum/common" @@ -117,7 +118,7 @@ func (ltrx *lesTxRelay) send(txs types.Transactions, count int) { ll := list enc, _ := rlp.EncodeToBytes(ll) - reqID := genReqID() + reqID := rand.Uint64() rq := &distReq{ getCost: func(dp distPeer) uint64 { peer := dp.(*serverPeer) From 59f259b058b85eea38cd2686051a9076abb1e712 Mon Sep 17 00:00:00 2001 From: gary rong Date: Sat, 22 May 2021 02:52:51 +0800 Subject: [PATCH 378/709] miner/stress: update stress tests (#22919) This PR updates the miner stress tests and moves them to standalone packages, so that they can be run directly. --- miner/{stress_clique.go => stress/clique/clique.go} | 5 ++--- miner/{stress_ethash.go => stress/ethash/main.go} | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) rename miner/{stress_clique.go => stress/clique/clique.go} (98%) rename miner/{stress_ethash.go => stress/ethash/main.go} (97%) diff --git a/miner/stress_clique.go b/miner/stress/clique/clique.go similarity index 98% rename from miner/stress_clique.go rename to miner/stress/clique/clique.go index c585e0b1f6..dea1ab7453 100644 --- a/miner/stress_clique.go +++ b/miner/stress/clique/clique.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build none - // This file contains a miner stress test based on the Clique consensus engine. package main @@ -36,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" @@ -192,7 +191,7 @@ func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { DatabaseCache: 256, DatabaseHandles: 256, TxPool: core.DefaultTxPoolConfig, - GPO: eth.DefaultConfig.GPO, + GPO: ethconfig.Defaults.GPO, Miner: miner.Config{ GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, diff --git a/miner/stress_ethash.go b/miner/stress/ethash/main.go similarity index 97% rename from miner/stress_ethash.go rename to miner/stress/ethash/main.go index 0b838d48b9..0f27c5e74c 100644 --- a/miner/stress_ethash.go +++ b/miner/stress/ethash/main.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build none - // This file contains a miner stress test based on the Ethash consensus engine. package main @@ -37,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" @@ -169,8 +168,8 @@ func makeMiner(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { DatabaseCache: 256, DatabaseHandles: 256, TxPool: core.DefaultTxPoolConfig, - GPO: eth.DefaultConfig.GPO, - Ethash: eth.DefaultConfig.Ethash, + GPO: ethconfig.Defaults.GPO, + Ethash: ethconfig.Defaults.Ethash, Miner: miner.Config{ GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, From 0d076d92db39940ff181a0b07970c21bbe3521c2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 22 May 2021 13:34:29 +0200 Subject: [PATCH 379/709] rlp: use atomic.Value for type cache (#22902) All encoding/decoding operations read the type cache to find the writer/decoder function responsible for a type. When analyzing CPU profiles of geth during sync, I found that the use of sync.RWMutex in cache lookups appears in the profiles. It seems we are running into CPU cache contention problems when package rlp is heavily used on all CPU cores during sync. This change makes it use atomic.Value + a writer lock instead of sync.RWMutex. In the common case where the typeinfo entry is present in the cache, we simply fetch the map and lookup the type. --- rlp/decode.go | 4 +-- rlp/encode.go | 4 +-- rlp/encode_test.go | 33 +++++++++++++++++++ rlp/typecache.go | 82 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 94 insertions(+), 29 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index 9767809717..e4262b64df 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -245,7 +245,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { } return decodeByteSlice, nil } - etypeinfo := cachedTypeInfo1(etype, tags{}) + etypeinfo := theTC.infoWhileGenerating(etype, tags{}) if etypeinfo.decoderErr != nil { return nil, etypeinfo.decoderErr } @@ -424,7 +424,7 @@ func zeroFields(structval reflect.Value, fields []field) { // makePtrDecoder creates a decoder that decodes into the pointer's element type. func makePtrDecoder(typ reflect.Type, tag tags) (decoder, error) { etype := typ.Elem() - etypeinfo := cachedTypeInfo1(etype, tags{}) + etypeinfo := theTC.infoWhileGenerating(etype, tags{}) switch { case etypeinfo.decoderErr != nil: return nil, etypeinfo.decoderErr diff --git a/rlp/encode.go b/rlp/encode.go index b7e74a133f..2e1b0102ca 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -517,7 +517,7 @@ func writeInterface(val reflect.Value, w *encbuf) error { } func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := cachedTypeInfo1(typ.Elem(), tags{}) + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } @@ -585,7 +585,7 @@ func makeStructWriter(typ reflect.Type) (writer, error) { } func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := cachedTypeInfo1(typ.Elem(), tags{}) + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 74e8ededcb..0177bb0350 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -23,6 +23,7 @@ import ( "io" "io/ioutil" "math/big" + "runtime" "sync" "testing" @@ -480,3 +481,35 @@ func BenchmarkEncodeBigInts(b *testing.B) { } } } + +func BenchmarkEncodeConcurrentInterface(b *testing.B) { + type struct1 struct { + A string + B *big.Int + C [20]byte + } + value := []interface{}{ + uint(999), + &struct1{A: "hello", B: big.NewInt(0xFFFFFFFF)}, + [10]byte{1, 2, 3, 4, 5, 6}, + []string{"yeah", "yeah", "yeah"}, + } + + var wg sync.WaitGroup + for cpu := 0; cpu < runtime.NumCPU(); cpu++ { + wg.Add(1) + go func() { + defer wg.Done() + + var buffer bytes.Buffer + for i := 0; i < b.N; i++ { + buffer.Reset() + err := Encode(&buffer, value) + if err != nil { + panic(err) + } + } + }() + } + wg.Wait() +} diff --git a/rlp/typecache.go b/rlp/typecache.go index 3910dcf080..62553d3b55 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -21,13 +21,10 @@ import ( "reflect" "strings" "sync" + "sync/atomic" ) -var ( - typeCacheMutex sync.RWMutex - typeCache = make(map[typekey]*typeinfo) -) - +// typeinfo is an entry in the type cache. type typeinfo struct { decoder decoder decoderErr error // error from makeDecoder @@ -65,41 +62,76 @@ type decoder func(*Stream, reflect.Value) error type writer func(reflect.Value, *encbuf) error +var theTC = newTypeCache() + +type typeCache struct { + cur atomic.Value + + // This lock synchronizes writers. + mu sync.Mutex + next map[typekey]*typeinfo +} + +func newTypeCache() *typeCache { + c := new(typeCache) + c.cur.Store(make(map[typekey]*typeinfo)) + return c +} + func cachedDecoder(typ reflect.Type) (decoder, error) { - info := cachedTypeInfo(typ, tags{}) + info := theTC.info(typ) return info.decoder, info.decoderErr } func cachedWriter(typ reflect.Type) (writer, error) { - info := cachedTypeInfo(typ, tags{}) + info := theTC.info(typ) return info.writer, info.writerErr } -func cachedTypeInfo(typ reflect.Type, tags tags) *typeinfo { - typeCacheMutex.RLock() - info := typeCache[typekey{typ, tags}] - typeCacheMutex.RUnlock() - if info != nil { +func (c *typeCache) info(typ reflect.Type) *typeinfo { + key := typekey{Type: typ} + if info := c.cur.Load().(map[typekey]*typeinfo)[key]; info != nil { return info } - // not in the cache, need to generate info for this type. - typeCacheMutex.Lock() - defer typeCacheMutex.Unlock() - return cachedTypeInfo1(typ, tags) + + // Not in the cache, need to generate info for this type. + return c.generate(typ, tags{}) +} + +func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { + c.mu.Lock() + defer c.mu.Unlock() + + cur := c.cur.Load().(map[typekey]*typeinfo) + if info := cur[typekey{typ, tags}]; info != nil { + return info + } + + // Copy cur to next. + c.next = make(map[typekey]*typeinfo, len(cur)+1) + for k, v := range cur { + c.next[k] = v + } + + // Generate. + info := c.infoWhileGenerating(typ, tags) + + // next -> cur + c.cur.Store(c.next) + c.next = nil + return info } -func cachedTypeInfo1(typ reflect.Type, tags tags) *typeinfo { +func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags tags) *typeinfo { key := typekey{typ, tags} - info := typeCache[key] - if info != nil { - // another goroutine got the write lock first + if info := c.next[key]; info != nil { return info } - // put a dummy value into the cache before generating. - // if the generator tries to lookup itself, it will get + // Put a dummy value into the cache before generating. + // If the generator tries to lookup itself, it will get // the dummy value and won't call itself recursively. - info = new(typeinfo) - typeCache[key] = info + info := new(typeinfo) + c.next[key] = info info.generate(typ, tags) return info } @@ -133,7 +165,7 @@ func structFields(typ reflect.Type) (fields []field, err error) { } else if anyOptional { return nil, fmt.Errorf(`rlp: struct field %v.%s needs "optional" tag`, typ, f.Name) } - info := cachedTypeInfo1(f.Type, tags) + info := theTC.infoWhileGenerating(f.Type, tags) fields = append(fields, field{i, info, tags.optional}) } } From 154ca32a8ade0a7d2461d0eb432361261a51a395 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 22 May 2021 15:10:16 +0200 Subject: [PATCH 380/709] rlp: optimize byte array handling (#22924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change improves the performance of encoding/decoding [N]byte. name old time/op new time/op delta DecodeByteArrayStruct-8 336ns ± 0% 246ns ± 0% -26.98% (p=0.000 n=9+10) EncodeByteArrayStruct-8 225ns ± 1% 148ns ± 1% -34.12% (p=0.000 n=10+10) name old alloc/op new alloc/op delta DecodeByteArrayStruct-8 120B ± 0% 48B ± 0% -60.00% (p=0.000 n=10+10) EncodeByteArrayStruct-8 0.00B 0.00B ~ (all equal) --- rlp/decode.go | 16 ++++++------- rlp/decode_test.go | 44 ++++++++++++++++++++++++++++++++++-- rlp/encode.go | 56 ++++++++++++++-------------------------------- rlp/encode_test.go | 19 ++++++++++++++++ rlp/safe.go | 26 +++++++++++++++++++++ rlp/unsafe.go | 35 +++++++++++++++++++++++++++++ 6 files changed, 146 insertions(+), 50 deletions(-) create mode 100644 rlp/safe.go create mode 100644 rlp/unsafe.go diff --git a/rlp/decode.go b/rlp/decode.go index e4262b64df..8121ab2e72 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -348,25 +348,23 @@ func decodeByteArray(s *Stream, val reflect.Value) error { if err != nil { return err } - vlen := val.Len() + slice := byteArrayBytes(val) switch kind { case Byte: - if vlen == 0 { + if len(slice) == 0 { return &decodeError{msg: "input string too long", typ: val.Type()} - } - if vlen > 1 { + } else if len(slice) > 1 { return &decodeError{msg: "input string too short", typ: val.Type()} } - bv, _ := s.Uint() - val.Index(0).SetUint(bv) + slice[0] = s.byteval + s.kind = -1 case String: - if uint64(vlen) < size { + if uint64(len(slice)) < size { return &decodeError{msg: "input string too long", typ: val.Type()} } - if uint64(vlen) > size { + if uint64(len(slice)) > size { return &decodeError{msg: "input string too short", typ: val.Type()} } - slice := val.Slice(0, vlen).Interface().([]byte) if err := s.readFull(slice); err != nil { return err } diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 87a3306332..36d254e18e 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -26,6 +26,8 @@ import ( "reflect" "strings" "testing" + + "github.com/ethereum/go-ethereum/common/math" ) func TestStreamKind(t *testing.T) { @@ -1063,7 +1065,7 @@ func ExampleStream() { // [102 111 111 98 97 114] } -func BenchmarkDecode(b *testing.B) { +func BenchmarkDecodeUints(b *testing.B) { enc := encodeTestSlice(90000) b.SetBytes(int64(len(enc))) b.ReportAllocs() @@ -1078,7 +1080,7 @@ func BenchmarkDecode(b *testing.B) { } } -func BenchmarkDecodeIntSliceReuse(b *testing.B) { +func BenchmarkDecodeUintsReused(b *testing.B) { enc := encodeTestSlice(100000) b.SetBytes(int64(len(enc))) b.ReportAllocs() @@ -1093,6 +1095,44 @@ func BenchmarkDecodeIntSliceReuse(b *testing.B) { } } +func BenchmarkDecodeByteArrayStruct(b *testing.B) { + enc, err := EncodeToBytes(&byteArrayStruct{}) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out byteArrayStruct + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeBigInts(b *testing.B) { + ints := make([]*big.Int, 200) + for i := range ints { + ints[i] = math.BigPow(2, int64(i)) + } + enc, err := EncodeToBytes(ints) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out []*big.Int + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + func encodeTestSlice(n uint) []byte { s := make([]uint, n) for i := uint(0); i < n; i++ { diff --git a/rlp/encode.go b/rlp/encode.go index 2e1b0102ca..3348644342 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -124,19 +124,15 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { } type encbuf struct { - str []byte // string data, contains everything except list headers - lheads []listhead // all list headers - lhsize int // sum of sizes of all encoded list headers - sizebuf [9]byte // auxiliary buffer for uint encoding - bufvalue reflect.Value // used in writeByteArrayCopy + str []byte // string data, contains everything except list headers + lheads []listhead // all list headers + lhsize int // sum of sizes of all encoded list headers + sizebuf [9]byte // auxiliary buffer for uint encoding } // encbufs are pooled. var encbufPool = sync.Pool{ - New: func() interface{} { - var bytes []byte - return &encbuf{bufvalue: reflect.ValueOf(&bytes).Elem()} - }, + New: func() interface{} { return new(encbuf) }, } func (w *encbuf) reset() { @@ -429,21 +425,14 @@ func writeBytes(val reflect.Value, w *encbuf) error { return nil } -var byteType = reflect.TypeOf(byte(0)) - func makeByteArrayWriter(typ reflect.Type) writer { - length := typ.Len() - if length == 0 { + switch typ.Len() { + case 0: return writeLengthZeroByteArray - } else if length == 1 { + case 1: return writeLengthOneByteArray - } - if typ.Elem() != byteType { - return writeNamedByteArray - } - return func(val reflect.Value, w *encbuf) error { - writeByteArrayCopy(length, val, w) - return nil + default: + return writeByteArray } } @@ -462,29 +451,18 @@ func writeLengthOneByteArray(val reflect.Value, w *encbuf) error { return nil } -// writeByteArrayCopy encodes byte arrays using reflect.Copy. This is -// the fast path for [N]byte where N > 1. -func writeByteArrayCopy(length int, val reflect.Value, w *encbuf) { - w.encodeStringHeader(length) - offset := len(w.str) - w.str = append(w.str, make([]byte, length)...) - w.bufvalue.SetBytes(w.str[offset:]) - reflect.Copy(w.bufvalue, val) -} - -// writeNamedByteArray encodes byte arrays with named element type. -// This exists because reflect.Copy can't be used with such types. -func writeNamedByteArray(val reflect.Value, w *encbuf) error { +func writeByteArray(val reflect.Value, w *encbuf) error { if !val.CanAddr() { - // Slice requires the value to be addressable. - // Make it addressable by copying. + // Getting the byte slice of val requires it to be addressable. Make it + // addressable by copying. copy := reflect.New(val.Type()).Elem() copy.Set(val) val = copy } - size := val.Len() - slice := val.Slice(0, size).Bytes() - w.encodeString(slice) + + slice := byteArrayBytes(val) + w.encodeStringHeader(len(slice)) + w.str = append(w.str, slice...) return nil } diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 0177bb0350..08a2a84c62 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -513,3 +513,22 @@ func BenchmarkEncodeConcurrentInterface(b *testing.B) { } wg.Wait() } + +type byteArrayStruct struct { + A [20]byte + B [32]byte + C [32]byte +} + +func BenchmarkEncodeByteArrayStruct(b *testing.B) { + var out bytes.Buffer + var value byteArrayStruct + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} diff --git a/rlp/safe.go b/rlp/safe.go new file mode 100644 index 0000000000..c881650a0d --- /dev/null +++ b/rlp/safe.go @@ -0,0 +1,26 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build nacl js !cgo + +package rlp + +import "reflect" + +// byteArrayBytes returns a slice of the byte array v. +func byteArrayBytes(v reflect.Value) []byte { + return v.Slice(0, v.Len()).Bytes() +} diff --git a/rlp/unsafe.go b/rlp/unsafe.go new file mode 100644 index 0000000000..94ed5405a8 --- /dev/null +++ b/rlp/unsafe.go @@ -0,0 +1,35 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !nacl,!js,cgo + +package rlp + +import ( + "reflect" + "unsafe" +) + +// byteArrayBytes returns a slice of the byte array v. +func byteArrayBytes(v reflect.Value) []byte { + len := v.Len() + var s []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + hdr.Data = v.UnsafeAddr() + hdr.Cap = len + hdr.Len = len + return s +} From 93407b14a66287d1a0ad0140b9c5f754c11ad437 Mon Sep 17 00:00:00 2001 From: Fire Man <55934298+basdevelop@users.noreply.github.com> Date: Mon, 24 May 2021 20:34:38 +0800 Subject: [PATCH 381/709] core: make txpool free space calculation more accurate (#22933) --- core/tx_pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 5db1d3df32..0abc092ecb 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -595,7 +595,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e return false, err } // If the transaction pool is full, discard underpriced transactions - if uint64(pool.all.Count()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue { + if uint64(pool.all.Slots()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it if !isLocal && pool.priced.Underpriced(tx) { log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice()) From 017cf71fbd51ede510dd0e40e02f89b5340eea93 Mon Sep 17 00:00:00 2001 From: ucwong Date: Tue, 25 May 2021 16:14:39 +0800 Subject: [PATCH 382/709] rlp, tests/fuzzers/bls12381: gofmt (#22937) --- rlp/safe.go | 2 +- tests/fuzzers/bls12381/bls12381_fuzz.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rlp/safe.go b/rlp/safe.go index c881650a0d..a80380aef4 100644 --- a/rlp/safe.go +++ b/rlp/safe.go @@ -22,5 +22,5 @@ import "reflect" // byteArrayBytes returns a slice of the byte array v. func byteArrayBytes(v reflect.Value) []byte { - return v.Slice(0, v.Len()).Bytes() + return v.Slice(0, v.Len()).Bytes() } diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index 298050ad36..c0f452f3ed 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -159,7 +159,7 @@ func FuzzCrossG1MultiExp(data []byte) int { gethPoints = append(gethPoints, new(bls12381.PointG1).Set(kp1)) gnarkPoints = append(gnarkPoints, *cp1) } - if len(gethScalars) == 0{ + if len(gethScalars) == 0 { return 0 } // compute multi exponentiation From 4d33de9b4975be0f3450fa44d6c912e4331ca0c8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 25 May 2021 21:56:25 +0200 Subject: [PATCH 383/709] rlp: optimize big.Int decoding for size <= 32 bytes (#22927) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change grows the static integer buffer in Stream to 32 bytes, making it possible to decode 256bit integers without allocating a temporary buffer. In the recent commit 088da24, Stream struct size decreased from 120 bytes down to 88 bytes. This commit grows the struct to 112 bytes again, but the size change will not degrade performance because Stream instances are internally cached in sync.Pool. name old time/op new time/op delta DecodeBigInts-8 12.2µs ± 0% 8.6µs ± 4% -29.58% (p=0.000 n=9+10) name old speed new speed delta DecodeBigInts-8 230MB/s ± 0% 326MB/s ± 4% +42.04% (p=0.000 n=9+10) --- rlp/decode.go | 60 ++++++++++++++++++++++++++++++++++++---------- rlp/decode_test.go | 22 ++++++++++++++--- rlp/encode_test.go | 8 +++++++ 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index 8121ab2e72..ac04d5d569 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -220,20 +220,51 @@ func decodeBigIntNoPtr(s *Stream, val reflect.Value) error { } func decodeBigInt(s *Stream, val reflect.Value) error { - b, err := s.Bytes() - if err != nil { + var buffer []byte + kind, size, err := s.Kind() + switch { + case err != nil: return wrapStreamError(err, val.Type()) + case kind == List: + return wrapStreamError(ErrExpectedString, val.Type()) + case kind == Byte: + buffer = s.uintbuf[:1] + buffer[0] = s.byteval + s.kind = -1 // re-arm Kind + case size == 0: + // Avoid zero-length read. + s.kind = -1 + case size <= uint64(len(s.uintbuf)): + // For integers smaller than s.uintbuf, allocating a buffer + // can be avoided. + buffer = s.uintbuf[:size] + if err := s.readFull(buffer); err != nil { + return wrapStreamError(err, val.Type()) + } + // Reject inputs where single byte encoding should have been used. + if size == 1 && buffer[0] < 128 { + return wrapStreamError(ErrCanonSize, val.Type()) + } + default: + // For large integers, a temporary buffer is needed. + buffer = make([]byte, size) + if err := s.readFull(buffer); err != nil { + return wrapStreamError(err, val.Type()) + } + } + + // Reject leading zero bytes. + if len(buffer) > 0 && buffer[0] == 0 { + return wrapStreamError(ErrCanonInt, val.Type()) } + + // Set the integer bytes. i := val.Interface().(*big.Int) if i == nil { i = new(big.Int) val.Set(reflect.ValueOf(i)) } - // Reject leading zero bytes. - if len(b) > 0 && b[0] == 0 { - return wrapStreamError(ErrCanonInt, val.Type()) - } - i.SetBytes(b) + i.SetBytes(buffer) return nil } @@ -563,7 +594,7 @@ type Stream struct { size uint64 // size of value ahead kinderr error // error from last readKind stack []uint64 // list sizes - uintbuf [8]byte // auxiliary buffer for integer decoding + uintbuf [32]byte // auxiliary buffer for integer decoding kind Kind // kind of value ahead byteval byte // value of single byte in type tag limited bool // true if input limit is in effect @@ -817,7 +848,7 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { s.kind = -1 s.kinderr = nil s.byteval = 0 - s.uintbuf = [8]byte{} + s.uintbuf = [32]byte{} } // Kind returns the kind and size of the next value in the @@ -927,17 +958,20 @@ func (s *Stream) readUint(size byte) (uint64, error) { b, err := s.readByte() return uint64(b), err default: + buffer := s.uintbuf[:8] + for i := range buffer { + buffer[i] = 0 + } start := int(8 - size) - s.uintbuf = [8]byte{} - if err := s.readFull(s.uintbuf[start:]); err != nil { + if err := s.readFull(buffer[start:]); err != nil { return 0, err } - if s.uintbuf[start] == 0 { + if buffer[start] == 0 { // Note: readUint is also used to decode integer values. // The error needs to be adjusted to become ErrCanonInt in this case. return 0, ErrCanonSize } - return binary.BigEndian.Uint64(s.uintbuf[:]), nil + return binary.BigEndian.Uint64(buffer[:]), nil } } diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 36d254e18e..7c3dafeac4 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -329,6 +329,11 @@ type recstruct struct { Child *recstruct `rlp:"nil"` } +type bigIntStruct struct { + I *big.Int + B string +} + type invalidNilTag struct { X []byte `rlp:"nil"` } @@ -405,10 +410,11 @@ type ignoredField struct { } var ( - veryBigInt = big.NewInt(0).Add( + veryBigInt = new(big.Int).Add( big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), big.NewInt(0xFFFF), ) + veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil) ) var decodeTests = []decodeTest{ @@ -479,12 +485,15 @@ var decodeTests = []decodeTest{ {input: "C0", ptr: new(string), error: "rlp: expected input string or byte for string"}, // big ints + {input: "80", ptr: new(*big.Int), value: big.NewInt(0)}, {input: "01", ptr: new(*big.Int), value: big.NewInt(1)}, {input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*big.Int), value: veryBigInt}, + {input: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", ptr: new(*big.Int), value: veryVeryBigInt}, {input: "10", ptr: new(big.Int), value: *big.NewInt(16)}, // non-pointer also works {input: "C0", ptr: new(*big.Int), error: "rlp: expected input string or byte for *big.Int"}, - {input: "820001", ptr: new(big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, - {input: "8105", ptr: new(big.Int), error: "rlp: non-canonical size information for *big.Int"}, + {input: "00", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, + {input: "820001", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, + {input: "8105", ptr: new(*big.Int), error: "rlp: non-canonical size information for *big.Int"}, // structs { @@ -497,6 +506,13 @@ var decodeTests = []decodeTest{ ptr: new(recstruct), value: recstruct{1, &recstruct{2, &recstruct{3, nil}}}, }, + { + // This checks that empty big.Int works correctly in struct context. It's easy to + // miss the update of s.kind for this case, so it needs its own test. + input: "C58083343434", + ptr: new(bigIntStruct), + value: bigIntStruct{new(big.Int), "444"}, + }, // struct errors { diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 08a2a84c62..25d4aac267 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -131,6 +131,14 @@ var encTests = []encTest{ val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), output: "A1010000000000000000000000000000000000000000000000000000000000000000", }, + { + val: veryBigInt, + output: "89FFFFFFFFFFFFFFFFFF", + }, + { + val: veryVeryBigInt, + output: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", + }, // non-pointer big.Int {val: *big.NewInt(0), output: "80"}, From 836c647bdd65896c0e5bef97466d5d514db419e7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 25 May 2021 22:20:36 +0200 Subject: [PATCH 384/709] eth: unregister peer only when handler exits (#22908) This removes the error log message that says Ethereum peer removal failed ... err="peer not registered" The error happened because removePeer was called multiple times: once to disconnect the peer, and another time when the handler exited. With this change, removePeer now has the sole purpose of disconnecting the peer. Unregistering happens exactly once, when the handler exits. --- eth/handler.go | 15 +++++++---- eth/handler_eth_test.go | 59 ++++++++++++++++++++++++----------------- p2p/peer.go | 16 ++++++++++- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 3f10750abf..cd16538044 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -287,7 +287,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Error("Ethereum peer registration failed", "err", err) return err } - defer h.removePeer(peer.ID()) + defer h.unregisterPeer(peer.ID()) p := h.peers.peer(peer.ID()) if p == nil { @@ -354,9 +354,16 @@ func (h *handler) runSnapExtension(peer *snap.Peer, handler snap.Handler) error return handler(peer) } -// removePeer unregisters a peer from the downloader and fetchers, removes it from -// the set of tracked peers and closes the network connection to it. +// removePeer requests disconnection of a peer. func (h *handler) removePeer(id string) { + peer := h.peers.peer(id) + if peer != nil { + peer.Peer.Disconnect(p2p.DiscUselessPeer) + } +} + +// unregisterPeer removes a peer from the downloader, fetchers and main peer set. +func (h *handler) unregisterPeer(id string) { // Create a custom logger to avoid printing the entire id var logger log.Logger if len(id) < 16 { @@ -384,8 +391,6 @@ func (h *handler) removePeer(id string) { if err := h.peers.unregisterPeer(id); err != nil { logger.Error("Ethereum peer removal failed", "err", err) } - // Hard disconnect at the networking layer - peer.Peer.Disconnect(p2p.DiscUselessPeer) } func (h *handler) Start(maxPeers int) { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 1d38e3b666..038de46990 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -144,8 +144,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { defer p2pNoFork.Close() defer p2pProFork.Close() - peerNoFork := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) - peerProFork := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) + peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil) + peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil) defer peerNoFork.Close() defer peerProFork.Close() @@ -206,8 +206,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { defer p2pNoFork.Close() defer p2pProFork.Close() - peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) - peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) + peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil) + peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil) defer peerNoFork.Close() defer peerProFork.Close() @@ -257,8 +257,8 @@ func testRecvTransactions(t *testing.T, protocol uint) { defer p2pSrc.Close() defer p2pSink.Close() - src := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pSrc, handler.txpool) - sink := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pSink, handler.txpool) + src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool) + sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool) defer src.Close() defer sink.Close() @@ -319,8 +319,8 @@ func testSendTransactions(t *testing.T, protocol uint) { defer p2pSrc.Close() defer p2pSink.Close() - src := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pSrc, handler.txpool) - sink := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pSink, handler.txpool) + src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool) + sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool) defer src.Close() defer sink.Close() @@ -407,8 +407,8 @@ func testTransactionPropagation(t *testing.T, protocol uint) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{byte(i)}, "", nil), sourcePipe, source.txpool) - sinkPeer := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, sink.txpool) + sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, source.txpool) + sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool) defer sourcePeer.Close() defer sinkPeer.Close() @@ -490,6 +490,8 @@ func TestCheckpointChallenge(t *testing.T) { } func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpoint bool, timeout bool, empty bool, match bool, drop bool) { + t.Parallel() + // Reduce the checkpoint handshake challenge timeout defer func(old time.Duration) { syncChallengeTimeout = old }(syncChallengeTimeout) syncChallengeTimeout = 250 * time.Millisecond @@ -513,20 +515,26 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo handler.handler.checkpointNumber = number handler.handler.checkpointHash = response.Hash() } - // Create a challenger peer and a challenged one + + // Create a challenger peer and a challenged one. p2pLocal, p2pRemote := p2p.MsgPipe() defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{1}, "", nil), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{2}, "", nil), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() - go handler.handler.runEthPeer(local, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(handler.handler), peer) - }) - // Run the handshake locally to avoid spinning up a remote handler + handlerDone := make(chan struct{}) + go func() { + defer close(handlerDone) + handler.handler.runEthPeer(local, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(handler.handler), peer) + }) + }() + + // Run the handshake locally to avoid spinning up a remote handler. var ( genesis = handler.chain.Genesis() head = handler.chain.CurrentBlock() @@ -535,12 +543,13 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } - // Connect a new peer and check that we receive the checkpoint challenge + + // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { t.Fatalf("challenge mismatch: %v", err) } - // Create a block to reply to the challenge if no timeout is simulated + // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { @@ -557,11 +566,13 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo } } } + // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) - // Verify that the remote peer is maintained or dropped + // Verify that the remote peer is maintained or dropped. if drop { + <-handlerDone if peers := handler.handler.peers.len(); peers != 0 { t.Fatalf("peer count mismatch: have %d, want %d", peers, 0) } @@ -608,8 +619,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{byte(i)}, "", nil), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -676,8 +687,8 @@ func testBroadcastMalformedBlock(t *testing.T, protocol uint) { defer p2pSrc.Close() defer p2pSink.Close() - src := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pSrc, source.txpool) - sink := eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pSink, source.txpool) + src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, source.txpool) + sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, source.txpool) defer src.Close() defer sink.Close() diff --git a/p2p/peer.go b/p2p/peer.go index 8ebc858392..b6d0dbd1ae 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -115,7 +115,8 @@ type Peer struct { disc chan DiscReason // events receives message send / receive events if set - events *event.Feed + events *event.Feed + testPipe *MsgPipeRW // for testing } // NewPeer returns a peer for testing purposes. @@ -128,6 +129,15 @@ func NewPeer(id enode.ID, name string, caps []Cap) *Peer { return peer } +// NewPeerPipe creates a peer for testing purposes. +// The message pipe given as the last parameter is closed when +// Disconnect is called on the peer. +func NewPeerPipe(id enode.ID, name string, caps []Cap, pipe *MsgPipeRW) *Peer { + p := NewPeer(id, name, caps) + p.testPipe = pipe + return p +} + // ID returns the node's public key. func (p *Peer) ID() enode.ID { return p.rw.node.ID() @@ -185,6 +195,10 @@ func (p *Peer) LocalAddr() net.Addr { // Disconnect terminates the peer connection with the given reason. // It returns immediately and does not wait until the connection is closed. func (p *Peer) Disconnect(reason DiscReason) { + if p.testPipe != nil { + p.testPipe.Close() + } + select { case p.disc <- reason: case <-p.closed: From 51b32cc7e44e174316d3ae970680cfa9b77f6146 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 26 May 2021 04:30:21 +0800 Subject: [PATCH 385/709] internal/ethapi: merge CallArgs and SendTxArgs (#22718) There are two transaction parameter structures defined in the codebase, although for different purposes. But most of the parameters are shared. So it's nice to reduce the code duplication by merging them together. Co-authored-by: Martin Holst Swende --- eth/tracers/api.go | 2 +- eth/tracers/api_test.go | 20 +-- graphql/graphql.go | 8 +- internal/ethapi/api.go | 244 ++++------------------------ internal/ethapi/transaction_args.go | 185 +++++++++++++++++++++ 5 files changed, 236 insertions(+), 223 deletions(-) create mode 100644 internal/ethapi/transaction_args.go diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 172054e9b8..1e8759e69c 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -730,7 +730,7 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. // You can provide -2 as a block number to trace on top of the pending block. -func (api *API) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { +func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { // Try to retrieve the specified block var ( err error diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 24bce320cf..9ff01d66d5 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -198,7 +198,7 @@ func TestTraceCall(t *testing.T) { var testSuite = []struct { blockNumber rpc.BlockNumber - call ethapi.CallArgs + call ethapi.TransactionArgs config *TraceCallConfig expectErr error expect interface{} @@ -206,7 +206,7 @@ func TestTraceCall(t *testing.T) { // Standard JSON trace upon the genesis, plain transfer. { blockNumber: rpc.BlockNumber(0), - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &accounts[0].addr, To: &accounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -223,7 +223,7 @@ func TestTraceCall(t *testing.T) { // Standard JSON trace upon the head, plain transfer. { blockNumber: rpc.BlockNumber(genBlocks), - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &accounts[0].addr, To: &accounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -240,7 +240,7 @@ func TestTraceCall(t *testing.T) { // Standard JSON trace upon the non-existent block, error expects { blockNumber: rpc.BlockNumber(genBlocks + 1), - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &accounts[0].addr, To: &accounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -252,7 +252,7 @@ func TestTraceCall(t *testing.T) { // Standard JSON trace upon the latest block { blockNumber: rpc.LatestBlockNumber, - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &accounts[0].addr, To: &accounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -269,7 +269,7 @@ func TestTraceCall(t *testing.T) { // Standard JSON trace upon the pending block { blockNumber: rpc.PendingBlockNumber, - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &accounts[0].addr, To: &accounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -329,7 +329,7 @@ func TestOverridenTraceCall(t *testing.T) { var testSuite = []struct { blockNumber rpc.BlockNumber - call ethapi.CallArgs + call ethapi.TransactionArgs config *TraceCallConfig expectErr error expect *callTrace @@ -337,7 +337,7 @@ func TestOverridenTraceCall(t *testing.T) { // Succcessful call with state overriding { blockNumber: rpc.PendingBlockNumber, - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -361,7 +361,7 @@ func TestOverridenTraceCall(t *testing.T) { // Invalid call without state overriding { blockNumber: rpc.PendingBlockNumber, - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[1].addr, Value: (*hexutil.Big)(big.NewInt(1000)), @@ -390,7 +390,7 @@ func TestOverridenTraceCall(t *testing.T) { // } { blockNumber: rpc.PendingBlockNumber, - call: ethapi.CallArgs{ + call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[2].addr, Data: newRPCBytes(common.Hex2Bytes("8381f58a")), // call number() diff --git a/graphql/graphql.go b/graphql/graphql.go index b1af7675bd..71d80d8abd 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -862,7 +862,7 @@ func (c *CallResult) Status() Long { } func (b *Block) Call(ctx context.Context, args struct { - Data ethapi.CallArgs + Data ethapi.TransactionArgs }) (*CallResult, error) { if b.numberOrHash == nil { _, err := b.resolve(ctx) @@ -887,7 +887,7 @@ func (b *Block) Call(ctx context.Context, args struct { } func (b *Block) EstimateGas(ctx context.Context, args struct { - Data ethapi.CallArgs + Data ethapi.TransactionArgs }) (Long, error) { if b.numberOrHash == nil { _, err := b.resolveHeader(ctx) @@ -937,7 +937,7 @@ func (p *Pending) Account(ctx context.Context, args struct { } func (p *Pending) Call(ctx context.Context, args struct { - Data ethapi.CallArgs + Data ethapi.TransactionArgs }) (*CallResult, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, vm.Config{}, 5*time.Second, p.backend.RPCGasCap()) @@ -957,7 +957,7 @@ func (p *Pending) Call(ctx context.Context, args struct { } func (p *Pending) EstimateGas(ctx context.Context, args struct { - Data ethapi.CallArgs + Data ethapi.TransactionArgs }) (Long, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) gas, err := ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 7bc0477bd2..b06df8ff9f 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -17,7 +17,6 @@ package ethapi import ( - "bytes" "context" "errors" "fmt" @@ -351,9 +350,9 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { // signTransaction sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool -func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) { +func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *TransactionArgs, passwd string) (*types.Transaction, error) { // Look up the wallet containing the requested signer - account := accounts.Account{Address: args.From} + account := accounts.Account{Address: args.from()} wallet, err := s.am.Find(account) if err != nil { return nil, err @@ -369,18 +368,18 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArg } // SendTransaction will create a transaction from the given arguments and -// tries to sign it with the key associated with args.From. If the given passwd isn't -// able to decrypt the key it fails. -func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) { +// tries to sign it with the key associated with args.From. If the given +// passwd isn't able to decrypt the key it fails. +func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { if args.Nonce == nil { // Hold the addresse's mutex around signing to prevent concurrent assignment of // the same nonce to multiple accounts. - s.nonceLock.LockAddr(args.From) - defer s.nonceLock.UnlockAddr(args.From) + s.nonceLock.LockAddr(args.from()) + defer s.nonceLock.UnlockAddr(args.from()) } signed, err := s.signTransaction(ctx, &args, passwd) if err != nil { - log.Warn("Failed transaction send attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err) + log.Warn("Failed transaction send attempt", "from", args.from(), "to", args.To, "value", args.Value.ToInt(), "err", err) return common.Hash{}, err } return SubmitTransaction(ctx, s.b, signed) @@ -390,9 +389,12 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs // tries to sign it with the key associated with args.From. If the given passwd isn't // able to decrypt the key it fails. The transaction is returned in RLP-form, not broadcast // to other nodes -func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs, passwd string) (*SignTransactionResult, error) { +func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args TransactionArgs, passwd string) (*SignTransactionResult, error) { // No need to obtain the noncelock mutex, since we won't be sending this // tx into the transaction pool, but right back to the user + if args.From == nil { + return nil, fmt.Errorf("sender not specified") + } if args.Gas == nil { return nil, fmt.Errorf("gas not specified") } @@ -408,7 +410,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs } signed, err := s.signTransaction(ctx, &args, passwd) if err != nil { - log.Warn("Failed transaction sign attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err) + log.Warn("Failed transaction sign attempt", "from", args.from(), "to", args.To, "value", args.Value.ToInt(), "err", err) return nil, err } data, err := signed.MarshalBinary() @@ -473,7 +475,7 @@ func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Byt // SignAndSendTransaction was renamed to SendTransaction. This method is deprecated // and will be removed in the future. It primary goal is to give clients time to update. -func (s *PrivateAccountAPI) SignAndSendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) { +func (s *PrivateAccountAPI) SignAndSendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { return s.SendTransaction(ctx, args, passwd) } @@ -566,6 +568,7 @@ type AccountResult struct { StorageHash common.Hash `json:"storageHash"` StorageProof []StorageResult `json:"storageProof"` } + type StorageResult struct { Key string `json:"key"` Value *hexutil.Big `json:"value"` @@ -751,58 +754,6 @@ func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.A return res[:], state.Error() } -// CallArgs represents the arguments for a call. -type CallArgs struct { - From *common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Value *hexutil.Big `json:"value"` - Data *hexutil.Bytes `json:"data"` - AccessList *types.AccessList `json:"accessList"` -} - -// ToMessage converts CallArgs to the Message type used by the core evm -func (args *CallArgs) ToMessage(globalGasCap uint64) types.Message { - // Set sender address or use zero address if none specified. - var addr common.Address - if args.From != nil { - addr = *args.From - } - - // Set default gas & gas price if none were set - gas := globalGasCap - if gas == 0 { - gas = uint64(math.MaxUint64 / 2) - } - if args.Gas != nil { - gas = uint64(*args.Gas) - } - if globalGasCap != 0 && globalGasCap < gas { - log.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap) - gas = globalGasCap - } - gasPrice := new(big.Int) - if args.GasPrice != nil { - gasPrice = args.GasPrice.ToInt() - } - value := new(big.Int) - if args.Value != nil { - value = args.Value.ToInt() - } - var data []byte - if args.Data != nil { - data = *args.Data - } - var accessList types.AccessList - if args.AccessList != nil { - accessList = *args.AccessList - } - - msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, nil, nil, data, accessList, false) - return msg -} - // OverrideAccount indicates the overriding fields of account during the execution // of a message call. // Note, state and stateDiff can't be specified at the same time. If state is @@ -855,7 +806,7 @@ func (diff *StateOverride) Apply(state *state.StateDB) error { return nil } -func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, vmCfg vm.Config, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { +func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, vmCfg vm.Config, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) @@ -943,7 +894,7 @@ func (e *revertError) ErrorData() interface{} { // // Note, this function doesn't make and changes in the state/blockchain and is // useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { +func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, vm.Config{}, 5*time.Second, s.b.RPCGasCap()) if err != nil { return nil, err @@ -955,7 +906,7 @@ func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOr return result.Return(), result.Err } -func DoEstimateGas(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, gasCap uint64) (hexutil.Uint64, error) { +func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, gasCap uint64) (hexutil.Uint64, error) { // Binary search the gas requirement, as it may be higher than the amount used var ( lo uint64 = params.TxGas - 1 @@ -1066,7 +1017,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args CallArgs, blockNrOrHash // EstimateGas returns an estimate of the amount of gas needed to execute the // given transaction against the current pending block. -func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { +func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash @@ -1324,7 +1275,7 @@ type accessListResult struct { // CreateAccessList creates a EIP-2930 type AccessList for the given transaction. // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. -func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args SendTxArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { +func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash @@ -1343,7 +1294,7 @@ func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args SendTxA // AccessList creates an access list for the given transaction. // If the accesslist creation fails an error is returned. // If the transaction itself fails, an vmErr is returned. -func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args SendTxArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { +func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { // Retrieve the execution context db, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if db == nil || err != nil { @@ -1361,21 +1312,15 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if args.To != nil { to = *args.To } else { - to = crypto.CreateAddress(args.From, uint64(*args.Nonce)) - } - var input []byte - if args.Input != nil { - input = *args.Input - } else if args.Data != nil { - input = *args.Data + to = crypto.CreateAddress(args.from(), uint64(*args.Nonce)) } // Retrieve the precompiles since they don't need to be added to the access list precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number)) // Create an initial tracer - prevTracer := vm.NewAccessListTracer(nil, args.From, to, precompiles) + prevTracer := vm.NewAccessListTracer(nil, args.from(), to, precompiles) if args.AccessList != nil { - prevTracer = vm.NewAccessListTracer(*args.AccessList, args.From, to, precompiles) + prevTracer = vm.NewAccessListTracer(*args.AccessList, args.from(), to, precompiles) } for { // Retrieve the current access list to expand @@ -1393,10 +1338,10 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH } // Copy the original db so we don't modify it statedb := db.Copy() - msg := types.NewMessage(args.From, args.To, uint64(*args.Nonce), args.Value.ToInt(), uint64(*args.Gas), args.GasPrice.ToInt(), nil, nil, input, accessList, false) + msg := types.NewMessage(args.from(), args.To, uint64(*args.Nonce), args.Value.ToInt(), uint64(*args.Gas), args.GasPrice.ToInt(), nil, nil, args.data(), accessList, false) // Apply the transaction with the access list tracer - tracer := vm.NewAccessListTracer(accessList, args.From, to, precompiles) + tracer := vm.NewAccessListTracer(accessList, args.from(), to, precompiles) config := vm.Config{Tracer: tracer, Debug: true} vmenv, _, err := b.GetEVM(ctx, msg, statedb, header, &config) if err != nil { @@ -1597,123 +1542,6 @@ func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transacti return wallet.SignTx(account, tx, s.b.ChainConfig().ChainID) } -// SendTxArgs represents the arguments to sumbit a new transaction into the transaction pool. -type SendTxArgs struct { - From common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Value *hexutil.Big `json:"value"` - Nonce *hexutil.Uint64 `json:"nonce"` - // We accept "data" and "input" for backwards-compatibility reasons. "input" is the - // newer name and should be preferred by clients. - Data *hexutil.Bytes `json:"data"` - Input *hexutil.Bytes `json:"input"` - - // For non-legacy transactions - AccessList *types.AccessList `json:"accessList,omitempty"` - ChainID *hexutil.Big `json:"chainId,omitempty"` -} - -// setDefaults fills in default values for unspecified tx fields. -func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { - if args.GasPrice == nil { - price, err := b.SuggestPrice(ctx) - if err != nil { - return err - } - args.GasPrice = (*hexutil.Big)(price) - } - if args.Value == nil { - args.Value = new(hexutil.Big) - } - if args.Nonce == nil { - nonce, err := b.GetPoolNonce(ctx, args.From) - if err != nil { - return err - } - args.Nonce = (*hexutil.Uint64)(&nonce) - } - if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { - return errors.New(`both "data" and "input" are set and not equal. Please use "input" to pass transaction call data`) - } - if args.To == nil { - // Contract creation - var input []byte - if args.Data != nil { - input = *args.Data - } else if args.Input != nil { - input = *args.Input - } - if len(input) == 0 { - return errors.New(`contract creation without any data provided`) - } - } - // Estimate the gas usage if necessary. - if args.Gas == nil { - // For backwards-compatibility reason, we try both input and data - // but input is preferred. - input := args.Input - if input == nil { - input = args.Data - } - callArgs := CallArgs{ - From: &args.From, // From shouldn't be nil - To: args.To, - GasPrice: args.GasPrice, - Value: args.Value, - Data: input, - AccessList: args.AccessList, - } - pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - estimated, err := DoEstimateGas(ctx, b, callArgs, pendingBlockNr, b.RPCGasCap()) - if err != nil { - return err - } - args.Gas = &estimated - log.Trace("Estimate gas usage automatically", "gas", args.Gas) - } - if args.ChainID == nil { - id := (*hexutil.Big)(b.ChainConfig().ChainID) - args.ChainID = id - } - return nil -} - -// toTransaction converts the arguments to a transaction. -// This assumes that setDefaults has been called. -func (args *SendTxArgs) toTransaction() *types.Transaction { - var input []byte - if args.Input != nil { - input = *args.Input - } else if args.Data != nil { - input = *args.Data - } - var data types.TxData - if args.AccessList == nil { - data = &types.LegacyTx{ - To: args.To, - Nonce: uint64(*args.Nonce), - Gas: uint64(*args.Gas), - GasPrice: (*big.Int)(args.GasPrice), - Value: (*big.Int)(args.Value), - Data: input, - } - } else { - data = &types.AccessListTx{ - To: args.To, - ChainID: (*big.Int)(args.ChainID), - Nonce: uint64(*args.Nonce), - Gas: uint64(*args.Gas), - GasPrice: (*big.Int)(args.GasPrice), - Value: (*big.Int)(args.Value), - Data: input, - AccessList: *args.AccessList, - } - } - return types.NewTx(data) -} - // SubmitTransaction is a helper function that submits tx to txPool and logs a message. func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (common.Hash, error) { // If the transaction fee cap is already specified, ensure the @@ -1746,9 +1574,9 @@ func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (c // SendTransaction creates a transaction for the given argument, sign it and submit it to the // transaction pool. -func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args SendTxArgs) (common.Hash, error) { +func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args TransactionArgs) (common.Hash, error) { // Look up the wallet containing the requested signer - account := accounts.Account{Address: args.From} + account := accounts.Account{Address: args.from()} wallet, err := s.b.AccountManager().Find(account) if err != nil { @@ -1758,8 +1586,8 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen if args.Nonce == nil { // Hold the addresse's mutex around signing to prevent concurrent assignment of // the same nonce to multiple accounts. - s.nonceLock.LockAddr(args.From) - defer s.nonceLock.UnlockAddr(args.From) + s.nonceLock.LockAddr(args.from()) + defer s.nonceLock.UnlockAddr(args.from()) } // Set some sanity defaults and terminate on failure @@ -1778,7 +1606,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen // FillTransaction fills the defaults (nonce, gas, gasPrice) on a given unsigned transaction, // and returns it to the caller for further processing (signing + broadcast) -func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args SendTxArgs) (*SignTransactionResult, error) { +func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { // Set some sanity defaults and terminate on failure if err := args.setDefaults(ctx, s.b); err != nil { return nil, err @@ -1836,7 +1664,7 @@ type SignTransactionResult struct { // SignTransaction will sign the given transaction with the from account. // The node needs to have the private key of the account corresponding with // the given from address and it needs to be unlocked. -func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args SendTxArgs) (*SignTransactionResult, error) { +func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { if args.Gas == nil { return nil, fmt.Errorf("gas not specified") } @@ -1853,7 +1681,7 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Sen if err := checkTxFee(args.GasPrice.ToInt(), uint64(*args.Gas), s.b.RPCTxFeeCap()); err != nil { return nil, err } - tx, err := s.sign(args.From, args.toTransaction()) + tx, err := s.sign(args.from(), args.toTransaction()) if err != nil { return nil, err } @@ -1889,7 +1717,7 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err // Resend accepts an existing transaction and a new gas price and limit. It will remove // the given transaction from the pool and reinsert it with the new gas price and limit. -func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { +func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs TransactionArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { if sendArgs.Nonce == nil { return common.Hash{}, fmt.Errorf("missing transaction nonce in transaction spec") } @@ -1918,7 +1746,7 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr for _, p := range pending { wantSigHash := s.signer.Hash(matchTx) pFrom, err := types.Sender(s.signer, p) - if err == nil && pFrom == sendArgs.From && s.signer.Hash(p) == wantSigHash { + if err == nil && pFrom == sendArgs.from() && s.signer.Hash(p) == wantSigHash { // Match. Re-sign and send the transaction. if gasPrice != nil && (*big.Int)(gasPrice).Sign() != 0 { sendArgs.GasPrice = gasPrice @@ -1926,7 +1754,7 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr if gasLimit != nil && *gasLimit != 0 { sendArgs.Gas = gasLimit } - signedTx, err := s.sign(sendArgs.From, sendArgs.toTransaction()) + signedTx, err := s.sign(sendArgs.from(), sendArgs.toTransaction()) if err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go new file mode 100644 index 0000000000..4385a9e97a --- /dev/null +++ b/internal/ethapi/transaction_args.go @@ -0,0 +1,185 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "bytes" + "context" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +// TransactionArgs represents the arguments to construct a new transaction +// or a message call. +type TransactionArgs struct { + From *common.Address `json:"from"` + To *common.Address `json:"to"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + Value *hexutil.Big `json:"value"` + Nonce *hexutil.Uint64 `json:"nonce"` + + // We accept "data" and "input" for backwards-compatibility reasons. + // "input" is the newer name and should be preferred by clients. + // Issue detail: https://github.com/ethereum/go-ethereum/issues/15628 + Data *hexutil.Bytes `json:"data"` + Input *hexutil.Bytes `json:"input"` + + // For non-legacy transactions + AccessList *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` +} + +// from retrieves the transaction sender address. +func (arg *TransactionArgs) from() common.Address { + if arg.From == nil { + return common.Address{} + } + return *arg.From +} + +// data retrieves the transaction calldata. Input field is preferred. +func (arg *TransactionArgs) data() []byte { + if arg.Input != nil { + return *arg.Input + } + if arg.Data != nil { + return *arg.Data + } + return nil +} + +// setDefaults fills in default values for unspecified tx fields. +func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { + if args.GasPrice == nil { + price, err := b.SuggestPrice(ctx) + if err != nil { + return err + } + args.GasPrice = (*hexutil.Big)(price) + } + if args.Value == nil { + args.Value = new(hexutil.Big) + } + if args.Nonce == nil { + nonce, err := b.GetPoolNonce(ctx, args.from()) + if err != nil { + return err + } + args.Nonce = (*hexutil.Uint64)(&nonce) + } + if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { + return errors.New(`both "data" and "input" are set and not equal. Please use "input" to pass transaction call data`) + } + if args.To == nil && len(args.data()) == 0 { + return errors.New(`contract creation without any data provided`) + } + // Estimate the gas usage if necessary. + if args.Gas == nil { + // These fields are immutable during the estimation, safe to + // pass the pointer directly. + callArgs := TransactionArgs{ + From: args.From, + To: args.To, + GasPrice: args.GasPrice, + Value: args.Value, + Data: args.Data, + AccessList: args.AccessList, + } + pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) + estimated, err := DoEstimateGas(ctx, b, callArgs, pendingBlockNr, b.RPCGasCap()) + if err != nil { + return err + } + args.Gas = &estimated + log.Trace("Estimate gas usage automatically", "gas", args.Gas) + } + if args.ChainID == nil { + id := (*hexutil.Big)(b.ChainConfig().ChainID) + args.ChainID = id + } + return nil +} + +// ToMessage converts TransactionArgs to the Message type used by the core evm +func (args *TransactionArgs) ToMessage(globalGasCap uint64) types.Message { + // Set sender address or use zero address if none specified. + addr := args.from() + + // Set default gas & gas price if none were set + gas := globalGasCap + if gas == 0 { + gas = uint64(math.MaxUint64 / 2) + } + if args.Gas != nil { + gas = uint64(*args.Gas) + } + if globalGasCap != 0 && globalGasCap < gas { + log.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap) + gas = globalGasCap + } + gasPrice := new(big.Int) + if args.GasPrice != nil { + gasPrice = args.GasPrice.ToInt() + } + value := new(big.Int) + if args.Value != nil { + value = args.Value.ToInt() + } + data := args.data() + var accessList types.AccessList + if args.AccessList != nil { + accessList = *args.AccessList + } + msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, nil, nil, data, accessList, false) + return msg +} + +// toTransaction converts the arguments to a transaction. +// This assumes that setDefaults has been called. +func (args *TransactionArgs) toTransaction() *types.Transaction { + var data types.TxData + if args.AccessList == nil { + data = &types.LegacyTx{ + To: args.To, + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(args.Value), + Data: args.data(), + } + } else { + data = &types.AccessListTx{ + To: args.To, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(args.Value), + Data: args.data(), + AccessList: *args.AccessList, + } + } + return types.NewTx(data) +} From 750115ff3903a4d54267e0934a3391e4e2c2e84a Mon Sep 17 00:00:00 2001 From: meowsbits Date: Tue, 25 May 2021 15:37:30 -0500 Subject: [PATCH 386/709] p2p/nat: skip TestUPNP in non-CI environments if discover fails (#22877) Fixes #21476 --- p2p/nat/natupnp_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/p2p/nat/natupnp_test.go b/p2p/nat/natupnp_test.go index 79f6d25ae8..17483a7036 100644 --- a/p2p/nat/natupnp_test.go +++ b/p2p/nat/natupnp_test.go @@ -21,6 +21,7 @@ import ( "io" "net" "net/http" + "os" "runtime" "strings" "testing" @@ -162,7 +163,11 @@ func TestUPNP_DDWRT(t *testing.T) { // Attempt to discover the fake device. discovered := discoverUPnP() if discovered == nil { - t.Fatalf("not discovered") + if os.Getenv("CI") != "" { + t.Fatalf("not discovered") + } else { + t.Skipf("UPnP not discovered (known issue, see https://github.com/ethereum/go-ethereum/issues/21476)") + } } upnp, _ := discovered.(*upnp) if upnp.service != "IGDv1-IP1" { From 6c7d6cf886484805ae218af9b372c96e3ca8315c Mon Sep 17 00:00:00 2001 From: Eugene Lepeico Date: Tue, 25 May 2021 23:47:14 +0300 Subject: [PATCH 387/709] tests: get test name from testing.T (#22941) There were 2 TODOs about that fix after Golang 1.8 release. It's here for 3 years already, so now should be the right time. --- tests/block_test.go | 4 ++-- tests/difficulty_test.go | 4 ++-- tests/init_test.go | 10 ++++------ tests/rlp_test.go | 2 +- tests/state_test.go | 9 ++++----- tests/transaction_test.go | 2 +- tests/vm_test.go | 4 ++-- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index 4820ba733f..f7fbaea2a4 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -48,10 +48,10 @@ func TestBlockchain(t *testing.T) { // using 4.6 TGas bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if err := bt.checkFailure(t, name+"/trie", test.Run(false)); err != nil { + if err := bt.checkFailure(t, test.Run(false)); err != nil { t.Errorf("test without snapshotter failed: %v", err) } - if err := bt.checkFailure(t, name+"/snap", test.Run(true)); err != nil { + if err := bt.checkFailure(t, test.Run(true)); err != nil { t.Errorf("test with snapshotter failed: %v", err) } }) diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index e80cd248bc..acbf96e712 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -79,12 +79,12 @@ func TestDifficulty(t *testing.T) { dt.config("difficulty.json", mainnetChainConfig) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, test *DifficultyTest) { - cfg := dt.findConfig(name) + cfg := dt.findConfig(t) if test.ParentDifficulty.Cmp(params.MinimumDifficulty) < 0 { t.Skip("difficulty below minimum") return } - if err := dt.checkFailure(t, name, test.Run(cfg)); err != nil { + if err := dt.checkFailure(t, test.Run(cfg)); err != nil { t.Error(err) } }) diff --git a/tests/init_test.go b/tests/init_test.go index 5af3e44bff..dc923dc75e 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -167,10 +167,9 @@ func (tm *testMatcher) findSkip(name string) (reason string, skipload bool) { } // findConfig returns the chain config matching defined patterns. -func (tm *testMatcher) findConfig(name string) *params.ChainConfig { - // TODO(fjl): name can be derived from testing.T when min Go version is 1.8 +func (tm *testMatcher) findConfig(t *testing.T) *params.ChainConfig { for _, m := range tm.configpat { - if m.p.MatchString(name) { + if m.p.MatchString(t.Name()) { return &m.config } } @@ -178,11 +177,10 @@ func (tm *testMatcher) findConfig(name string) *params.ChainConfig { } // checkFailure checks whether a failure is expected. -func (tm *testMatcher) checkFailure(t *testing.T, name string, err error) error { - // TODO(fjl): name can be derived from t when min Go version is 1.8 +func (tm *testMatcher) checkFailure(t *testing.T, err error) error { failReason := "" for _, m := range tm.failpat { - if m.p.MatchString(name) { + if m.p.MatchString(t.Name()) { failReason = m.reason break } diff --git a/tests/rlp_test.go b/tests/rlp_test.go index 1601625df5..79a1683eb2 100644 --- a/tests/rlp_test.go +++ b/tests/rlp_test.go @@ -24,7 +24,7 @@ func TestRLP(t *testing.T) { t.Parallel() tm := new(testMatcher) tm.walk(t, rlpTestDir, func(t *testing.T, name string, test *RLPTest) { - if err := tm.checkFailure(t, name, test.Run()); err != nil { + if err := tm.checkFailure(t, test.Run()); err != nil { t.Error(err) } }) diff --git a/tests/state_test.go b/tests/state_test.go index b77a898c21..43009afdd5 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -64,12 +64,11 @@ func TestState(t *testing.T) { for _, subtest := range test.Subtests() { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) - name := name + "/" + key t.Run(key+"/trie", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { _, _, err := test.Run(subtest, vmconfig, false) - return st.checkFailure(t, name+"/trie", err) + return st.checkFailure(t, err) }) }) t.Run(key+"/snap", func(t *testing.T) { @@ -78,7 +77,7 @@ func TestState(t *testing.T) { if _, err := snaps.Journal(statedb.IntermediateRoot(false)); err != nil { return err } - return st.checkFailure(t, name+"/snap", err) + return st.checkFailure(t, err) }) }) } @@ -117,6 +116,6 @@ func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { } else { t.Log("EVM operation log:\n" + buf.String()) } - //t.Logf("EVM output: 0x%x", tracer.Output()) - //t.Logf("EVM error: %v", tracer.Error()) + // t.Logf("EVM output: 0x%x", tracer.Output()) + // t.Logf("EVM error: %v", tracer.Error()) } diff --git a/tests/transaction_test.go b/tests/transaction_test.go index 0e3670d04b..cb0f262318 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -47,7 +47,7 @@ func TestTransaction(t *testing.T) { txt.skipLoad("^ttValue/TransactionWithHighValueOverflow.json") txt.walk(t, transactionTestDir, func(t *testing.T, name string, test *TransactionTest) { cfg := params.MainnetChainConfig - if err := txt.checkFailure(t, name, test.Run(cfg)); err != nil { + if err := txt.checkFailure(t, test.Run(cfg)); err != nil { t.Error(err) } }) diff --git a/tests/vm_test.go b/tests/vm_test.go index fb839827ac..2150df9e23 100644 --- a/tests/vm_test.go +++ b/tests/vm_test.go @@ -30,10 +30,10 @@ func TestVM(t *testing.T) { vmt.walk(t, vmTestDir, func(t *testing.T, name string, test *VMTest) { withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, name+"/trie", test.Run(vmconfig, false)) + return vmt.checkFailure(t, test.Run(vmconfig, false)) }) withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, name+"/snap", test.Run(vmconfig, true)) + return vmt.checkFailure(t, test.Run(vmconfig, true)) }) }) } From 49bde05a55260469abcbeb64fd3c7b85c7536b5c Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 25 May 2021 23:09:11 +0200 Subject: [PATCH 388/709] cmd/devp2p: refactor eth test suite (#22843) This PR refactors the eth test suite to make it more readable and easier to use. Some notable differences: - A new file helpers.go stores all of the methods used between both eth66 and eth65 and below tests, as well as methods shared among many test functions. - suite.go now contains all of the test functions for both eth65 tests and eth66 tests. - The utesting.T object doesn't get passed through to other helper methods, but is instead only used within the scope of the test function, whereas helper methods return errors, so only the test function itself can fatal out in the case of an error. - The full test suite now only takes 13.5 seconds to run. --- cmd/devp2p/internal/ethtest/chain.go | 20 +- cmd/devp2p/internal/ethtest/eth66_suite.go | 521 ----------- .../internal/ethtest/eth66_suiteHelpers.go | 333 ------- cmd/devp2p/internal/ethtest/helpers.go | 635 +++++++++++++ cmd/devp2p/internal/ethtest/suite.go | 844 +++++++++++------- cmd/devp2p/internal/ethtest/transaction.go | 298 +++++-- cmd/devp2p/internal/ethtest/types.go | 238 ++--- 7 files changed, 1467 insertions(+), 1422 deletions(-) delete mode 100644 cmd/devp2p/internal/ethtest/eth66_suite.go delete mode 100644 cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go create mode 100644 cmd/devp2p/internal/ethtest/helpers.go diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index 83c55181ad..34a20c515b 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -54,10 +54,24 @@ func (c *Chain) Len() int { return len(c.blocks) } -// TD calculates the total difficulty of the chain. -func (c *Chain) TD(height int) *big.Int { // TODO later on channge scheme so that the height is included in range +// TD calculates the total difficulty of the chain at the +// chain head. +func (c *Chain) TD() *big.Int { sum := big.NewInt(0) - for _, block := range c.blocks[:height] { + for _, block := range c.blocks[:c.Len()] { + sum.Add(sum, block.Difficulty()) + } + return sum +} + +// TotalDifficultyAt calculates the total difficulty of the chain +// at the given block height. +func (c *Chain) TotalDifficultyAt(height int) *big.Int { + sum := big.NewInt(0) + if height >= c.Len() { + return sum + } + for _, block := range c.blocks[:height+1] { sum.Add(sum, block.Difficulty()) } return sum diff --git a/cmd/devp2p/internal/ethtest/eth66_suite.go b/cmd/devp2p/internal/ethtest/eth66_suite.go deleted file mode 100644 index 903a90c7eb..0000000000 --- a/cmd/devp2p/internal/ethtest/eth66_suite.go +++ /dev/null @@ -1,521 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethtest - -import ( - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/internal/utesting" - "github.com/ethereum/go-ethereum/p2p" -) - -// Is_66 checks if the node supports the eth66 protocol version, -// and if not, exists the test suite -func (s *Suite) Is_66(t *utesting.T) { - conn := s.dial66(t) - conn.handshake(t) - if conn.negotiatedProtoVersion < 66 { - t.Fail() - } -} - -// TestStatus_66 attempts to connect to the given node and exchange -// a status message with it on the eth66 protocol, and then check to -// make sure the chain head is correct. -func (s *Suite) TestStatus_66(t *utesting.T) { - conn := s.dial66(t) - defer conn.Close() - // get protoHandshake - conn.handshake(t) - // get status - switch msg := conn.statusExchange66(t, s.chain).(type) { - case *Status: - status := *msg - if status.ProtocolVersion != uint32(66) { - t.Fatalf("mismatch in version: wanted 66, got %d", status.ProtocolVersion) - } - t.Logf("got status message: %s", pretty.Sdump(msg)) - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } -} - -// TestGetBlockHeaders_66 tests whether the given node can respond to -// an eth66 `GetBlockHeaders` request and that the response is accurate. -func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) { - conn := s.setupConnection66(t) - defer conn.Close() - // get block headers - req := ð.GetBlockHeadersPacket66{ - RequestId: 3, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 2, - Skip: 1, - Reverse: false, - }, - } - // write message - headers, err := s.getBlockHeaders66(conn, req, req.RequestId) - if err != nil { - t.Fatalf("could not get block headers: %v", err) - } - // check for correct headers - if !headersMatch(t, s.chain, headers) { - t.Fatal("received wrong header(s)") - } -} - -// TestSimultaneousRequests_66 sends two simultaneous `GetBlockHeader` requests -// with different request IDs and checks to make sure the node responds with the correct -// headers per request. -func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) { - // create two connections - conn := s.setupConnection66(t) - defer conn.Close() - // create two requests - req1 := ð.GetBlockHeadersPacket66{ - RequestId: 111, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 2, - Skip: 1, - Reverse: false, - }, - } - req2 := ð.GetBlockHeadersPacket66{ - RequestId: 222, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 4, - Skip: 1, - Reverse: false, - }, - } - // write first request - if err := conn.write66(req1, GetBlockHeaders{}.Code()); err != nil { - t.Fatalf("failed to write to connection: %v", err) - } - // write second request - if err := conn.write66(req2, GetBlockHeaders{}.Code()); err != nil { - t.Fatalf("failed to write to connection: %v", err) - } - // wait for responses - headers1, err := s.waitForBlockHeadersResponse66(conn, req1.RequestId) - if err != nil { - t.Fatalf("error while waiting for block headers: %v", err) - } - headers2, err := s.waitForBlockHeadersResponse66(conn, req2.RequestId) - if err != nil { - t.Fatalf("error while waiting for block headers: %v", err) - } - // check headers of both responses - if !headersMatch(t, s.chain, headers1) { - t.Fatalf("wrong header(s) in response to req1: got %v", headers1) - } - if !headersMatch(t, s.chain, headers2) { - t.Fatalf("wrong header(s) in response to req2: got %v", headers2) - } -} - -// TestBroadcast_66 tests whether a block announcement is correctly -// propagated to the given node's peer(s) on the eth66 protocol. -func (s *Suite) TestBroadcast_66(t *utesting.T) { - s.sendNextBlock66(t) -} - -// TestGetBlockBodies_66 tests whether the given node can respond to -// a `GetBlockBodies` request and that the response is accurate over -// the eth66 protocol. -func (s *Suite) TestGetBlockBodies_66(t *utesting.T) { - conn := s.setupConnection66(t) - defer conn.Close() - // create block bodies request - id := uint64(55) - req := ð.GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ - s.chain.blocks[54].Hash(), - s.chain.blocks[75].Hash(), - }, - } - if err := conn.write66(req, GetBlockBodies{}.Code()); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - - reqID, msg := conn.readAndServe66(s.chain, timeout) - switch msg := msg.(type) { - case BlockBodies: - if reqID != req.RequestId { - t.Fatalf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID) - } - t.Logf("received %d block bodies", len(msg)) - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } -} - -// TestLargeAnnounce_66 tests the announcement mechanism with a large block. -func (s *Suite) TestLargeAnnounce_66(t *utesting.T) { - nextBlock := len(s.chain.blocks) - blocks := []*NewBlock{ - { - Block: largeBlock(), - TD: s.fullChain.TD(nextBlock + 1), - }, - { - Block: s.fullChain.blocks[nextBlock], - TD: largeNumber(2), - }, - { - Block: largeBlock(), - TD: largeNumber(2), - }, - { - Block: s.fullChain.blocks[nextBlock], - TD: s.fullChain.TD(nextBlock + 1), - }, - } - - for i, blockAnnouncement := range blocks[0:3] { - t.Logf("Testing malicious announcement: %v\n", i) - sendConn := s.setupConnection66(t) - if err := sendConn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // Invalid announcement, check that peer disconnected - switch msg := sendConn.ReadAndServe(s.chain, time.Second*8).(type) { - case *Disconnect: - case *Error: - break - default: - t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) - } - sendConn.Close() - } - // Test the last block as a valid block - s.sendNextBlock66(t) -} - -func (s *Suite) TestOldAnnounce_66(t *utesting.T) { - sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t) - defer sendConn.Close() - defer recvConn.Close() - - s.oldAnnounce(t, sendConn, recvConn) -} - -// TestMaliciousHandshake_66 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) { - conn := s.dial66(t) - defer conn.Close() - // write hello to client - pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] - handshakes := []*Hello{ - { - Version: 5, - Caps: []p2p.Cap{ - {Name: largeString(2), Version: 66}, - }, - ID: pub0, - }, - { - Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - {Name: "eth", Version: 66}, - }, - ID: append(pub0, byte(0)), - }, - { - Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - {Name: "eth", Version: 66}, - }, - ID: append(pub0, pub0...), - }, - { - Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - {Name: "eth", Version: 66}, - }, - ID: largeBuffer(2), - }, - { - Version: 5, - Caps: []p2p.Cap{ - {Name: largeString(2), Version: 66}, - }, - ID: largeBuffer(2), - }, - } - for i, handshake := range handshakes { - t.Logf("Testing malicious handshake %v\n", i) - // Init the handshake - if err := conn.Write(handshake); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // check that the peer disconnected - timeout := 20 * time.Second - // Discard one hello - for i := 0; i < 2; i++ { - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { - case *Disconnect: - case *Error: - case *Hello: - // Hello's are sent concurrently, so ignore them - continue - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } - } - // Dial for the next round - conn = s.dial66(t) - } -} - -// TestMaliciousStatus_66 sends a status package with a large total difficulty. -func (s *Suite) TestMaliciousStatus_66(t *utesting.T) { - conn := s.dial66(t) - defer conn.Close() - // get protoHandshake - conn.handshake(t) - status := &Status{ - ProtocolVersion: uint32(66), - NetworkID: s.chain.chainConfig.ChainID.Uint64(), - TD: largeNumber(2), - Head: s.chain.blocks[s.chain.Len()-1].Hash(), - Genesis: s.chain.blocks[0].Hash(), - ForkID: s.chain.ForkID(), - } - // get status - switch msg := conn.statusExchange(t, s.chain, status).(type) { - case *Status: - t.Logf("%+v\n", msg) - default: - t.Fatalf("expected status, got: %#v ", msg) - } - // wait for disconnect - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { - case *Disconnect: - case *Error: - return - default: - t.Fatalf("expected disconnect, got: %s", pretty.Sdump(msg)) - } -} - -func (s *Suite) TestTransaction_66(t *utesting.T) { - tests := []*types.Transaction{ - getNextTxFromChain(t, s), - unknownTx(t, s), - } - for i, tx := range tests { - t.Logf("Testing tx propagation: %v\n", i) - sendSuccessfulTx66(t, s, tx) - } -} - -func (s *Suite) TestMaliciousTx_66(t *utesting.T) { - badTxs := []*types.Transaction{ - getOldTxFromChain(t, s), - invalidNonceTx(t, s), - hugeAmount(t, s), - hugeGasPrice(t, s), - hugeData(t, s), - } - sendConn := s.setupConnection66(t) - defer sendConn.Close() - // set up receiving connection before sending txs to make sure - // no announcements are missed - recvConn := s.setupConnection66(t) - defer recvConn.Close() - - for i, tx := range badTxs { - t.Logf("Testing malicious tx propagation: %v\n", i) - if err := sendConn.Write(&Transactions{tx}); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - - } - // check to make sure bad txs aren't propagated - waitForTxPropagation(t, s, badTxs, recvConn) -} - -// TestZeroRequestID_66 checks that a request ID of zero is still handled -// by the node. -func (s *Suite) TestZeroRequestID_66(t *utesting.T) { - conn := s.setupConnection66(t) - defer conn.Close() - - req := ð.GetBlockHeadersPacket66{ - RequestId: 0, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Number: 0, - }, - Amount: 2, - }, - } - headers, err := s.getBlockHeaders66(conn, req, req.RequestId) - if err != nil { - t.Fatalf("could not get block headers: %v", err) - } - if !headersMatch(t, s.chain, headers) { - t.Fatal("received wrong header(s)") - } -} - -// TestSameRequestID_66 sends two requests with the same request ID -// concurrently to a single node. -func (s *Suite) TestSameRequestID_66(t *utesting.T) { - conn := s.setupConnection66(t) - // create two requests with the same request ID - reqID := uint64(1234) - request1 := ð.GetBlockHeadersPacket66{ - RequestId: reqID, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Number: 1, - }, - Amount: 2, - }, - } - request2 := ð.GetBlockHeadersPacket66{ - RequestId: reqID, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Number: 33, - }, - Amount: 2, - }, - } - // write the first request - err := conn.write66(request1, GetBlockHeaders{}.Code()) - if err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // perform second request - headers2, err := s.getBlockHeaders66(conn, request2, reqID) - if err != nil { - t.Fatalf("could not get block headers: %v", err) - return - } - // wait for response to first request - headers1, err := s.waitForBlockHeadersResponse66(conn, reqID) - if err != nil { - t.Fatalf("could not get BlockHeaders response: %v", err) - } - // check if headers match - if !headersMatch(t, s.chain, headers1) || !headersMatch(t, s.chain, headers2) { - t.Fatal("received wrong header(s)") - } -} - -// TestLargeTxRequest_66 tests whether a node can fulfill a large GetPooledTransactions -// request. -func (s *Suite) TestLargeTxRequest_66(t *utesting.T) { - // send the next block to ensure the node is no longer syncing and is able to accept - // txs - s.sendNextBlock66(t) - // send 2000 transactions to the node - hashMap, txs := generateTxs(t, s, 2000) - sendConn := s.setupConnection66(t) - defer sendConn.Close() - - sendMultipleSuccessfulTxs(t, s, sendConn, txs) - // set up connection to receive to ensure node is peered with the receiving connection - // before tx request is sent - recvConn := s.setupConnection66(t) - defer recvConn.Close() - // create and send pooled tx request - hashes := make([]common.Hash, 0) - for _, hash := range hashMap { - hashes = append(hashes, hash) - } - getTxReq := ð.GetPooledTransactionsPacket66{ - RequestId: 1234, - GetPooledTransactionsPacket: hashes, - } - if err := recvConn.write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { - t.Fatalf("could not write to conn: %v", err) - } - // check that all received transactions match those that were sent to node - switch msg := recvConn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { - case PooledTransactions: - for _, gotTx := range msg { - if _, exists := hashMap[gotTx.Hash()]; !exists { - t.Fatalf("unexpected tx received: %v", gotTx.Hash()) - } - } - default: - t.Fatalf("unexpected %s", pretty.Sdump(msg)) - } -} - -// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions -// request upon receiving a NewPooledTransactionHashes announcement. -func (s *Suite) TestNewPooledTxs_66(t *utesting.T) { - // send the next block to ensure the node is no longer syncing and is able to accept - // txs - s.sendNextBlock66(t) - // generate 50 txs - hashMap, _ := generateTxs(t, s, 50) - // create new pooled tx hashes announcement - hashes := make([]common.Hash, 0) - for _, hash := range hashMap { - hashes = append(hashes, hash) - } - announce := NewPooledTransactionHashes(hashes) - // send announcement - conn := s.setupConnection66(t) - defer conn.Close() - if err := conn.Write(announce); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // wait for GetPooledTxs request - for { - _, msg := conn.readAndServe66(s.chain, timeout) - switch msg := msg.(type) { - case GetPooledTransactions: - if len(msg) != len(hashes) { - t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) - } - return - case *NewPooledTransactionHashes, *NewBlock, *NewBlockHashes: - // ignore propagated txs and blocks from old tests - continue - default: - t.Fatalf("unexpected %s", pretty.Sdump(msg)) - } - } -} diff --git a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go b/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go deleted file mode 100644 index 3c5b22f0b5..0000000000 --- a/cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethtest - -import ( - "fmt" - "reflect" - "time" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/internal/utesting" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" - "github.com/stretchr/testify/assert" -) - -func (c *Conn) statusExchange66(t *utesting.T, chain *Chain) Message { - status := &Status{ - ProtocolVersion: uint32(66), - NetworkID: chain.chainConfig.ChainID.Uint64(), - TD: chain.TD(chain.Len()), - Head: chain.blocks[chain.Len()-1].Hash(), - Genesis: chain.blocks[0].Hash(), - ForkID: chain.ForkID(), - } - return c.statusExchange(t, chain, status) -} - -func (s *Suite) dial66(t *utesting.T) *Conn { - conn, err := s.dial() - if err != nil { - t.Fatalf("could not dial: %v", err) - } - conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) - conn.ourHighestProtoVersion = 66 - return conn -} - -func (c *Conn) write66(req eth.Packet, code int) error { - payload, err := rlp.EncodeToBytes(req) - if err != nil { - return err - } - _, err = c.Conn.Write(uint64(code), payload) - return err -} - -func (c *Conn) read66() (uint64, Message) { - code, rawData, _, err := c.Conn.Read() - if err != nil { - return 0, errorf("could not read from connection: %v", err) - } - - var msg Message - - switch int(code) { - case (Hello{}).Code(): - msg = new(Hello) - - case (Ping{}).Code(): - msg = new(Ping) - case (Pong{}).Code(): - msg = new(Pong) - case (Disconnect{}).Code(): - msg = new(Disconnect) - case (Status{}).Code(): - msg = new(Status) - case (GetBlockHeaders{}).Code(): - ethMsg := new(eth.GetBlockHeadersPacket66) - if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) - case (BlockHeaders{}).Code(): - ethMsg := new(eth.BlockHeadersPacket66) - if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) - case (GetBlockBodies{}).Code(): - ethMsg := new(eth.GetBlockBodiesPacket66) - if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) - case (BlockBodies{}).Code(): - ethMsg := new(eth.BlockBodiesPacket66) - if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) - case (NewBlock{}).Code(): - msg = new(NewBlock) - case (NewBlockHashes{}).Code(): - msg = new(NewBlockHashes) - case (Transactions{}).Code(): - msg = new(Transactions) - case (NewPooledTransactionHashes{}).Code(): - msg = new(NewPooledTransactionHashes) - case (GetPooledTransactions{}.Code()): - ethMsg := new(eth.GetPooledTransactionsPacket66) - if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) - case (PooledTransactions{}.Code()): - ethMsg := new(eth.PooledTransactionsPacket66) - if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) - default: - msg = errorf("invalid message code: %d", code) - } - - if msg != nil { - if err := rlp.DecodeBytes(rawData, msg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) - } - return 0, msg - } - return 0, errorf("invalid message: %s", string(rawData)) -} - -func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { - for { - id, msg := c.readAndServe66(chain, timeout) - if id == requestID { - return msg - } - } -} - -// ReadAndServe serves GetBlockHeaders requests while waiting -// on another message from the node. -func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { - start := time.Now() - for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(10 * time.Second)) - - reqID, msg := c.read66() - - switch msg := msg.(type) { - case *Ping: - c.Write(&Pong{}) - case *GetBlockHeaders: - headers, err := chain.GetHeaders(*msg) - if err != nil { - return 0, errorf("could not get headers for inbound header request: %v", err) - } - resp := ð.BlockHeadersPacket66{ - RequestId: reqID, - BlockHeadersPacket: eth.BlockHeadersPacket(headers), - } - if err := c.write66(resp, BlockHeaders{}.Code()); err != nil { - return 0, errorf("could not write to connection: %v", err) - } - default: - return reqID, msg - } - } - return 0, errorf("no message received within %v", timeout) -} - -func (s *Suite) setupConnection66(t *utesting.T) *Conn { - // create conn - sendConn := s.dial66(t) - sendConn.handshake(t) - sendConn.statusExchange66(t, s.chain) - return sendConn -} - -func (s *Suite) testAnnounce66(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) { - // Announce the block. - if err := sendConn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - s.waitAnnounce66(t, receiveConn, blockAnnouncement) -} - -func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) { - for { - _, msg := conn.readAndServe66(s.chain, timeout) - switch msg := msg.(type) { - case *NewBlock: - t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block)) - assert.Equal(t, - blockAnnouncement.Block.Header(), msg.Block.Header(), - "wrong block header in announcement", - ) - assert.Equal(t, - blockAnnouncement.TD, msg.TD, - "wrong TD in announcement", - ) - return - case *NewBlockHashes: - blockHashes := *msg - t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes)) - assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash, - "wrong block hash in announcement", - ) - return - case *NewPooledTransactionHashes: - // ignore old txs being propagated - continue - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } - } -} - -// waitForBlock66 waits for confirmation from the client that it has -// imported the given block. -func (c *Conn) waitForBlock66(block *types.Block) error { - defer c.SetReadDeadline(time.Time{}) - - c.SetReadDeadline(time.Now().Add(20 * time.Second)) - // note: if the node has not yet imported the block, it will respond - // to the GetBlockHeaders request with an empty BlockHeaders response, - // so the GetBlockHeaders request must be sent again until the BlockHeaders - // response contains the desired header. - for { - req := eth.GetBlockHeadersPacket66{ - RequestId: 54, - GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ - Origin: eth.HashOrNumber{ - Hash: block.Hash(), - }, - Amount: 1, - }, - } - if err := c.write66(req, GetBlockHeaders{}.Code()); err != nil { - return err - } - - reqID, msg := c.read66() - // check message - switch msg := msg.(type) { - case BlockHeaders: - // check request ID - if reqID != req.RequestId { - return fmt.Errorf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID) - } - for _, header := range msg { - if header.Number.Uint64() == block.NumberU64() { - return nil - } - } - time.Sleep(100 * time.Millisecond) - case *NewPooledTransactionHashes: - // ignore old announcements - continue - default: - return fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) - } - } -} - -func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) { - sendConn := s.setupConnection66(t) - defer sendConn.Close() - sendSuccessfulTxWithConn(t, s, tx, sendConn) -} - -// waitForBlockHeadersResponse66 waits for a BlockHeaders message with the given expected request ID -func (s *Suite) waitForBlockHeadersResponse66(conn *Conn, expectedID uint64) (BlockHeaders, error) { - reqID, msg := conn.readAndServe66(s.chain, timeout) - switch msg := msg.(type) { - case BlockHeaders: - if reqID != expectedID { - return nil, fmt.Errorf("request ID mismatch: wanted %d, got %d", expectedID, reqID) - } - return msg, nil - default: - return nil, fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) - } -} - -func (s *Suite) getBlockHeaders66(conn *Conn, req eth.Packet, expectedID uint64) (BlockHeaders, error) { - if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil { - return nil, fmt.Errorf("could not write to connection: %v", err) - } - return s.waitForBlockHeadersResponse66(conn, expectedID) -} - -func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) bool { - mismatched := 0 - for _, header := range headers { - num := header.Number.Uint64() - t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash())) - if !reflect.DeepEqual(chain.blocks[int(num)].Header(), header) { - mismatched += 1 - t.Logf("received wrong header: %v", pretty.Sdump(header)) - } - } - return mismatched == 0 -} - -func (s *Suite) sendNextBlock66(t *utesting.T) { - sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t) - defer sendConn.Close() - defer receiveConn.Close() - - // create new block announcement - nextBlock := len(s.chain.blocks) - blockAnnouncement := &NewBlock{ - Block: s.fullChain.blocks[nextBlock], - TD: s.fullChain.TD(nextBlock + 1), - } - // send announcement and wait for node to request the header - s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement) - // wait for client to update its chain - if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil { - t.Fatal(err) - } - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) -} diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go new file mode 100644 index 0000000000..d99376124d --- /dev/null +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -0,0 +1,635 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethtest + +import ( + "fmt" + "net" + "reflect" + "strings" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/rlpx" +) + +var ( + pretty = spew.ConfigState{ + Indent: " ", + DisableCapacities: true, + DisablePointerAddresses: true, + SortKeys: true, + } + timeout = 20 * time.Second +) + +// Is_66 checks if the node supports the eth66 protocol version, +// and if not, exists the test suite +func (s *Suite) Is_66(t *utesting.T) { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + if err := conn.handshake(); err != nil { + t.Fatalf("handshake failed: %v", err) + } + if conn.negotiatedProtoVersion < 66 { + t.Fail() + } +} + +// dial attempts to dial the given node and perform a handshake, +// returning the created Conn if successful. +func (s *Suite) dial() (*Conn, error) { + // dial + fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", s.Dest.IP(), s.Dest.TCP())) + if err != nil { + return nil, err + } + conn := Conn{Conn: rlpx.NewConn(fd, s.Dest.Pubkey())} + // do encHandshake + conn.ourKey, _ = crypto.GenerateKey() + _, err = conn.Handshake(conn.ourKey) + if err != nil { + conn.Close() + return nil, err + } + // set default p2p capabilities + conn.caps = []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + } + conn.ourHighestProtoVersion = 65 + return &conn, nil +} + +// dial66 attempts to dial the given node and perform a handshake, +// returning the created Conn with additional eth66 capabilities if +// successful +func (s *Suite) dial66() (*Conn, error) { + conn, err := s.dial() + if err != nil { + return nil, fmt.Errorf("dial failed: %v", err) + } + conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) + conn.ourHighestProtoVersion = 66 + return conn, nil +} + +// peer performs both the protocol handshake and the status message +// exchange with the node in order to peer with it. +func (c *Conn) peer(chain *Chain, status *Status) error { + if err := c.handshake(); err != nil { + return fmt.Errorf("handshake failed: %v", err) + } + if _, err := c.statusExchange(chain, status); err != nil { + return fmt.Errorf("status exchange failed: %v", err) + } + return nil +} + +// handshake performs a protocol handshake with the node. +func (c *Conn) handshake() error { + defer c.SetDeadline(time.Time{}) + c.SetDeadline(time.Now().Add(10 * time.Second)) + // write hello to client + pub0 := crypto.FromECDSAPub(&c.ourKey.PublicKey)[1:] + ourHandshake := &Hello{ + Version: 5, + Caps: c.caps, + ID: pub0, + } + if err := c.Write(ourHandshake); err != nil { + return fmt.Errorf("write to connection failed: %v", err) + } + // read hello from client + switch msg := c.Read().(type) { + case *Hello: + // set snappy if version is at least 5 + if msg.Version >= 5 { + c.SetSnappy(true) + } + c.negotiateEthProtocol(msg.Caps) + if c.negotiatedProtoVersion == 0 { + return fmt.Errorf("unexpected eth protocol version") + } + return nil + default: + return fmt.Errorf("bad handshake: %#v", msg) + } +} + +// negotiateEthProtocol sets the Conn's eth protocol version to highest +// advertised capability from peer. +func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) { + var highestEthVersion uint + for _, capability := range caps { + if capability.Name != "eth" { + continue + } + if capability.Version > highestEthVersion && capability.Version <= c.ourHighestProtoVersion { + highestEthVersion = capability.Version + } + } + c.negotiatedProtoVersion = highestEthVersion +} + +// statusExchange performs a `Status` message exchange with the given node. +func (c *Conn) statusExchange(chain *Chain, status *Status) (Message, error) { + defer c.SetDeadline(time.Time{}) + c.SetDeadline(time.Now().Add(20 * time.Second)) + + // read status message from client + var message Message +loop: + for { + switch msg := c.Read().(type) { + case *Status: + if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want { + return nil, fmt.Errorf("wrong head block in status, want: %#x (block %d) have %#x", + want, chain.blocks[chain.Len()-1].NumberU64(), have) + } + if have, want := msg.TD.Cmp(chain.TD()), 0; have != want { + return nil, fmt.Errorf("wrong TD in status: have %v want %v", have, want) + } + if have, want := msg.ForkID, chain.ForkID(); !reflect.DeepEqual(have, want) { + return nil, fmt.Errorf("wrong fork ID in status: have %v, want %v", have, want) + } + if have, want := msg.ProtocolVersion, c.ourHighestProtoVersion; have != uint32(want) { + return nil, fmt.Errorf("wrong protocol version: have %v, want %v", have, want) + } + message = msg + break loop + case *Disconnect: + return nil, fmt.Errorf("disconnect received: %v", msg.Reason) + case *Ping: + c.Write(&Pong{}) // TODO (renaynay): in the future, this should be an error + // (PINGs should not be a response upon fresh connection) + default: + return nil, fmt.Errorf("bad status message: %s", pretty.Sdump(msg)) + } + } + // make sure eth protocol version is set for negotiation + if c.negotiatedProtoVersion == 0 { + return nil, fmt.Errorf("eth protocol version must be set in Conn") + } + if status == nil { + // default status message + status = &Status{ + ProtocolVersion: uint32(c.negotiatedProtoVersion), + NetworkID: chain.chainConfig.ChainID.Uint64(), + TD: chain.TD(), + Head: chain.blocks[chain.Len()-1].Hash(), + Genesis: chain.blocks[0].Hash(), + ForkID: chain.ForkID(), + } + } + if err := c.Write(status); err != nil { + return nil, fmt.Errorf("write to connection failed: %v", err) + } + return message, nil +} + +// createSendAndRecvConns creates two connections, one for sending messages to the +// node, and one for receiving messages from the node. +func (s *Suite) createSendAndRecvConns(isEth66 bool) (*Conn, *Conn, error) { + var ( + sendConn *Conn + recvConn *Conn + err error + ) + if isEth66 { + sendConn, err = s.dial66() + if err != nil { + return nil, nil, fmt.Errorf("dial failed: %v", err) + } + recvConn, err = s.dial66() + if err != nil { + sendConn.Close() + return nil, nil, fmt.Errorf("dial failed: %v", err) + } + } else { + sendConn, err = s.dial() + if err != nil { + return nil, nil, fmt.Errorf("dial failed: %v", err) + } + recvConn, err = s.dial() + if err != nil { + sendConn.Close() + return nil, nil, fmt.Errorf("dial failed: %v", err) + } + } + return sendConn, recvConn, nil +} + +// readAndServe serves GetBlockHeaders requests while waiting +// on another message from the node. +func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { + start := time.Now() + for time.Since(start) < timeout { + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + switch msg := c.Read().(type) { + case *Ping: + c.Write(&Pong{}) + case *GetBlockHeaders: + req := *msg + headers, err := chain.GetHeaders(req) + if err != nil { + return errorf("could not get headers for inbound header request: %v", err) + } + if err := c.Write(headers); err != nil { + return errorf("could not write to connection: %v", err) + } + default: + return msg + } + } + return errorf("no message received within %v", timeout) +} + +// readAndServe66 serves eth66 GetBlockHeaders requests while waiting +// on another message from the node. +func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { + start := time.Now() + for time.Since(start) < timeout { + c.SetReadDeadline(time.Now().Add(10 * time.Second)) + + reqID, msg := c.Read66() + + switch msg := msg.(type) { + case *Ping: + c.Write(&Pong{}) + case *GetBlockHeaders: + headers, err := chain.GetHeaders(*msg) + if err != nil { + return 0, errorf("could not get headers for inbound header request: %v", err) + } + resp := ð.BlockHeadersPacket66{ + RequestId: reqID, + BlockHeadersPacket: eth.BlockHeadersPacket(headers), + } + if err := c.Write66(resp, BlockHeaders{}.Code()); err != nil { + return 0, errorf("could not write to connection: %v", err) + } + default: + return reqID, msg + } + } + return 0, errorf("no message received within %v", timeout) +} + +// headersRequest executes the given `GetBlockHeaders` request. +func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, isEth66 bool, reqID uint64) (BlockHeaders, error) { + defer c.SetReadDeadline(time.Time{}) + c.SetReadDeadline(time.Now().Add(20 * time.Second)) + // if on eth66 connection, perform eth66 GetBlockHeaders request + if isEth66 { + return getBlockHeaders66(chain, c, request, reqID) + } + if err := c.Write(request); err != nil { + return nil, err + } + switch msg := c.readAndServe(chain, timeout).(type) { + case *BlockHeaders: + return *msg, nil + default: + return nil, fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) + } +} + +// getBlockHeaders66 executes the given `GetBlockHeaders` request over the eth66 protocol. +func getBlockHeaders66(chain *Chain, conn *Conn, request *GetBlockHeaders, id uint64) (BlockHeaders, error) { + // write request + packet := eth.GetBlockHeadersPacket(*request) + req := ð.GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &packet, + } + if err := conn.Write66(req, GetBlockHeaders{}.Code()); err != nil { + return nil, fmt.Errorf("could not write to connection: %v", err) + } + // wait for response + msg := conn.waitForResponse(chain, timeout, req.RequestId) + headers, ok := msg.(BlockHeaders) + if !ok { + return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) + } + return headers, nil +} + +// headersMatch returns whether the received headers match the given request +func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { + return reflect.DeepEqual(expected, headers) +} + +// waitForResponse reads from the connection until a response with the expected +// request ID is received. +func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { + for { + id, msg := c.readAndServe66(chain, timeout) + if id == requestID { + return msg + } + } +} + +// sendNextBlock broadcasts the next block in the chain and waits +// for the node to propagate the block and import it into its chain. +func (s *Suite) sendNextBlock(isEth66 bool) error { + // set up sending and receiving connections + sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + if err != nil { + return err + } + defer sendConn.Close() + defer recvConn.Close() + if err = sendConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + if err = recvConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + // create new block announcement + nextBlock := s.fullChain.blocks[s.chain.Len()] + blockAnnouncement := &NewBlock{ + Block: nextBlock, + TD: s.fullChain.TotalDifficultyAt(s.chain.Len()), + } + // send announcement and wait for node to request the header + if err = s.testAnnounce(sendConn, recvConn, blockAnnouncement); err != nil { + return fmt.Errorf("failed to announce block: %v", err) + } + // wait for client to update its chain + if err = s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + return fmt.Errorf("failed to receive confirmation of block import: %v", err) + } + // update test suite chain + s.chain.blocks = append(s.chain.blocks, nextBlock) + return nil +} + +// testAnnounce writes a block announcement to the node and waits for the node +// to propagate it. +func (s *Suite) testAnnounce(sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) error { + if err := sendConn.Write(blockAnnouncement); err != nil { + return fmt.Errorf("could not write to connection: %v", err) + } + return s.waitAnnounce(receiveConn, blockAnnouncement) +} + +// waitAnnounce waits for a NewBlock or NewBlockHashes announcement from the node. +func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { + for { + switch msg := conn.readAndServe(s.chain, timeout).(type) { + case *NewBlock: + if !reflect.DeepEqual(blockAnnouncement.Block.Header(), msg.Block.Header()) { + return fmt.Errorf("wrong header in block announcement: \nexpected %v "+ + "\ngot %v", blockAnnouncement.Block.Header(), msg.Block.Header()) + } + if !reflect.DeepEqual(blockAnnouncement.TD, msg.TD) { + return fmt.Errorf("wrong TD in announcement: expected %v, got %v", blockAnnouncement.TD, msg.TD) + } + return nil + case *NewBlockHashes: + hashes := *msg + if blockAnnouncement.Block.Hash() != hashes[0].Hash { + return fmt.Errorf("wrong block hash in announcement: expected %v, got %v", blockAnnouncement.Block.Hash(), hashes[0].Hash) + } + return nil + case *NewPooledTransactionHashes: + // ignore tx announcements from previous tests + continue + default: + return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) + } + } +} + +func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) error { + defer conn.SetReadDeadline(time.Time{}) + conn.SetReadDeadline(time.Now().Add(20 * time.Second)) + // create request + req := &GetBlockHeaders{ + Origin: eth.HashOrNumber{ + Hash: block.Hash(), + }, + Amount: 1, + } + // loop until BlockHeaders response contains desired block, confirming the + // node imported the block + for { + var ( + headers BlockHeaders + err error + ) + if isEth66 { + requestID := uint64(54) + headers, err = conn.headersRequest(req, s.chain, eth66, requestID) + } else { + headers, err = conn.headersRequest(req, s.chain, eth65, 0) + } + if err != nil { + return fmt.Errorf("GetBlockHeader request failed: %v", err) + } + // if headers response is empty, node hasn't imported block yet, try again + if len(headers) == 0 { + time.Sleep(100 * time.Millisecond) + continue + } + if !reflect.DeepEqual(block.Header(), headers[0]) { + return fmt.Errorf("wrong header returned: wanted %v, got %v", block.Header(), headers[0]) + } + return nil + } +} + +func (s *Suite) oldAnnounce(isEth66 bool) error { + sendConn, receiveConn, err := s.createSendAndRecvConns(isEth66) + if err != nil { + return err + } + defer sendConn.Close() + defer receiveConn.Close() + if err := sendConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + if err := receiveConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + // create old block announcement + oldBlockAnnounce := &NewBlock{ + Block: s.chain.blocks[len(s.chain.blocks)/2], + TD: s.chain.blocks[len(s.chain.blocks)/2].Difficulty(), + } + if err := sendConn.Write(oldBlockAnnounce); err != nil { + return fmt.Errorf("could not write to connection: %v", err) + } + // wait to see if the announcement is propagated + switch msg := receiveConn.readAndServe(s.chain, time.Second*8).(type) { + case *NewBlock: + block := *msg + if block.Block.Hash() == oldBlockAnnounce.Block.Hash() { + return fmt.Errorf("unexpected: block propagated: %s", pretty.Sdump(msg)) + } + case *NewBlockHashes: + hashes := *msg + for _, hash := range hashes { + if hash.Hash == oldBlockAnnounce.Block.Hash() { + return fmt.Errorf("unexpected: block announced: %s", pretty.Sdump(msg)) + } + } + case *Error: + errMsg := *msg + // check to make sure error is timeout (propagation didn't come through == test successful) + if !strings.Contains(errMsg.String(), "timeout") { + return fmt.Errorf("unexpected error: %v", pretty.Sdump(msg)) + } + default: + return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) + } + return nil +} + +func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { + var ( + conn *Conn + err error + ) + if isEth66 { + conn, err = s.dial66() + if err != nil { + return fmt.Errorf("dial failed: %v", err) + } + } else { + conn, err = s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) + } + } + defer conn.Close() + // write hello to client + pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] + handshakes := []*Hello{ + { + Version: 5, + Caps: []p2p.Cap{ + {Name: largeString(2), Version: 64}, + }, + ID: pub0, + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + }, + ID: append(pub0, byte(0)), + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + }, + ID: append(pub0, pub0...), + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: "eth", Version: 64}, + {Name: "eth", Version: 65}, + }, + ID: largeBuffer(2), + }, + { + Version: 5, + Caps: []p2p.Cap{ + {Name: largeString(2), Version: 64}, + }, + ID: largeBuffer(2), + }, + } + for i, handshake := range handshakes { + t.Logf("Testing malicious handshake %v\n", i) + if err := conn.Write(handshake); err != nil { + return fmt.Errorf("could not write to connection: %v", err) + } + // check that the peer disconnected + for i := 0; i < 2; i++ { + switch msg := conn.readAndServe(s.chain, 20*time.Second).(type) { + case *Disconnect: + case *Error: + case *Hello: + // Discard one hello as Hello's are sent concurrently + continue + default: + return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) + } + } + // dial for the next round + if isEth66 { + conn, err = s.dial66() + if err != nil { + return fmt.Errorf("dial failed: %v", err) + } + } else { + conn, err = s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) + } + } + } + return nil +} + +func (s *Suite) maliciousStatus(conn *Conn) error { + if err := conn.handshake(); err != nil { + return fmt.Errorf("handshake failed: %v", err) + } + status := &Status{ + ProtocolVersion: uint32(conn.negotiatedProtoVersion), + NetworkID: s.chain.chainConfig.ChainID.Uint64(), + TD: largeNumber(2), + Head: s.chain.blocks[s.chain.Len()-1].Hash(), + Genesis: s.chain.blocks[0].Hash(), + ForkID: s.chain.ForkID(), + } + // get status + msg, err := conn.statusExchange(s.chain, status) + if err != nil { + return fmt.Errorf("status exchange failed: %v", err) + } + switch msg := msg.(type) { + case *Status: + default: + return fmt.Errorf("expected status, got: %#v ", msg) + } + // wait for disconnect + switch msg := conn.readAndServe(s.chain, timeout).(type) { + case *Disconnect: + return nil + case *Error: + return nil + default: + return fmt.Errorf("expected disconnect, got: %s", pretty.Sdump(msg)) + } +} diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index abc6bcddce..ad832dddd2 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -17,33 +17,16 @@ package ethtest import ( - "fmt" - "net" - "strings" "time" - "github.com/davecgh/go-spew/spew" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/internal/utesting" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/rlpx" - "github.com/stretchr/testify/assert" ) -var pretty = spew.ConfigState{ - Indent: " ", - DisableCapacities: true, - DisablePointerAddresses: true, - SortKeys: true, -} - -var timeout = 20 * time.Second - -// Suite represents a structure used to test the eth -// protocol of a node(s). +// Suite represents a structure used to test a node's conformance +// to the eth protocol. type Suite struct { Dest *enode.Node @@ -70,35 +53,35 @@ func (s *Suite) AllEthTests() []utesting.Test { return []utesting.Test{ // status {Name: "TestStatus", Fn: s.TestStatus}, - {Name: "TestStatus_66", Fn: s.TestStatus_66}, + {Name: "TestStatus66", Fn: s.TestStatus66}, // get block headers {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, - {Name: "TestGetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, - {Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66}, - {Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66}, - {Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66}, + {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, + {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, + {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, + {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, // get block bodies {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, - {Name: "TestGetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, + {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, // broadcast {Name: "TestBroadcast", Fn: s.TestBroadcast}, - {Name: "TestBroadcast_66", Fn: s.TestBroadcast_66}, + {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, - {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, + {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, - {Name: "TestOldAnnounce_66", Fn: s.TestOldAnnounce_66}, + {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, // malicious handshakes + status {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, - {Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66}, - {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66}, + {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, + {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, // test transactions {Name: "TestTransaction", Fn: s.TestTransaction}, - {Name: "TestTransaction_66", Fn: s.TestTransaction_66}, + {Name: "TestTransaction66", Fn: s.TestTransaction66}, {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, - {Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66}, - {Name: "TestLargeTxRequest_66", Fn: s.TestLargeTxRequest_66}, - {Name: "TestNewPooledTxs_66", Fn: s.TestNewPooledTxs_66}, + {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, + {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, + {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, } } @@ -109,6 +92,7 @@ func (s *Suite) EthTests() []utesting.Test { {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, {Name: "TestBroadcast", Fn: s.TestBroadcast}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, {Name: "TestTransaction", Fn: s.TestTransaction}, @@ -119,90 +103,101 @@ func (s *Suite) EthTests() []utesting.Test { func (s *Suite) Eth66Tests() []utesting.Test { return []utesting.Test{ // only proceed with eth66 test suite if node supports eth 66 protocol - {Name: "TestStatus_66", Fn: s.TestStatus_66}, - {Name: "TestGetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66}, - {Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66}, - {Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66}, - {Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66}, - {Name: "TestGetBlockBodies_66", Fn: s.TestGetBlockBodies_66}, - {Name: "TestBroadcast_66", Fn: s.TestBroadcast_66}, - {Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66}, - {Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66}, - {Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66}, - {Name: "TestTransaction_66", Fn: s.TestTransaction_66}, - {Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66}, - {Name: "TestLargeTxRequest_66", Fn: s.TestLargeTxRequest_66}, - {Name: "TestNewPooledTxs_66", Fn: s.TestNewPooledTxs_66}, + {Name: "TestStatus66", Fn: s.TestStatus66}, + {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, + {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, + {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, + {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, + {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, + {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, + {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, + {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, + {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, + {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, + {Name: "TestTransaction66", Fn: s.TestTransaction66}, + {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, + {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, + {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, } } +var ( + eth66 = true // indicates whether suite should negotiate eth66 connection + eth65 = false // indicates whether suite should negotiate eth65 connection or below. +) + // TestStatus attempts to connect to the given node and exchange -// a status message with it, and then check to make sure -// the chain head is correct. +// a status message with it. func (s *Suite) TestStatus(t *utesting.T) { conn, err := s.dial() if err != nil { - t.Fatalf("could not dial: %v", err) + t.Fatalf("dial failed: %v", err) } defer conn.Close() - // get protoHandshake - conn.handshake(t) - // get status - switch msg := conn.statusExchange(t, s.chain, nil).(type) { - case *Status: - t.Logf("got status message: %s", pretty.Sdump(msg)) - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) } } -// TestMaliciousStatus sends a status package with a large total difficulty. -func (s *Suite) TestMaliciousStatus(t *utesting.T) { - conn, err := s.dial() +// TestStatus66 attempts to connect to the given node and exchange +// a status message with it on the eth66 protocol. +func (s *Suite) TestStatus66(t *utesting.T) { + conn, err := s.dial66() if err != nil { - t.Fatalf("could not dial: %v", err) + t.Fatalf("dial failed: %v", err) } defer conn.Close() - // get protoHandshake - conn.handshake(t) - status := &Status{ - ProtocolVersion: uint32(conn.negotiatedProtoVersion), - NetworkID: s.chain.chainConfig.ChainID.Uint64(), - TD: largeNumber(2), - Head: s.chain.blocks[s.chain.Len()-1].Hash(), - Genesis: s.chain.blocks[0].Hash(), - ForkID: s.chain.ForkID(), - } - // get status - switch msg := conn.statusExchange(t, s.chain, status).(type) { - case *Status: - t.Logf("%+v\n", msg) - default: - t.Fatalf("expected status, got: %#v ", msg) - } - // wait for disconnect - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { - case *Disconnect: - case *Error: - return - default: - t.Fatalf("expected disconnect, got: %s", pretty.Sdump(msg)) + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) } } // TestGetBlockHeaders tests whether the given node can respond to -// a `GetBlockHeaders` request and that the response is accurate. +// a `GetBlockHeaders` request accurately. func (s *Suite) TestGetBlockHeaders(t *utesting.T) { conn, err := s.dial() if err != nil { - t.Fatalf("could not dial: %v", err) + t.Fatalf("dial failed: %v", err) } defer conn.Close() + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("handshake(s) failed: %v", err) + } + // write request + req := &GetBlockHeaders{ + Origin: eth.HashOrNumber{ + Hash: s.chain.blocks[1].Hash(), + }, + Amount: 2, + Skip: 1, + Reverse: false, + } + headers, err := conn.headersRequest(req, s.chain, eth65, 0) + if err != nil { + t.Fatalf("GetBlockHeaders request failed: %v", err) + } + // check for correct headers + expected, err := s.chain.GetHeaders(*req) + if err != nil { + t.Fatalf("failed to get headers for given request: %v", err) + } + if !headersMatch(expected, headers) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) + } +} - conn.handshake(t) - conn.statusExchange(t, s.chain, nil) - - // get block headers +// TestGetBlockHeaders66 tests whether the given node can respond to +// an eth66 `GetBlockHeaders` request and that the response is accurate. +func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + if err = conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + // write request req := &GetBlockHeaders{ Origin: eth.HashOrNumber{ Hash: s.chain.blocks[1].Hash(), @@ -211,21 +206,185 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { Skip: 1, Reverse: false, } + headers, err := conn.headersRequest(req, s.chain, eth66, 33) + if err != nil { + t.Fatalf("could not get block headers: %v", err) + } + // check for correct headers + expected, err := s.chain.GetHeaders(*req) + if err != nil { + t.Fatalf("failed to get headers for given request: %v", err) + } + if !headersMatch(expected, headers) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) + } +} - if err := conn.Write(req); err != nil { - t.Fatalf("could not write to connection: %v", err) +// TestSimultaneousRequests66 sends two simultaneous `GetBlockHeader` requests from +// the same connection with different request IDs and checks to make sure the node +// responds with the correct headers per request. +func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { + // create a connection + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + // create two requests + req1 := ð.GetBlockHeadersPacket66{ + RequestId: uint64(111), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Hash: s.chain.blocks[1].Hash(), + }, + Amount: 2, + Skip: 1, + Reverse: false, + }, + } + req2 := ð.GetBlockHeadersPacket66{ + RequestId: uint64(222), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Hash: s.chain.blocks[1].Hash(), + }, + Amount: 4, + Skip: 1, + Reverse: false, + }, + } + // write the first request + if err := conn.Write66(req1, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + // write the second request + if err := conn.Write66(req2, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + // wait for responses + msg := conn.waitForResponse(s.chain, timeout, req1.RequestId) + headers1, ok := msg.(BlockHeaders) + if !ok { + t.Fatalf("unexpected %s", pretty.Sdump(msg)) + } + msg = conn.waitForResponse(s.chain, timeout, req2.RequestId) + headers2, ok := msg.(BlockHeaders) + if !ok { + t.Fatalf("unexpected %s", pretty.Sdump(msg)) + } + // check received headers for accuracy + expected1, err := s.chain.GetHeaders(GetBlockHeaders(*req1.GetBlockHeadersPacket)) + if err != nil { + t.Fatalf("failed to get expected headers for request 1: %v", err) + } + expected2, err := s.chain.GetHeaders(GetBlockHeaders(*req2.GetBlockHeadersPacket)) + if err != nil { + t.Fatalf("failed to get expected headers for request 2: %v", err) + } + if !headersMatch(expected1, headers1) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } + if !headersMatch(expected2, headers2) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) + } +} - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { - case *BlockHeaders: - headers := *msg - for _, header := range headers { - num := header.Number.Uint64() - t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash())) - assert.Equal(t, s.chain.blocks[int(num)].Header(), header) - } - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) +// TestSameRequestID66 sends two requests with the same request ID to a +// single node. +func (s *Suite) TestSameRequestID66(t *utesting.T) { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + // create requests + reqID := uint64(1234) + request1 := ð.GetBlockHeadersPacket66{ + RequestId: reqID, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Number: 1, + }, + Amount: 2, + }, + } + request2 := ð.GetBlockHeadersPacket66{ + RequestId: reqID, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{ + Number: 33, + }, + Amount: 2, + }, + } + // write the requests + if err = conn.Write66(request1, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + if err = conn.Write66(request2, GetBlockHeaders{}.Code()); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + // wait for responses + msg := conn.waitForResponse(s.chain, timeout, reqID) + headers1, ok := msg.(BlockHeaders) + if !ok { + t.Fatalf("unexpected %s", pretty.Sdump(msg)) + } + msg = conn.waitForResponse(s.chain, timeout, reqID) + headers2, ok := msg.(BlockHeaders) + if !ok { + t.Fatalf("unexpected %s", pretty.Sdump(msg)) + } + // check if headers match + expected1, err := s.chain.GetHeaders(GetBlockHeaders(*request1.GetBlockHeadersPacket)) + if err != nil { + t.Fatalf("failed to get expected block headers: %v", err) + } + expected2, err := s.chain.GetHeaders(GetBlockHeaders(*request2.GetBlockHeadersPacket)) + if err != nil { + t.Fatalf("failed to get expected block headers: %v", err) + } + if !headersMatch(expected1, headers1) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) + } + if !headersMatch(expected2, headers2) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) + } +} + +// TestZeroRequestID_66 checks that a message with a request ID of zero is still handled +// by the node. +func (s *Suite) TestZeroRequestID66(t *utesting.T) { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + req := &GetBlockHeaders{ + Origin: eth.HashOrNumber{ + Number: 0, + }, + Amount: 2, + } + headers, err := conn.headersRequest(req, s.chain, eth66, 0) + if err != nil { + t.Fatalf("failed to get block headers: %v", err) + } + expected, err := s.chain.GetHeaders(*req) + if err != nil { + t.Fatalf("failed to get expected block headers: %v", err) + } + if !headersMatch(expected, headers) { + t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) } } @@ -234,12 +393,12 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { func (s *Suite) TestGetBlockBodies(t *utesting.T) { conn, err := s.dial() if err != nil { - t.Fatalf("could not dial: %v", err) + t.Fatalf("dial failed: %v", err) } defer conn.Close() - - conn.handshake(t) - conn.statusExchange(t, s.chain, nil) + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } // create block bodies request req := &GetBlockBodies{ s.chain.blocks[54].Hash(), @@ -248,126 +407,125 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { if err := conn.Write(req); err != nil { t.Fatalf("could not write to connection: %v", err) } - - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { + // wait for response + switch msg := conn.readAndServe(s.chain, timeout).(type) { case *BlockBodies: t.Logf("received %d block bodies", len(*msg)) + if len(*msg) != len(*req) { + t.Fatalf("wrong bodies in response: expected %d bodies, "+ + "got %d", len(*req), len(*msg)) + } default: t.Fatalf("unexpected: %s", pretty.Sdump(msg)) } } +// TestGetBlockBodies66 tests whether the given node can respond to +// a `GetBlockBodies` request and that the response is accurate over +// the eth66 protocol. +func (s *Suite) TestGetBlockBodies66(t *utesting.T) { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + // create block bodies request + req := ð.GetBlockBodiesPacket66{ + RequestId: uint64(55), + GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ + s.chain.blocks[54].Hash(), + s.chain.blocks[75].Hash(), + }, + } + if err := conn.Write66(req, GetBlockBodies{}.Code()); err != nil { + t.Fatalf("could not write to connection: %v", err) + } + // wait for block bodies response + msg := conn.waitForResponse(s.chain, timeout, req.RequestId) + blockBodies, ok := msg.(BlockBodies) + if !ok { + t.Fatalf("unexpected: %s", pretty.Sdump(msg)) + } + t.Logf("received %d block bodies", len(blockBodies)) + if len(blockBodies) != len(req.GetBlockBodiesPacket) { + t.Fatalf("wrong bodies in response: expected %d bodies, "+ + "got %d", len(req.GetBlockBodiesPacket), len(blockBodies)) + } +} + // TestBroadcast tests whether a block announcement is correctly // propagated to the given node's peer(s). func (s *Suite) TestBroadcast(t *utesting.T) { - s.sendNextBlock(t) + if err := s.sendNextBlock(eth65); err != nil { + t.Fatalf("block broadcast failed: %v", err) + } } -func (s *Suite) sendNextBlock(t *utesting.T) { - sendConn, receiveConn := s.setupConnection(t), s.setupConnection(t) - defer sendConn.Close() - defer receiveConn.Close() - - // create new block announcement - nextBlock := len(s.chain.blocks) - blockAnnouncement := &NewBlock{ - Block: s.fullChain.blocks[nextBlock], - TD: s.fullChain.TD(nextBlock + 1), - } - // send announcement and wait for node to request the header - s.testAnnounce(t, sendConn, receiveConn, blockAnnouncement) - // wait for client to update its chain - if err := receiveConn.waitForBlock(s.fullChain.blocks[nextBlock]); err != nil { - t.Fatal(err) +// TestBroadcast66 tests whether a block announcement is correctly +// propagated to the given node's peer(s) on the eth66 protocol. +func (s *Suite) TestBroadcast66(t *utesting.T) { + if err := s.sendNextBlock(eth66); err != nil { + t.Fatalf("block broadcast failed: %v", err) } - // update test suite chain - s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock]) } -// TestMaliciousHandshake tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake(t *utesting.T) { - conn, err := s.dial() - if err != nil { - t.Fatalf("could not dial: %v", err) - } - defer conn.Close() - // write hello to client - pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] - handshakes := []*Hello{ - { - Version: 5, - Caps: []p2p.Cap{ - {Name: largeString(2), Version: 64}, - }, - ID: pub0, - }, - { - Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - }, - ID: append(pub0, byte(0)), - }, +// TestLargeAnnounce tests the announcement mechanism with a large block. +func (s *Suite) TestLargeAnnounce(t *utesting.T) { + nextBlock := len(s.chain.blocks) + blocks := []*NewBlock{ { - Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - }, - ID: append(pub0, pub0...), + Block: largeBlock(), + TD: s.fullChain.TotalDifficultyAt(nextBlock), }, { - Version: 5, - Caps: []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, - }, - ID: largeBuffer(2), + Block: s.fullChain.blocks[nextBlock], + TD: largeNumber(2), }, { - Version: 5, - Caps: []p2p.Cap{ - {Name: largeString(2), Version: 64}, - }, - ID: largeBuffer(2), + Block: largeBlock(), + TD: largeNumber(2), }, } - for i, handshake := range handshakes { - t.Logf("Testing malicious handshake %v\n", i) - // Init the handshake - if err := conn.Write(handshake); err != nil { - t.Fatalf("could not write to connection: %v", err) + + for i, blockAnnouncement := range blocks { + t.Logf("Testing malicious announcement: %v\n", i) + conn, err := s.dial() + if err != nil { + t.Fatalf("dial failed: %v", err) } - // check that the peer disconnected - timeout := 20 * time.Second - // Discard one hello - for i := 0; i < 2; i++ { - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { - case *Disconnect: - case *Error: - case *Hello: - // Hello's are send concurrently, so ignore them - continue - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } + if err = conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) } - // Dial for the next round - conn, err = s.dial() - if err != nil { - t.Fatalf("could not dial: %v", err) + if err = conn.Write(blockAnnouncement); err != nil { + t.Fatalf("could not write to connection: %v", err) } + // Invalid announcement, check that peer disconnected + switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { + case *Disconnect: + case *Error: + break + default: + t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) + } + conn.Close() + } + // Test the last block as a valid block + if err := s.sendNextBlock(eth65); err != nil { + t.Fatalf("failed to broadcast next block: %v", err) } } -// TestLargeAnnounce tests the announcement mechanism with a large block. -func (s *Suite) TestLargeAnnounce(t *utesting.T) { +// TestLargeAnnounce66 tests the announcement mechanism with a large +// block over the eth66 protocol. +func (s *Suite) TestLargeAnnounce66(t *utesting.T) { nextBlock := len(s.chain.blocks) blocks := []*NewBlock{ { Block: largeBlock(), - TD: s.fullChain.TD(nextBlock + 1), + TD: s.fullChain.TotalDifficultyAt(nextBlock), }, { Block: s.fullChain.blocks[nextBlock], @@ -377,174 +535,220 @@ func (s *Suite) TestLargeAnnounce(t *utesting.T) { Block: largeBlock(), TD: largeNumber(2), }, - { - Block: s.fullChain.blocks[nextBlock], - TD: s.fullChain.TD(nextBlock + 1), - }, } for i, blockAnnouncement := range blocks[0:3] { t.Logf("Testing malicious announcement: %v\n", i) - sendConn := s.setupConnection(t) - if err := sendConn.Write(blockAnnouncement); err != nil { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + if err := conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + if err := conn.Write(blockAnnouncement); err != nil { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := sendConn.ReadAndServe(s.chain, time.Second*8).(type) { + switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { case *Disconnect: case *Error: break default: t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) } - sendConn.Close() + conn.Close() } // Test the last block as a valid block - s.sendNextBlock(t) + if err := s.sendNextBlock(eth66); err != nil { + t.Fatalf("failed to broadcast next block: %v", err) + } } +// TestOldAnnounce tests the announcement mechanism with an old block. func (s *Suite) TestOldAnnounce(t *utesting.T) { - sendConn, recvConn := s.setupConnection(t), s.setupConnection(t) - defer sendConn.Close() - defer recvConn.Close() + if err := s.oldAnnounce(eth65); err != nil { + t.Fatal(err) + } +} + +// TestOldAnnounce66 tests the announcement mechanism with an old block, +// over the eth66 protocol. +func (s *Suite) TestOldAnnounce66(t *utesting.T) { + if err := s.oldAnnounce(eth66); err != nil { + t.Fatal(err) + } +} + +// TestMaliciousHandshake tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake(t *utesting.T) { + if err := s.maliciousHandshakes(t, eth65); err != nil { + t.Fatal(err) + } +} - s.oldAnnounce(t, sendConn, recvConn) +// TestMaliciousHandshake66 tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake66(t *utesting.T) { + if err := s.maliciousHandshakes(t, eth66); err != nil { + t.Fatal(err) + } } -func (s *Suite) oldAnnounce(t *utesting.T, sendConn, receiveConn *Conn) { - oldBlockAnnounce := &NewBlock{ - Block: s.chain.blocks[len(s.chain.blocks)/2], - TD: s.chain.blocks[len(s.chain.blocks)/2].Difficulty(), +// TestMaliciousStatus sends a status package with a large total difficulty. +func (s *Suite) TestMaliciousStatus(t *utesting.T) { + conn, err := s.dial() + if err != nil { + t.Fatalf("dial failed: %v", err) } + defer conn.Close() - if err := sendConn.Write(oldBlockAnnounce); err != nil { - t.Fatalf("could not write to connection: %v", err) + if err := s.maliciousStatus(conn); err != nil { + t.Fatal(err) } +} - switch msg := receiveConn.ReadAndServe(s.chain, time.Second*8).(type) { - case *NewBlock: - block := *msg - if block.Block.Hash() == oldBlockAnnounce.Block.Hash() { - t.Fatalf("unexpected: block propagated: %s", pretty.Sdump(msg)) - } - case *NewBlockHashes: - hashes := *msg - for _, hash := range hashes { - if hash.Hash == oldBlockAnnounce.Block.Hash() { - t.Fatalf("unexpected: block announced: %s", pretty.Sdump(msg)) - } - } - case *Error: - errMsg := *msg - // check to make sure error is timeout (propagation didn't come through == test successful) - if !strings.Contains(errMsg.String(), "timeout") { - t.Fatalf("unexpected error: %v", pretty.Sdump(msg)) - } - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) +// TestMaliciousStatus66 sends a status package with a large total +// difficulty over the eth66 protocol. +func (s *Suite) TestMaliciousStatus66(t *utesting.T) { + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + + if err := s.maliciousStatus(conn); err != nil { + t.Fatal(err) } } -func (s *Suite) testAnnounce(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) { - // Announce the block. - if err := sendConn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) +// TestTransaction sends a valid transaction to the node and +// checks if the transaction gets propagated. +func (s *Suite) TestTransaction(t *utesting.T) { + if err := s.sendSuccessfulTxs(t, eth65); err != nil { + t.Fatal(err) } - s.waitAnnounce(t, receiveConn, blockAnnouncement) } -func (s *Suite) waitAnnounce(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) { - switch msg := conn.ReadAndServe(s.chain, timeout).(type) { - case *NewBlock: - t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block)) - assert.Equal(t, - blockAnnouncement.Block.Header(), msg.Block.Header(), - "wrong block header in announcement", - ) - assert.Equal(t, - blockAnnouncement.TD, msg.TD, - "wrong TD in announcement", - ) - case *NewBlockHashes: - message := *msg - t.Logf("received NewBlockHashes message: %s", pretty.Sdump(message)) - assert.Equal(t, blockAnnouncement.Block.Hash(), message[0].Hash, - "wrong block hash in announcement", - ) - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) +// TestTransaction66 sends a valid transaction to the node and +// checks if the transaction gets propagated. +func (s *Suite) TestTransaction66(t *utesting.T) { + if err := s.sendSuccessfulTxs(t, eth66); err != nil { + t.Fatal(err) } } -func (s *Suite) setupConnection(t *utesting.T) *Conn { - // create conn - sendConn, err := s.dial() - if err != nil { - t.Fatalf("could not dial: %v", err) +// TestMaliciousTx sends several invalid transactions and tests whether +// the node will propagate them. +func (s *Suite) TestMaliciousTx(t *utesting.T) { + if err := s.sendMaliciousTxs(t, eth65); err != nil { + t.Fatal(err) + } +} + +// TestMaliciousTx66 sends several invalid transactions and tests whether +// the node will propagate them. +func (s *Suite) TestMaliciousTx66(t *utesting.T) { + if err := s.sendMaliciousTxs(t, eth66); err != nil { + t.Fatal(err) } - sendConn.handshake(t) - sendConn.statusExchange(t, s.chain, nil) - return sendConn } -// dial attempts to dial the given node and perform a handshake, -// returning the created Conn if successful. -func (s *Suite) dial() (*Conn, error) { - var conn Conn - // dial - fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", s.Dest.IP(), s.Dest.TCP())) +// TestLargeTxRequest66 tests whether a node can fulfill a large GetPooledTransactions +// request. +func (s *Suite) TestLargeTxRequest66(t *utesting.T) { + // send the next block to ensure the node is no longer syncing and + // is able to accept txs + if err := s.sendNextBlock(eth66); err != nil { + t.Fatalf("failed to send next block: %v", err) + } + // send 2000 transactions to the node + hashMap, txs, err := generateTxs(s, 2000) if err != nil { - return nil, err + t.Fatalf("failed to generate transactions: %v", err) } - conn.Conn = rlpx.NewConn(fd, s.Dest.Pubkey()) - // do encHandshake - conn.ourKey, _ = crypto.GenerateKey() - _, err = conn.Handshake(conn.ourKey) + if err = sendMultipleSuccessfulTxs(t, s, txs); err != nil { + t.Fatalf("failed to send multiple txs: %v", err) + } + // set up connection to receive to ensure node is peered with the receiving connection + // before tx request is sent + conn, err := s.dial66() if err != nil { - return nil, err + t.Fatalf("dial failed: %v", err) } - // set default p2p capabilities - conn.caps = []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, + defer conn.Close() + if err = conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + // create and send pooled tx request + hashes := make([]common.Hash, 0) + for _, hash := range hashMap { + hashes = append(hashes, hash) + } + getTxReq := ð.GetPooledTransactionsPacket66{ + RequestId: 1234, + GetPooledTransactionsPacket: hashes, + } + if err = conn.Write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { + t.Fatalf("could not write to conn: %v", err) + } + // check that all received transactions match those that were sent to node + switch msg := conn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { + case PooledTransactions: + for _, gotTx := range msg { + if _, exists := hashMap[gotTx.Hash()]; !exists { + t.Fatalf("unexpected tx received: %v", gotTx.Hash()) + } + } + default: + t.Fatalf("unexpected %s", pretty.Sdump(msg)) } - conn.ourHighestProtoVersion = 65 - return &conn, nil } -func (s *Suite) TestTransaction(t *utesting.T) { - tests := []*types.Transaction{ - getNextTxFromChain(t, s), - unknownTx(t, s), +// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions +// request upon receiving a NewPooledTransactionHashes announcement. +func (s *Suite) TestNewPooledTxs66(t *utesting.T) { + // send the next block to ensure the node is no longer syncing and + // is able to accept txs + if err := s.sendNextBlock(eth66); err != nil { + t.Fatalf("failed to send next block: %v", err) } - for i, tx := range tests { - t.Logf("Testing tx propagation: %v\n", i) - sendSuccessfulTx(t, s, tx) + // generate 50 txs + hashMap, _, err := generateTxs(s, 50) + if err != nil { + t.Fatalf("failed to generate transactions: %v", err) } -} - -func (s *Suite) TestMaliciousTx(t *utesting.T) { - badTxs := []*types.Transaction{ - getOldTxFromChain(t, s), - invalidNonceTx(t, s), - hugeAmount(t, s), - hugeGasPrice(t, s), - hugeData(t, s), - } - sendConn := s.setupConnection(t) - defer sendConn.Close() - // set up receiving connection before sending txs to make sure - // no announcements are missed - recvConn := s.setupConnection(t) - defer recvConn.Close() - - for i, tx := range badTxs { - t.Logf("Testing malicious tx propagation: %v\n", i) - if err := sendConn.Write(&Transactions{tx}); err != nil { - t.Fatalf("could not write to connection: %v", err) + // create new pooled tx hashes announcement + hashes := make([]common.Hash, 0) + for _, hash := range hashMap { + hashes = append(hashes, hash) + } + announce := NewPooledTransactionHashes(hashes) + // send announcement + conn, err := s.dial66() + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + if err = conn.peer(s.chain, nil); err != nil { + t.Fatalf("peering failed: %v", err) + } + if err = conn.Write(announce); err != nil { + t.Fatalf("failed to write to connection: %v", err) + } + // wait for GetPooledTxs request + for { + _, msg := conn.readAndServe66(s.chain, timeout) + switch msg := msg.(type) { + case GetPooledTransactions: + if len(msg) != len(hashes) { + t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) + } + return + case *NewPooledTransactionHashes: + // ignore propagated txs from old tests + continue + default: + t.Fatalf("unexpected %s", pretty.Sdump(msg)) } - } - // check to make sure bad txs aren't propagated - waitForTxPropagation(t, s, badTxs, recvConn) } diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index a6166bd2e3..d2dbe0a7d6 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -17,6 +17,7 @@ package ethtest import ( + "fmt" "math/big" "strings" "time" @@ -31,58 +32,171 @@ import ( //var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) { - sendConn := s.setupConnection(t) - defer sendConn.Close() - sendSuccessfulTxWithConn(t, s, tx, sendConn) +func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { + tests := []*types.Transaction{ + getNextTxFromChain(s), + unknownTx(s), + } + for i, tx := range tests { + if tx == nil { + return fmt.Errorf("could not find tx to send") + } + t.Logf("Testing tx propagation %d: sending tx %v %v %v\n", i, tx.Hash().String(), tx.GasPrice(), tx.Gas()) + // get previous tx if exists for reference in case of old tx propagation + var prevTx *types.Transaction + if i != 0 { + prevTx = tests[i-1] + } + // write tx to connection + if err := sendSuccessfulTx(s, tx, prevTx, isEth66); err != nil { + return fmt.Errorf("send successful tx test failed: %v", err) + } + } + return nil } -func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, sendConn *Conn) { - t.Logf("sending tx: %v %v %v\n", tx.Hash().String(), tx.GasPrice(), tx.Gas()) +func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction, isEth66 bool) error { + sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + if err != nil { + return err + } + defer sendConn.Close() + defer recvConn.Close() + if err = sendConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } // Send the transaction - if err := sendConn.Write(&Transactions{tx}); err != nil { - t.Fatal(err) + if err = sendConn.Write(&Transactions{tx}); err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + // peer receiving connection to node + if err = recvConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) } // update last nonce seen nonce = tx.Nonce() - - recvConn := s.setupConnection(t) // Wait for the transaction announcement - switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { - case *Transactions: - recTxs := *msg - for _, gotTx := range recTxs { - if gotTx.Hash() == tx.Hash() { - // Ok - return + for { + switch msg := recvConn.readAndServe(s.chain, timeout).(type) { + case *Transactions: + recTxs := *msg + // if you receive an old tx propagation, read from connection again + if len(recTxs) == 1 && prevTx != nil { + if recTxs[0] == prevTx { + continue + } } - } - t.Fatalf("missing transaction: got %v missing %v", recTxs, tx.Hash()) - case *NewPooledTransactionHashes: - txHashes := *msg - for _, gotHash := range txHashes { - if gotHash == tx.Hash() { - return + for _, gotTx := range recTxs { + if gotTx.Hash() == tx.Hash() { + // Ok + return nil + } } + return fmt.Errorf("missing transaction: got %v missing %v", recTxs, tx.Hash()) + case *NewPooledTransactionHashes: + txHashes := *msg + // if you receive an old tx propagation, read from connection again + if len(txHashes) == 1 && prevTx != nil { + if txHashes[0] == prevTx.Hash() { + continue + } + } + for _, gotHash := range txHashes { + if gotHash == tx.Hash() { + // Ok + return nil + } + } + return fmt.Errorf("missing transaction announcement: got %v missing %v", txHashes, tx.Hash()) + default: + return fmt.Errorf("unexpected message in sendSuccessfulTx: %s", pretty.Sdump(msg)) } - t.Fatalf("missing transaction announcement: got %v missing %v", txHashes, tx.Hash()) - default: - t.Fatalf("unexpected message in sendSuccessfulTx: %s", pretty.Sdump(msg)) } } +func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { + badTxs := []*types.Transaction{ + getOldTxFromChain(s), + invalidNonceTx(s), + hugeAmount(s), + hugeGasPrice(s), + hugeData(s), + } + // setup receiving connection before sending malicious txs + var ( + recvConn *Conn + err error + ) + if isEth66 { + recvConn, err = s.dial66() + } else { + recvConn, err = s.dial() + } + if err != nil { + return fmt.Errorf("dial failed: %v", err) + } + defer recvConn.Close() + if err = recvConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + for i, tx := range badTxs { + t.Logf("Testing malicious tx propagation: %v\n", i) + if err = sendMaliciousTx(s, tx, isEth66); err != nil { + return fmt.Errorf("malicious tx test failed:\ntx: %v\nerror: %v", tx, err) + } + } + // check to make sure bad txs aren't propagated + return checkMaliciousTxPropagation(s, badTxs, recvConn) +} + +func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { + // setup connection + var ( + conn *Conn + err error + ) + if isEth66 { + conn, err = s.dial66() + } else { + conn, err = s.dial() + } + if err != nil { + return fmt.Errorf("dial failed: %v", err) + } + defer conn.Close() + if err = conn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + // write malicious tx + if err = conn.Write(&Transactions{tx}); err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + return nil +} + var nonce = uint64(99) -func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*types.Transaction) { +// sendMultipleSuccessfulTxs sends the given transactions to the node and +// expects the node to accept and propagate them. +func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction) error { txMsg := Transactions(txs) t.Logf("sending %d txs\n", len(txs)) - recvConn := s.setupConnection(t) + sendConn, recvConn, err := s.createSendAndRecvConns(true) + if err != nil { + return err + } + defer sendConn.Close() defer recvConn.Close() - + if err = sendConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + if err = recvConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } // Send the transactions - if err := sendConn.Write(&txMsg); err != nil { - t.Fatal(err) + if err = sendConn.Write(&txMsg); err != nil { + return fmt.Errorf("failed to write message to connection: %v", err) } // update nonce nonce = txs[len(txs)-1].Nonce() @@ -90,7 +204,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*t recvHashes := make([]common.Hash, 0) // all txs should be announced within 3 announcements for i := 0; i < 3; i++ { - switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) { + switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: for _, tx := range *msg { recvHashes = append(recvHashes, tx.Hash()) @@ -99,7 +213,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*t recvHashes = append(recvHashes, *msg...) default: if !strings.Contains(pretty.Sdump(msg), "i/o timeout") { - t.Fatalf("unexpected message while waiting to receive txs: %s", pretty.Sdump(msg)) + return fmt.Errorf("unexpected message while waiting to receive txs: %s", pretty.Sdump(msg)) } } // break once all 2000 txs have been received @@ -112,7 +226,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*t continue } else { t.Logf("successfully received all %d txs", len(txs)) - return + return nil } } } @@ -121,13 +235,15 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*t for _, missing := range missingTxs { t.Logf("missing tx: %v", missing.Hash()) } - t.Fatalf("missing %d txs", len(missingTxs)) + return fmt.Errorf("missing %d txs", len(missingTxs)) } + return nil } -func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, recvConn *Conn) { - // Wait for another transaction announcement - switch msg := recvConn.ReadAndServe(s.chain, time.Second*8).(type) { +// checkMaliciousTxPropagation checks whether the given malicious transactions were +// propagated by the node. +func checkMaliciousTxPropagation(s *Suite, txs []*types.Transaction, conn *Conn) error { + switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { case *Transactions: // check to see if any of the failing txs were in the announcement recvTxs := make([]common.Hash, len(*msg)) @@ -136,25 +252,20 @@ func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, rec } badTxs, _ := compareReceivedTxs(recvTxs, txs) if len(badTxs) > 0 { - for _, tx := range badTxs { - t.Logf("received bad tx: %v", tx) - } - t.Fatalf("received %d bad txs", len(badTxs)) + return fmt.Errorf("received %d bad txs: \n%v", len(badTxs), badTxs) } case *NewPooledTransactionHashes: badTxs, _ := compareReceivedTxs(*msg, txs) if len(badTxs) > 0 { - for _, tx := range badTxs { - t.Logf("received bad tx: %v", tx) - } - t.Fatalf("received %d bad txs", len(badTxs)) + return fmt.Errorf("received %d bad txs: \n%v", len(badTxs), badTxs) } case *Error: // Transaction should not be announced -> wait for timeout - return + return nil default: - t.Fatalf("unexpected message in sendFailingTx: %s", pretty.Sdump(msg)) + return fmt.Errorf("unexpected message in sendFailingTx: %s", pretty.Sdump(msg)) } + return nil } // compareReceivedTxs compares the received set of txs against the given set of txs, @@ -180,118 +291,129 @@ func compareReceivedTxs(recvTxs []common.Hash, txs []*types.Transaction) (presen return present, missing } -func unknownTx(t *utesting.T, s *Suite) *types.Transaction { - tx := getNextTxFromChain(t, s) +func unknownTx(s *Suite) *types.Transaction { + tx := getNextTxFromChain(s) + if tx == nil { + return nil + } var to common.Address if tx.To() != nil { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce()+1, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data()) - return signWithFaucet(t, s.chain.chainConfig, txNew) + return signWithFaucet(s.chain.chainConfig, txNew) } -func getNextTxFromChain(t *utesting.T, s *Suite) *types.Transaction { +func getNextTxFromChain(s *Suite) *types.Transaction { // Get a new transaction - var tx *types.Transaction for _, blocks := range s.fullChain.blocks[s.chain.Len():] { txs := blocks.Transactions() if txs.Len() != 0 { - tx = txs[0] - break + return txs[0] } } - if tx == nil { - t.Fatal("could not find transaction") - } - return tx + return nil } -func generateTxs(t *utesting.T, s *Suite, numTxs int) (map[common.Hash]common.Hash, []*types.Transaction) { +func generateTxs(s *Suite, numTxs int) (map[common.Hash]common.Hash, []*types.Transaction, error) { txHashMap := make(map[common.Hash]common.Hash, numTxs) txs := make([]*types.Transaction, numTxs) - nextTx := getNextTxFromChain(t, s) + nextTx := getNextTxFromChain(s) + if nextTx == nil { + return nil, nil, fmt.Errorf("failed to get the next transaction") + } gas := nextTx.Gas() nonce = nonce + 1 // generate txs for i := 0; i < numTxs; i++ { - tx := generateTx(t, s.chain.chainConfig, nonce, gas) + tx := generateTx(s.chain.chainConfig, nonce, gas) + if tx == nil { + return nil, nil, fmt.Errorf("failed to get the next transaction") + } txHashMap[tx.Hash()] = tx.Hash() txs[i] = tx nonce = nonce + 1 } - return txHashMap, txs + return txHashMap, txs, nil } -func generateTx(t *utesting.T, chainConfig *params.ChainConfig, nonce uint64, gas uint64) *types.Transaction { +func generateTx(chainConfig *params.ChainConfig, nonce uint64, gas uint64) *types.Transaction { var to common.Address tx := types.NewTransaction(nonce, to, big.NewInt(1), gas, big.NewInt(1), []byte{}) - return signWithFaucet(t, chainConfig, tx) + return signWithFaucet(chainConfig, tx) } -func getOldTxFromChain(t *utesting.T, s *Suite) *types.Transaction { - var tx *types.Transaction +func getOldTxFromChain(s *Suite) *types.Transaction { for _, blocks := range s.fullChain.blocks[:s.chain.Len()-1] { txs := blocks.Transactions() if txs.Len() != 0 { - tx = txs[0] - break + return txs[0] } } - if tx == nil { - t.Fatal("could not find transaction") - } - return tx + return nil } -func invalidNonceTx(t *utesting.T, s *Suite) *types.Transaction { - tx := getNextTxFromChain(t, s) +func invalidNonceTx(s *Suite) *types.Transaction { + tx := getNextTxFromChain(s) + if tx == nil { + return nil + } var to common.Address if tx.To() != nil { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce()-2, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data()) - return signWithFaucet(t, s.chain.chainConfig, txNew) + return signWithFaucet(s.chain.chainConfig, txNew) } -func hugeAmount(t *utesting.T, s *Suite) *types.Transaction { - tx := getNextTxFromChain(t, s) +func hugeAmount(s *Suite) *types.Transaction { + tx := getNextTxFromChain(s) + if tx == nil { + return nil + } amount := largeNumber(2) var to common.Address if tx.To() != nil { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce(), to, amount, tx.Gas(), tx.GasPrice(), tx.Data()) - return signWithFaucet(t, s.chain.chainConfig, txNew) + return signWithFaucet(s.chain.chainConfig, txNew) } -func hugeGasPrice(t *utesting.T, s *Suite) *types.Transaction { - tx := getNextTxFromChain(t, s) +func hugeGasPrice(s *Suite) *types.Transaction { + tx := getNextTxFromChain(s) + if tx == nil { + return nil + } gasPrice := largeNumber(2) var to common.Address if tx.To() != nil { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), gasPrice, tx.Data()) - return signWithFaucet(t, s.chain.chainConfig, txNew) + return signWithFaucet(s.chain.chainConfig, txNew) } -func hugeData(t *utesting.T, s *Suite) *types.Transaction { - tx := getNextTxFromChain(t, s) +func hugeData(s *Suite) *types.Transaction { + tx := getNextTxFromChain(s) + if tx == nil { + return nil + } var to common.Address if tx.To() != nil { to = *tx.To() } txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), tx.GasPrice(), largeBuffer(2)) - return signWithFaucet(t, s.chain.chainConfig, txNew) + return signWithFaucet(s.chain.chainConfig, txNew) } -func signWithFaucet(t *utesting.T, chainConfig *params.ChainConfig, tx *types.Transaction) *types.Transaction { +func signWithFaucet(chainConfig *params.ChainConfig, tx *types.Transaction) *types.Transaction { signer := types.LatestSigner(chainConfig) signedTx, err := types.SignTx(tx, signer, faucetKey) if err != nil { - t.Fatalf("could not sign tx: %v\n", err) + return nil } return signedTx } diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index 50a69b9418..e49ea284e9 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -19,13 +19,8 @@ package ethtest import ( "crypto/ecdsa" "fmt" - "reflect" - "time" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" "github.com/ethereum/go-ethereum/rlp" @@ -137,6 +132,7 @@ type Conn struct { caps []p2p.Cap } +// Read reads an eth packet from the connection. func (c *Conn) Read() Message { code, rawData, _, err := c.Conn.Read() if err != nil { @@ -185,32 +181,83 @@ func (c *Conn) Read() Message { return msg } -// ReadAndServe serves GetBlockHeaders requests while waiting -// on another message from the node. -func (c *Conn) ReadAndServe(chain *Chain, timeout time.Duration) Message { - start := time.Now() - for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(5 * time.Second)) - switch msg := c.Read().(type) { - case *Ping: - c.Write(&Pong{}) - case *GetBlockHeaders: - req := *msg - headers, err := chain.GetHeaders(req) - if err != nil { - return errorf("could not get headers for inbound header request: %v", err) - } - - if err := c.Write(headers); err != nil { - return errorf("could not write to connection: %v", err) - } - default: - return msg +// Read66 reads an eth66 packet from the connection. +func (c *Conn) Read66() (uint64, Message) { + code, rawData, _, err := c.Conn.Read() + if err != nil { + return 0, errorf("could not read from connection: %v", err) + } + + var msg Message + switch int(code) { + case (Hello{}).Code(): + msg = new(Hello) + case (Ping{}).Code(): + msg = new(Ping) + case (Pong{}).Code(): + msg = new(Pong) + case (Disconnect{}).Code(): + msg = new(Disconnect) + case (Status{}).Code(): + msg = new(Status) + case (GetBlockHeaders{}).Code(): + ethMsg := new(eth.GetBlockHeadersPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) } + return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) + case (BlockHeaders{}).Code(): + ethMsg := new(eth.BlockHeadersPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) + case (GetBlockBodies{}).Code(): + ethMsg := new(eth.GetBlockBodiesPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) + case (BlockBodies{}).Code(): + ethMsg := new(eth.BlockBodiesPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) + case (NewBlock{}).Code(): + msg = new(NewBlock) + case (NewBlockHashes{}).Code(): + msg = new(NewBlockHashes) + case (Transactions{}).Code(): + msg = new(Transactions) + case (NewPooledTransactionHashes{}).Code(): + msg = new(NewPooledTransactionHashes) + case (GetPooledTransactions{}.Code()): + ethMsg := new(eth.GetPooledTransactionsPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) + case (PooledTransactions{}.Code()): + ethMsg := new(eth.PooledTransactionsPacket66) + if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) + default: + msg = errorf("invalid message code: %d", code) + } + + if msg != nil { + if err := rlp.DecodeBytes(rawData, msg); err != nil { + return 0, errorf("could not rlp decode message: %v", err) + } + return 0, msg } - return errorf("no message received within %v", timeout) + return 0, errorf("invalid message: %s", string(rawData)) } +// Write writes a eth packet to the connection. func (c *Conn) Write(msg Message) error { // check if message is eth protocol message var ( @@ -225,135 +272,12 @@ func (c *Conn) Write(msg Message) error { return err } -// handshake checks to make sure a `HELLO` is received. -func (c *Conn) handshake(t *utesting.T) Message { - defer c.SetDeadline(time.Time{}) - c.SetDeadline(time.Now().Add(10 * time.Second)) - - // write hello to client - pub0 := crypto.FromECDSAPub(&c.ourKey.PublicKey)[1:] - ourHandshake := &Hello{ - Version: 5, - Caps: c.caps, - ID: pub0, - } - if err := c.Write(ourHandshake); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // read hello from client - switch msg := c.Read().(type) { - case *Hello: - // set snappy if version is at least 5 - if msg.Version >= 5 { - c.SetSnappy(true) - } - c.negotiateEthProtocol(msg.Caps) - if c.negotiatedProtoVersion == 0 { - t.Fatalf("unexpected eth protocol version") - } - return msg - default: - t.Fatalf("bad handshake: %#v", msg) - return nil - } -} - -// negotiateEthProtocol sets the Conn's eth protocol version -// to highest advertised capability from peer -func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) { - var highestEthVersion uint - for _, capability := range caps { - if capability.Name != "eth" { - continue - } - if capability.Version > highestEthVersion && capability.Version <= c.ourHighestProtoVersion { - highestEthVersion = capability.Version - } - } - c.negotiatedProtoVersion = highestEthVersion -} - -// statusExchange performs a `Status` message exchange with the given -// node. -func (c *Conn) statusExchange(t *utesting.T, chain *Chain, status *Status) Message { - defer c.SetDeadline(time.Time{}) - c.SetDeadline(time.Now().Add(20 * time.Second)) - - // read status message from client - var message Message -loop: - for { - switch msg := c.Read().(type) { - case *Status: - if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want { - t.Fatalf("wrong head block in status, want: %#x (block %d) have %#x", - want, chain.blocks[chain.Len()-1].NumberU64(), have) - } - if have, want := msg.TD.Cmp(chain.TD(chain.Len())), 0; have != want { - t.Fatalf("wrong TD in status: have %v want %v", have, want) - } - if have, want := msg.ForkID, chain.ForkID(); !reflect.DeepEqual(have, want) { - t.Fatalf("wrong fork ID in status: have %v, want %v", have, want) - } - message = msg - break loop - case *Disconnect: - t.Fatalf("disconnect received: %v", msg.Reason) - case *Ping: - c.Write(&Pong{}) // TODO (renaynay): in the future, this should be an error - // (PINGs should not be a response upon fresh connection) - default: - t.Fatalf("bad status message: %s", pretty.Sdump(msg)) - } - } - // make sure eth protocol version is set for negotiation - if c.negotiatedProtoVersion == 0 { - t.Fatalf("eth protocol version must be set in Conn") - } - if status == nil { - // write status message to client - status = &Status{ - ProtocolVersion: uint32(c.negotiatedProtoVersion), - NetworkID: chain.chainConfig.ChainID.Uint64(), - TD: chain.TD(chain.Len()), - Head: chain.blocks[chain.Len()-1].Hash(), - Genesis: chain.blocks[0].Hash(), - ForkID: chain.ForkID(), - } - } - - if err := c.Write(status); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - - return message -} - -// waitForBlock waits for confirmation from the client that it has -// imported the given block. -func (c *Conn) waitForBlock(block *types.Block) error { - defer c.SetReadDeadline(time.Time{}) - - c.SetReadDeadline(time.Now().Add(20 * time.Second)) - // note: if the node has not yet imported the block, it will respond - // to the GetBlockHeaders request with an empty BlockHeaders response, - // so the GetBlockHeaders request must be sent again until the BlockHeaders - // response contains the desired header. - for { - req := &GetBlockHeaders{Origin: eth.HashOrNumber{Hash: block.Hash()}, Amount: 1} - if err := c.Write(req); err != nil { - return err - } - switch msg := c.Read().(type) { - case *BlockHeaders: - for _, header := range *msg { - if header.Number.Uint64() == block.NumberU64() { - return nil - } - } - time.Sleep(100 * time.Millisecond) - default: - return fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) - } +// Write66 writes an eth66 packet to the connection. +func (c *Conn) Write66(req eth.Packet, code int) error { + payload, err := rlp.EncodeToBytes(req) + if err != nil { + return err } + _, err = c.Conn.Write(uint64(code), payload) + return err } From 10962b685ef08579846342b3ab80507306ccd494 Mon Sep 17 00:00:00 2001 From: meowsbits Date: Tue, 25 May 2021 16:22:46 -0500 Subject: [PATCH 389/709] ethstats: fix URL parser for '@' or ':' in node name/password (#21640) Fixes the case (example below) where the value passed to --ethstats flag would be parsed wrongly because the node name and/or password value contained the special characters '@' or ':' --ethstats "ETC Labs Metrics @meowsbits":mypass@ws://mordor.dash.fault.dev:3000 --- ethstats/ethstats.go | 39 +++++++++++++++++------ ethstats/ethstats_test.go | 67 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 ethstats/ethstats_test.go diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index c7acb9481c..ef83e5a4eb 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -24,7 +24,6 @@ import ( "fmt" "math/big" "net/http" - "regexp" "runtime" "strconv" "strings" @@ -144,21 +143,43 @@ func (w *connWrapper) Close() error { return w.conn.Close() } +// parseEthstatsURL parses the netstats connection url. +// URL argument should be of the form +// If non-erroring, the returned slice contains 3 elements: [nodename, pass, host] +func parseEthstatsURL(url string) (parts []string, err error) { + err = fmt.Errorf("invalid netstats url: \"%s\", should be nodename:secret@host:port", url) + + hostIndex := strings.LastIndex(url, "@") + if hostIndex == -1 || hostIndex == len(url)-1 { + return nil, err + } + preHost, host := url[:hostIndex], url[hostIndex+1:] + + passIndex := strings.LastIndex(preHost, ":") + if passIndex == -1 { + return []string{preHost, "", host}, nil + } + nodename, pass := preHost[:passIndex], "" + if passIndex != len(preHost)-1 { + pass = preHost[passIndex+1:] + } + + return []string{nodename, pass, host}, nil +} + // New returns a monitoring service ready for stats reporting. func New(node *node.Node, backend backend, engine consensus.Engine, url string) error { - // Parse the netstats connection url - re := regexp.MustCompile("([^:@]*)(:([^@]*))?@(.+)") - parts := re.FindStringSubmatch(url) - if len(parts) != 5 { - return fmt.Errorf("invalid netstats url: \"%s\", should be nodename:secret@host:port", url) + parts, err := parseEthstatsURL(url) + if err != nil { + return err } ethstats := &Service{ backend: backend, engine: engine, server: node.Server(), - node: parts[1], - pass: parts[3], - host: parts[4], + node: parts[0], + pass: parts[1], + host: parts[2], pongCh: make(chan struct{}), histCh: make(chan []uint64, 1), } diff --git a/ethstats/ethstats_test.go b/ethstats/ethstats_test.go new file mode 100644 index 0000000000..92cec50c4d --- /dev/null +++ b/ethstats/ethstats_test.go @@ -0,0 +1,67 @@ +package ethstats + +import ( + "strconv" + "testing" +) + +func TestParseEthstatsURL(t *testing.T) { + cases := []struct { + url string + node, pass, host string + }{ + { + url: `"debug meowsbits":mypass@ws://mordor.dash.fault.dev:3000`, + node: "debug meowsbits", pass: "mypass", host: "ws://mordor.dash.fault.dev:3000", + }, + { + url: `"debug @meowsbits":mypass@ws://mordor.dash.fault.dev:3000`, + node: "debug @meowsbits", pass: "mypass", host: "ws://mordor.dash.fault.dev:3000", + }, + { + url: `"debug: @meowsbits":mypass@ws://mordor.dash.fault.dev:3000`, + node: "debug: @meowsbits", pass: "mypass", host: "ws://mordor.dash.fault.dev:3000", + }, + { + url: `name:@ws://mordor.dash.fault.dev:3000`, + node: "name", pass: "", host: "ws://mordor.dash.fault.dev:3000", + }, + { + url: `name@ws://mordor.dash.fault.dev:3000`, + node: "name", pass: "", host: "ws://mordor.dash.fault.dev:3000", + }, + { + url: `:mypass@ws://mordor.dash.fault.dev:3000`, + node: "", pass: "mypass", host: "ws://mordor.dash.fault.dev:3000", + }, + { + url: `:@ws://mordor.dash.fault.dev:3000`, + node: "", pass: "", host: "ws://mordor.dash.fault.dev:3000", + }, + } + + for i, c := range cases { + parts, err := parseEthstatsURL(c.url) + if err != nil { + t.Fatal(err) + } + node, pass, host := parts[0], parts[1], parts[2] + + // unquote because the value provided will be used as a CLI flag value, so unescaped quotes will be removed + nodeUnquote, err := strconv.Unquote(node) + if err == nil { + node = nodeUnquote + } + + if node != c.node { + t.Errorf("case=%d mismatch node value, got: %v ,want: %v", i, node, c.node) + } + if pass != c.pass { + t.Errorf("case=%d mismatch pass value, got: %v ,want: %v", i, pass, c.pass) + } + if host != c.host { + t.Errorf("case=%d mismatch host value, got: %v ,want: %v", i, host, c.host) + } + } + +} From 05dab7f6bde376d1608ee7800b38cee7ce600d4d Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Wed, 26 May 2021 02:39:41 -0400 Subject: [PATCH 390/709] internal/ethapi: remove unused vm.Config parameter of DoCall (#22942) --- graphql/graphql.go | 5 ++--- internal/ethapi/api.go | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 71d80d8abd..94287b0d6b 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" @@ -870,7 +869,7 @@ func (b *Block) Call(ctx context.Context, args struct { return nil, err } } - result, err := ethapi.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, vm.Config{}, 5*time.Second, b.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, 5*time.Second, b.backend.RPCGasCap()) if err != nil { return nil, err } @@ -940,7 +939,7 @@ func (p *Pending) Call(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (*CallResult, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, vm.Config{}, 5*time.Second, p.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, 5*time.Second, p.backend.RPCGasCap()) if err != nil { return nil, err } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b06df8ff9f..bf7c8e533d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -806,7 +806,7 @@ func (diff *StateOverride) Apply(state *state.StateDB) error { return nil } -func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, vmCfg vm.Config, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { +func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) @@ -895,7 +895,7 @@ func (e *revertError) ErrorData() interface{} { // Note, this function doesn't make and changes in the state/blockchain and is // useful to execute and retrieve values. func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { - result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, vm.Config{}, 5*time.Second, s.b.RPCGasCap()) + result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, 5*time.Second, s.b.RPCGasCap()) if err != nil { return nil, err } @@ -969,7 +969,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr executable := func(gas uint64) (bool, *core.ExecutionResult, error) { args.Gas = (*hexutil.Uint64)(&gas) - result, err := DoCall(ctx, b, args, blockNrOrHash, nil, vm.Config{}, 0, gasCap) + result, err := DoCall(ctx, b, args, blockNrOrHash, nil, 0, gasCap) if err != nil { if errors.Is(err, core.ErrIntrinsicGas) { return true, nil, nil // Special case, raise gas limit From c73652da0bb0ca4a4ecf3b88b0efed085be9adc4 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 26 May 2021 15:58:09 +0800 Subject: [PATCH 391/709] core/state/snapshot: fix flaky tests (#22944) * core/state/snapshot: fix flaky tests * core/state/snapshot: fix tests --- core/state/snapshot/generate_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 3a669085f7..a92517b315 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -71,7 +71,7 @@ func TestGeneration(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -136,7 +136,7 @@ func TestGenerateExistentState(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -309,7 +309,7 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -361,7 +361,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -406,7 +406,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // Snapshot generation succeeded t.Errorf("Snapshot generated against corrupt account trie") - case <-time.After(250 * time.Millisecond): + case <-time.After(time.Second): // Not generated fast enough, hopefully blocked inside on missing trie node fail } // Signal abortion to the generator and wait for it to tear down @@ -466,7 +466,7 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // Snapshot generation succeeded t.Errorf("Snapshot generated against corrupt storage trie") - case <-time.After(250 * time.Millisecond): + case <-time.After(time.Second): // Not generated fast enough, hopefully blocked inside on missing trie node fail } // Signal abortion to the generator and wait for it to tear down @@ -525,7 +525,7 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // Snapshot generation succeeded t.Errorf("Snapshot generated against corrupt storage trie") - case <-time.After(250 * time.Millisecond): + case <-time.After(time.Second): // Not generated fast enough, hopefully blocked inside on missing trie node fail } // Signal abortion to the generator and wait for it to tear down @@ -588,7 +588,7 @@ func TestGenerateWithExtraAccounts(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -646,7 +646,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -699,7 +699,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -743,7 +743,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -775,7 +775,7 @@ func TestGenerateFromEmptySnap(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(1 * time.Second): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) @@ -822,7 +822,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { case <-snap.genPending: // Snapshot generation succeeded - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) From 5869789d7500dba00e8f78382d5af82ef73a9b0e Mon Sep 17 00:00:00 2001 From: Mike Burr Date: Wed, 26 May 2021 14:33:00 -0600 Subject: [PATCH 392/709] ethstats: fix typo in comment (#22952) Trivial but helpful to understanding. --- ethstats/ethstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index ef83e5a4eb..42d88f6db6 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -353,7 +353,7 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core // it, if they themselves are requests it initiates a reply, and lastly it drops // unknown packets. func (s *Service) readLoop(conn *connWrapper) { - // If the read loop exists, close the connection + // If the read loop exits, close the connection defer conn.Close() for { From 2e7714f8648b375f69f4146e702d5f19b36e55ba Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 27 May 2021 10:13:35 +0200 Subject: [PATCH 393/709] cmd/utils: avoid large alloc in --dev mode (#22949) * cmd/utils: avoid 1Gb alloc in --dev mode * cmd/geth: avoid 512Mb alloc in genesis query tests --- cmd/geth/genesis_test.go | 2 +- cmd/utils/flags.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index cbc1b38374..0563ef3c42 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -84,7 +84,7 @@ func TestCustomGenesis(t *testing.T) { runGeth(t, "--datadir", datadir, "init", json).WaitExit() // Query the custom genesis block - geth := runGeth(t, "--networkid", "1337", "--syncmode=full", + geth := runGeth(t, "--networkid", "1337", "--syncmode=full", "--cache", "16", "--datadir", datadir, "--maxpeers", "0", "--port", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--exec", tt.query, "console") diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1ae07108e4..c41286e4ec 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1234,6 +1234,9 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name) } + if ctx.GlobalIsSet(DeveloperFlag.Name) { + cfg.UseLightweightKDF = true + } if ctx.GlobalIsSet(LightKDFFlag.Name) { cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name) } @@ -1647,6 +1650,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 } + cfg.SyncMode = downloader.FullSync // Create new developer account or reuse existing one var ( developer accounts.Account From 7194c847b6e7f545f2aad57d8eae0a046e08d7a4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 27 May 2021 10:19:13 +0200 Subject: [PATCH 394/709] p2p/rlpx: reduce allocation and syscalls (#22899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change significantly improves the performance of RLPx message reads and writes. In the previous implementation, reading and writing of message frames performed multiple reads and writes on the underlying network connection, and allocated a new []byte buffer for every read. In the new implementation, reads and writes re-use buffers, and perform much fewer system calls on the underlying connection. This doubles the theoretically achievable throughput on a single connection, as shown by the benchmark result: name old speed new speed delta Throughput-8 70.3MB/s ± 0% 155.4MB/s ± 0% +121.11% (p=0.000 n=9+8) The change also removes support for the legacy, pre-EIP-8 handshake encoding. As of May 2021, no actively maintained client sends this format. --- p2p/rlpx/buffer.go | 127 +++++++++++++ p2p/rlpx/buffer_test.go | 51 ++++++ p2p/rlpx/rlpx.go | 385 ++++++++++++++++++++-------------------- p2p/rlpx/rlpx_test.go | 123 +++++++++---- p2p/transport.go | 5 + rlp/raw.go | 8 + rlp/raw_test.go | 6 + 7 files changed, 478 insertions(+), 227 deletions(-) create mode 100644 p2p/rlpx/buffer.go create mode 100644 p2p/rlpx/buffer_test.go diff --git a/p2p/rlpx/buffer.go b/p2p/rlpx/buffer.go new file mode 100644 index 0000000000..bb38e10577 --- /dev/null +++ b/p2p/rlpx/buffer.go @@ -0,0 +1,127 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlpx + +import ( + "io" +) + +// readBuffer implements buffering for network reads. This type is similar to bufio.Reader, +// with two crucial differences: the buffer slice is exposed, and the buffer keeps all +// read data available until reset. +// +// How to use this type: +// +// Keep a readBuffer b alongside the underlying network connection. When reading a packet +// from the connection, first call b.reset(). This empties b.data. Now perform reads +// through b.read() until the end of the packet is reached. The complete packet data is +// now available in b.data. +type readBuffer struct { + data []byte + end int +} + +// reset removes all processed data which was read since the last call to reset. +// After reset, len(b.data) is zero. +func (b *readBuffer) reset() { + unprocessed := b.end - len(b.data) + copy(b.data[:unprocessed], b.data[len(b.data):b.end]) + b.end = unprocessed + b.data = b.data[:0] +} + +// read reads at least n bytes from r, returning the bytes. +// The returned slice is valid until the next call to reset. +func (b *readBuffer) read(r io.Reader, n int) ([]byte, error) { + offset := len(b.data) + have := b.end - len(b.data) + + // If n bytes are available in the buffer, there is no need to read from r at all. + if have >= n { + b.data = b.data[:offset+n] + return b.data[offset : offset+n], nil + } + + // Make buffer space available. + need := n - have + b.grow(need) + + // Read. + rn, err := io.ReadAtLeast(r, b.data[b.end:cap(b.data)], need) + if err != nil { + return nil, err + } + b.end += rn + b.data = b.data[:offset+n] + return b.data[offset : offset+n], nil +} + +// grow ensures the buffer has at least n bytes of unused space. +func (b *readBuffer) grow(n int) { + if cap(b.data)-b.end >= n { + return + } + need := n - (cap(b.data) - b.end) + offset := len(b.data) + b.data = append(b.data[:cap(b.data)], make([]byte, need)...) + b.data = b.data[:offset] +} + +// writeBuffer implements buffering for network writes. This is essentially +// a convenience wrapper around a byte slice. +type writeBuffer struct { + data []byte +} + +func (b *writeBuffer) reset() { + b.data = b.data[:0] +} + +func (b *writeBuffer) appendZero(n int) []byte { + offset := len(b.data) + b.data = append(b.data, make([]byte, n)...) + return b.data[offset : offset+n] +} + +func (b *writeBuffer) Write(data []byte) (int, error) { + b.data = append(b.data, data...) + return len(data), nil +} + +const maxUint24 = int(^uint32(0) >> 8) + +func readUint24(b []byte) uint32 { + return uint32(b[2]) | uint32(b[1])<<8 | uint32(b[0])<<16 +} + +func putUint24(v uint32, b []byte) { + b[0] = byte(v >> 16) + b[1] = byte(v >> 8) + b[2] = byte(v) +} + +// growslice ensures b has the wanted length by either expanding it to its capacity +// or allocating a new slice if b has insufficient capacity. +func growslice(b []byte, wantLength int) []byte { + if len(b) >= wantLength { + return b + } + if cap(b) >= wantLength { + return b[:cap(b)] + } + return make([]byte, wantLength) +} diff --git a/p2p/rlpx/buffer_test.go b/p2p/rlpx/buffer_test.go new file mode 100644 index 0000000000..9fee4172bd --- /dev/null +++ b/p2p/rlpx/buffer_test.go @@ -0,0 +1,51 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlpx + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/stretchr/testify/assert" +) + +func TestReadBufferReset(t *testing.T) { + reader := bytes.NewReader(hexutil.MustDecode("0x010202030303040505")) + var b readBuffer + + s1, _ := b.read(reader, 1) + s2, _ := b.read(reader, 2) + s3, _ := b.read(reader, 3) + + assert.Equal(t, []byte{1}, s1) + assert.Equal(t, []byte{2, 2}, s2) + assert.Equal(t, []byte{3, 3, 3}, s3) + + b.reset() + + s4, _ := b.read(reader, 1) + s5, _ := b.read(reader, 2) + + assert.Equal(t, []byte{4}, s4) + assert.Equal(t, []byte{5, 5}, s5) + + s6, err := b.read(reader, 2) + + assert.EqualError(t, err, "EOF") + assert.Nil(t, s6) +} diff --git a/p2p/rlpx/rlpx.go b/p2p/rlpx/rlpx.go index 2021bf08be..326c7c4941 100644 --- a/p2p/rlpx/rlpx.go +++ b/p2p/rlpx/rlpx.go @@ -48,19 +48,45 @@ import ( // This type is not generally safe for concurrent use, but reading and writing of messages // may happen concurrently after the handshake. type Conn struct { - dialDest *ecdsa.PublicKey - conn net.Conn - handshake *handshakeState - snappy bool + dialDest *ecdsa.PublicKey + conn net.Conn + session *sessionState + + // These are the buffers for snappy compression. + // Compression is enabled if they are non-nil. + snappyReadBuffer []byte + snappyWriteBuffer []byte } -type handshakeState struct { +// sessionState contains the session keys. +type sessionState struct { enc cipher.Stream dec cipher.Stream - macCipher cipher.Block - egressMAC hash.Hash - ingressMAC hash.Hash + egressMAC hashMAC + ingressMAC hashMAC + rbuf readBuffer + wbuf writeBuffer +} + +// hashMAC holds the state of the RLPx v4 MAC contraption. +type hashMAC struct { + cipher cipher.Block + hash hash.Hash + aesBuffer [16]byte + hashBuffer [32]byte + seedBuffer [32]byte +} + +func newHashMAC(cipher cipher.Block, h hash.Hash) hashMAC { + m := hashMAC{cipher: cipher, hash: h} + if cipher.BlockSize() != len(m.aesBuffer) { + panic(fmt.Errorf("invalid MAC cipher block size %d", cipher.BlockSize())) + } + if h.Size() != len(m.hashBuffer) { + panic(fmt.Errorf("invalid MAC digest size %d", h.Size())) + } + return m } // NewConn wraps the given network connection. If dialDest is non-nil, the connection @@ -76,7 +102,13 @@ func NewConn(conn net.Conn, dialDest *ecdsa.PublicKey) *Conn { // after the devp2p Hello message exchange when the negotiated version indicates that // compression is available on both ends of the connection. func (c *Conn) SetSnappy(snappy bool) { - c.snappy = snappy + if snappy { + c.snappyReadBuffer = []byte{} + c.snappyWriteBuffer = []byte{} + } else { + c.snappyReadBuffer = nil + c.snappyWriteBuffer = nil + } } // SetReadDeadline sets the deadline for all future read operations. @@ -95,12 +127,13 @@ func (c *Conn) SetDeadline(time time.Time) error { } // Read reads a message from the connection. +// The returned data buffer is valid until the next call to Read. func (c *Conn) Read() (code uint64, data []byte, wireSize int, err error) { - if c.handshake == nil { + if c.session == nil { panic("can't ReadMsg before handshake") } - frame, err := c.handshake.readFrame(c.conn) + frame, err := c.session.readFrame(c.conn) if err != nil { return 0, nil, 0, err } @@ -111,7 +144,7 @@ func (c *Conn) Read() (code uint64, data []byte, wireSize int, err error) { wireSize = len(data) // If snappy is enabled, verify and decompress message. - if c.snappy { + if c.snappyReadBuffer != nil { var actualSize int actualSize, err = snappy.DecodedLen(data) if err != nil { @@ -120,51 +153,55 @@ func (c *Conn) Read() (code uint64, data []byte, wireSize int, err error) { if actualSize > maxUint24 { return code, nil, 0, errPlainMessageTooLarge } - data, err = snappy.Decode(nil, data) + c.snappyReadBuffer = growslice(c.snappyReadBuffer, actualSize) + data, err = snappy.Decode(c.snappyReadBuffer, data) } return code, data, wireSize, err } -func (h *handshakeState) readFrame(conn io.Reader) ([]byte, error) { - // read the header - headbuf := make([]byte, 32) - if _, err := io.ReadFull(conn, headbuf); err != nil { +func (h *sessionState) readFrame(conn io.Reader) ([]byte, error) { + h.rbuf.reset() + + // Read the frame header. + header, err := h.rbuf.read(conn, 32) + if err != nil { return nil, err } - // verify header mac - shouldMAC := updateMAC(h.ingressMAC, h.macCipher, headbuf[:16]) - if !hmac.Equal(shouldMAC, headbuf[16:]) { + // Verify header MAC. + wantHeaderMAC := h.ingressMAC.computeHeader(header[:16]) + if !hmac.Equal(wantHeaderMAC, header[16:]) { return nil, errors.New("bad header MAC") } - h.dec.XORKeyStream(headbuf[:16], headbuf[:16]) // first half is now decrypted - fsize := readInt24(headbuf) - // ignore protocol type for now - // read the frame content - var rsize = fsize // frame size rounded up to 16 byte boundary + // Decrypt the frame header to get the frame size. + h.dec.XORKeyStream(header[:16], header[:16]) + fsize := readUint24(header[:16]) + // Frame size rounded up to 16 byte boundary for padding. + rsize := fsize if padding := fsize % 16; padding > 0 { rsize += 16 - padding } - framebuf := make([]byte, rsize) - if _, err := io.ReadFull(conn, framebuf); err != nil { + + // Read the frame content. + frame, err := h.rbuf.read(conn, int(rsize)) + if err != nil { return nil, err } - // read and validate frame MAC. we can re-use headbuf for that. - h.ingressMAC.Write(framebuf) - fmacseed := h.ingressMAC.Sum(nil) - if _, err := io.ReadFull(conn, headbuf[:16]); err != nil { + // Validate frame MAC. + frameMAC, err := h.rbuf.read(conn, 16) + if err != nil { return nil, err } - shouldMAC = updateMAC(h.ingressMAC, h.macCipher, fmacseed) - if !hmac.Equal(shouldMAC, headbuf[:16]) { + wantFrameMAC := h.ingressMAC.computeFrame(frame) + if !hmac.Equal(wantFrameMAC, frameMAC) { return nil, errors.New("bad frame MAC") } - // decrypt frame content - h.dec.XORKeyStream(framebuf, framebuf) - return framebuf[:fsize], nil + // Decrypt the frame data. + h.dec.XORKeyStream(frame, frame) + return frame[:fsize], nil } // Write writes a message to the connection. @@ -172,83 +209,90 @@ func (h *handshakeState) readFrame(conn io.Reader) ([]byte, error) { // Write returns the written size of the message data. This may be less than or equal to // len(data) depending on whether snappy compression is enabled. func (c *Conn) Write(code uint64, data []byte) (uint32, error) { - if c.handshake == nil { + if c.session == nil { panic("can't WriteMsg before handshake") } if len(data) > maxUint24 { return 0, errPlainMessageTooLarge } - if c.snappy { - data = snappy.Encode(nil, data) + if c.snappyWriteBuffer != nil { + // Ensure the buffer has sufficient size. + // Package snappy will allocate its own buffer if the provided + // one is smaller than MaxEncodedLen. + c.snappyWriteBuffer = growslice(c.snappyWriteBuffer, snappy.MaxEncodedLen(len(data))) + data = snappy.Encode(c.snappyWriteBuffer, data) } wireSize := uint32(len(data)) - err := c.handshake.writeFrame(c.conn, code, data) + err := c.session.writeFrame(c.conn, code, data) return wireSize, err } -func (h *handshakeState) writeFrame(conn io.Writer, code uint64, data []byte) error { - ptype, _ := rlp.EncodeToBytes(code) +func (h *sessionState) writeFrame(conn io.Writer, code uint64, data []byte) error { + h.wbuf.reset() - // write header - headbuf := make([]byte, 32) - fsize := len(ptype) + len(data) + // Write header. + fsize := rlp.IntSize(code) + len(data) if fsize > maxUint24 { return errPlainMessageTooLarge } - putInt24(uint32(fsize), headbuf) - copy(headbuf[3:], zeroHeader) - h.enc.XORKeyStream(headbuf[:16], headbuf[:16]) // first half is now encrypted + header := h.wbuf.appendZero(16) + putUint24(uint32(fsize), header) + copy(header[3:], zeroHeader) + h.enc.XORKeyStream(header, header) - // write header MAC - copy(headbuf[16:], updateMAC(h.egressMAC, h.macCipher, headbuf[:16])) - if _, err := conn.Write(headbuf); err != nil { - return err - } + // Write header MAC. + h.wbuf.Write(h.egressMAC.computeHeader(header)) - // write encrypted frame, updating the egress MAC hash with - // the data written to conn. - tee := cipher.StreamWriter{S: h.enc, W: io.MultiWriter(conn, h.egressMAC)} - if _, err := tee.Write(ptype); err != nil { - return err - } - if _, err := tee.Write(data); err != nil { - return err - } + // Encode and encrypt the frame data. + offset := len(h.wbuf.data) + h.wbuf.data = rlp.AppendUint64(h.wbuf.data, code) + h.wbuf.Write(data) if padding := fsize % 16; padding > 0 { - if _, err := tee.Write(zero16[:16-padding]); err != nil { - return err - } + h.wbuf.appendZero(16 - padding) } + framedata := h.wbuf.data[offset:] + h.enc.XORKeyStream(framedata, framedata) - // write frame MAC. egress MAC hash is up to date because - // frame content was written to it as well. - fmacseed := h.egressMAC.Sum(nil) - mac := updateMAC(h.egressMAC, h.macCipher, fmacseed) - _, err := conn.Write(mac) + // Write frame MAC. + h.wbuf.Write(h.egressMAC.computeFrame(framedata)) + + _, err := conn.Write(h.wbuf.data) return err } -func readInt24(b []byte) uint32 { - return uint32(b[2]) | uint32(b[1])<<8 | uint32(b[0])<<16 +// computeHeader computes the MAC of a frame header. +func (m *hashMAC) computeHeader(header []byte) []byte { + sum1 := m.hash.Sum(m.hashBuffer[:0]) + return m.compute(sum1, header) } -func putInt24(v uint32, b []byte) { - b[0] = byte(v >> 16) - b[1] = byte(v >> 8) - b[2] = byte(v) +// computeFrame computes the MAC of framedata. +func (m *hashMAC) computeFrame(framedata []byte) []byte { + m.hash.Write(framedata) + seed := m.hash.Sum(m.seedBuffer[:0]) + return m.compute(seed, seed[:16]) } -// updateMAC reseeds the given hash with encrypted seed. -// it returns the first 16 bytes of the hash sum after seeding. -func updateMAC(mac hash.Hash, block cipher.Block, seed []byte) []byte { - aesbuf := make([]byte, aes.BlockSize) - block.Encrypt(aesbuf, mac.Sum(nil)) - for i := range aesbuf { - aesbuf[i] ^= seed[i] +// compute computes the MAC of a 16-byte 'seed'. +// +// To do this, it encrypts the current value of the hash state, then XORs the ciphertext +// with seed. The obtained value is written back into the hash state and hash output is +// taken again. The first 16 bytes of the resulting sum are the MAC value. +// +// This MAC construction is a horrible, legacy thing. +func (m *hashMAC) compute(sum1, seed []byte) []byte { + if len(seed) != len(m.aesBuffer) { + panic("invalid MAC seed") + } + + m.cipher.Encrypt(m.aesBuffer[:], sum1) + for i := range m.aesBuffer { + m.aesBuffer[i] ^= seed[i] } - mac.Write(aesbuf) - return mac.Sum(nil)[:16] + m.hash.Write(m.aesBuffer[:]) + sum2 := m.hash.Sum(m.hashBuffer[:0]) + return sum2[:16] } // Handshake performs the handshake. This must be called before any data is written @@ -257,23 +301,26 @@ func (c *Conn) Handshake(prv *ecdsa.PrivateKey) (*ecdsa.PublicKey, error) { var ( sec Secrets err error + h handshakeState ) if c.dialDest != nil { - sec, err = initiatorEncHandshake(c.conn, prv, c.dialDest) + sec, err = h.runInitiator(c.conn, prv, c.dialDest) } else { - sec, err = receiverEncHandshake(c.conn, prv) + sec, err = h.runRecipient(c.conn, prv) } if err != nil { return nil, err } c.InitWithSecrets(sec) + c.session.rbuf = h.rbuf + c.session.wbuf = h.wbuf return sec.remote, err } // InitWithSecrets injects connection secrets as if a handshake had // been performed. This cannot be called after the handshake. func (c *Conn) InitWithSecrets(sec Secrets) { - if c.handshake != nil { + if c.session != nil { panic("can't handshake twice") } macc, err := aes.NewCipher(sec.MAC) @@ -287,12 +334,11 @@ func (c *Conn) InitWithSecrets(sec Secrets) { // we use an all-zeroes IV for AES because the key used // for encryption is ephemeral. iv := make([]byte, encc.BlockSize()) - c.handshake = &handshakeState{ + c.session = &sessionState{ enc: cipher.NewCTR(encc, iv), dec: cipher.NewCTR(encc, iv), - macCipher: macc, - egressMAC: sec.EgressMAC, - ingressMAC: sec.IngressMAC, + egressMAC: newHashMAC(macc, sec.EgressMAC), + ingressMAC: newHashMAC(macc, sec.IngressMAC), } } @@ -303,28 +349,18 @@ func (c *Conn) Close() error { // Constants for the handshake. const ( - maxUint24 = int(^uint32(0) >> 8) - sskLen = 16 // ecies.MaxSharedKeyLength(pubKey) / 2 sigLen = crypto.SignatureLength // elliptic S256 pubLen = 64 // 512 bit pubkey in uncompressed representation without format byte shaLen = 32 // hash length (for nonce etc) - authMsgLen = sigLen + shaLen + pubLen + shaLen + 1 - authRespLen = pubLen + shaLen + 1 - eciesOverhead = 65 /* pubkey */ + 16 /* IV */ + 32 /* MAC */ - - encAuthMsgLen = authMsgLen + eciesOverhead // size of encrypted pre-EIP-8 initiator handshake - encAuthRespLen = authRespLen + eciesOverhead // size of encrypted pre-EIP-8 handshake reply ) var ( // this is used in place of actual frame header data. // TODO: replace this when Msg contains the protocol type code. zeroHeader = []byte{0xC2, 0x80, 0x80} - // sixteen zero bytes - zero16 = make([]byte, 16) // errPlainMessageTooLarge is returned if a decompressed message length exceeds // the allowed 24 bits (i.e. length >= 16MB). @@ -338,19 +374,20 @@ type Secrets struct { remote *ecdsa.PublicKey } -// encHandshake contains the state of the encryption handshake. -type encHandshake struct { +// handshakeState contains the state of the encryption handshake. +type handshakeState struct { initiator bool remote *ecies.PublicKey // remote-pubk initNonce, respNonce []byte // nonce randomPrivKey *ecies.PrivateKey // ecdhe-random remoteRandomPub *ecies.PublicKey // ecdhe-random-pubk + + rbuf readBuffer + wbuf writeBuffer } // RLPx v4 handshake auth (defined in EIP-8). type authMsgV4 struct { - gotPlain bool // whether read packet had plain format. - Signature [sigLen]byte InitiatorPubkey [pubLen]byte Nonce [shaLen]byte @@ -370,17 +407,16 @@ type authRespV4 struct { Rest []rlp.RawValue `rlp:"tail"` } -// receiverEncHandshake negotiates a session token on conn. +// runRecipient negotiates a session token on conn. // it should be called on the listening side of the connection. // // prv is the local client's private key. -func receiverEncHandshake(conn io.ReadWriter, prv *ecdsa.PrivateKey) (s Secrets, err error) { +func (h *handshakeState) runRecipient(conn io.ReadWriter, prv *ecdsa.PrivateKey) (s Secrets, err error) { authMsg := new(authMsgV4) - authPacket, err := readHandshakeMsg(authMsg, encAuthMsgLen, prv, conn) + authPacket, err := h.readMsg(authMsg, prv, conn) if err != nil { return s, err } - h := new(encHandshake) if err := h.handleAuthMsg(authMsg, prv); err != nil { return s, err } @@ -389,22 +425,18 @@ func receiverEncHandshake(conn io.ReadWriter, prv *ecdsa.PrivateKey) (s Secrets, if err != nil { return s, err } - var authRespPacket []byte - if authMsg.gotPlain { - authRespPacket, err = authRespMsg.sealPlain(h) - } else { - authRespPacket, err = sealEIP8(authRespMsg, h) - } + authRespPacket, err := h.sealEIP8(authRespMsg) if err != nil { return s, err } if _, err = conn.Write(authRespPacket); err != nil { return s, err } + return h.secrets(authPacket, authRespPacket) } -func (h *encHandshake) handleAuthMsg(msg *authMsgV4, prv *ecdsa.PrivateKey) error { +func (h *handshakeState) handleAuthMsg(msg *authMsgV4, prv *ecdsa.PrivateKey) error { // Import the remote identity. rpub, err := importPublicKey(msg.InitiatorPubkey[:]) if err != nil { @@ -438,7 +470,7 @@ func (h *encHandshake) handleAuthMsg(msg *authMsgV4, prv *ecdsa.PrivateKey) erro // secrets is called after the handshake is completed. // It extracts the connection secrets from the handshake values. -func (h *encHandshake) secrets(auth, authResp []byte) (Secrets, error) { +func (h *handshakeState) secrets(auth, authResp []byte) (Secrets, error) { ecdheSecret, err := h.randomPrivKey.GenerateShared(h.remoteRandomPub, sskLen, sskLen) if err != nil { return Secrets{}, err @@ -471,21 +503,23 @@ func (h *encHandshake) secrets(auth, authResp []byte) (Secrets, error) { // staticSharedSecret returns the static shared secret, the result // of key agreement between the local and remote static node key. -func (h *encHandshake) staticSharedSecret(prv *ecdsa.PrivateKey) ([]byte, error) { +func (h *handshakeState) staticSharedSecret(prv *ecdsa.PrivateKey) ([]byte, error) { return ecies.ImportECDSA(prv).GenerateShared(h.remote, sskLen, sskLen) } -// initiatorEncHandshake negotiates a session token on conn. +// runInitiator negotiates a session token on conn. // it should be called on the dialing side of the connection. // // prv is the local client's private key. -func initiatorEncHandshake(conn io.ReadWriter, prv *ecdsa.PrivateKey, remote *ecdsa.PublicKey) (s Secrets, err error) { - h := &encHandshake{initiator: true, remote: ecies.ImportECDSAPublic(remote)} +func (h *handshakeState) runInitiator(conn io.ReadWriter, prv *ecdsa.PrivateKey, remote *ecdsa.PublicKey) (s Secrets, err error) { + h.initiator = true + h.remote = ecies.ImportECDSAPublic(remote) + authMsg, err := h.makeAuthMsg(prv) if err != nil { return s, err } - authPacket, err := sealEIP8(authMsg, h) + authPacket, err := h.sealEIP8(authMsg) if err != nil { return s, err } @@ -495,18 +529,19 @@ func initiatorEncHandshake(conn io.ReadWriter, prv *ecdsa.PrivateKey, remote *ec } authRespMsg := new(authRespV4) - authRespPacket, err := readHandshakeMsg(authRespMsg, encAuthRespLen, prv, conn) + authRespPacket, err := h.readMsg(authRespMsg, prv, conn) if err != nil { return s, err } if err := h.handleAuthResp(authRespMsg); err != nil { return s, err } + return h.secrets(authPacket, authRespPacket) } // makeAuthMsg creates the initiator handshake message. -func (h *encHandshake) makeAuthMsg(prv *ecdsa.PrivateKey) (*authMsgV4, error) { +func (h *handshakeState) makeAuthMsg(prv *ecdsa.PrivateKey) (*authMsgV4, error) { // Generate random initiator nonce. h.initNonce = make([]byte, shaLen) _, err := rand.Read(h.initNonce) @@ -538,13 +573,13 @@ func (h *encHandshake) makeAuthMsg(prv *ecdsa.PrivateKey) (*authMsgV4, error) { return msg, nil } -func (h *encHandshake) handleAuthResp(msg *authRespV4) (err error) { +func (h *handshakeState) handleAuthResp(msg *authRespV4) (err error) { h.respNonce = msg.Nonce[:] h.remoteRandomPub, err = importPublicKey(msg.RandomPubkey[:]) return err } -func (h *encHandshake) makeAuthResp() (msg *authRespV4, err error) { +func (h *handshakeState) makeAuthResp() (msg *authRespV4, err error) { // Generate random nonce. h.respNonce = make([]byte, shaLen) if _, err = rand.Read(h.respNonce); err != nil { @@ -558,81 +593,53 @@ func (h *encHandshake) makeAuthResp() (msg *authRespV4, err error) { return msg, nil } -func (msg *authMsgV4) decodePlain(input []byte) { - n := copy(msg.Signature[:], input) - n += shaLen // skip sha3(initiator-ephemeral-pubk) - n += copy(msg.InitiatorPubkey[:], input[n:]) - copy(msg.Nonce[:], input[n:]) - msg.Version = 4 - msg.gotPlain = true -} +// readMsg reads an encrypted handshake message, decoding it into msg. +func (h *handshakeState) readMsg(msg interface{}, prv *ecdsa.PrivateKey, r io.Reader) ([]byte, error) { + h.rbuf.reset() + h.rbuf.grow(512) -func (msg *authRespV4) sealPlain(hs *encHandshake) ([]byte, error) { - buf := make([]byte, authRespLen) - n := copy(buf, msg.RandomPubkey[:]) - copy(buf[n:], msg.Nonce[:]) - return ecies.Encrypt(rand.Reader, hs.remote, buf, nil, nil) -} + // Read the size prefix. + prefix, err := h.rbuf.read(r, 2) + if err != nil { + return nil, err + } + size := binary.BigEndian.Uint16(prefix) -func (msg *authRespV4) decodePlain(input []byte) { - n := copy(msg.RandomPubkey[:], input) - copy(msg.Nonce[:], input[n:]) - msg.Version = 4 + // Read the handshake packet. + packet, err := h.rbuf.read(r, int(size)) + if err != nil { + return nil, err + } + dec, err := ecies.ImportECDSA(prv).Decrypt(packet, nil, prefix) + if err != nil { + return nil, err + } + // Can't use rlp.DecodeBytes here because it rejects + // trailing data (forward-compatibility). + s := rlp.NewStream(bytes.NewReader(dec), 0) + err = s.Decode(msg) + return h.rbuf.data[:len(prefix)+len(packet)], err } -var padSpace = make([]byte, 300) +// sealEIP8 encrypts a handshake message. +func (h *handshakeState) sealEIP8(msg interface{}) ([]byte, error) { + h.wbuf.reset() -func sealEIP8(msg interface{}, h *encHandshake) ([]byte, error) { - buf := new(bytes.Buffer) - if err := rlp.Encode(buf, msg); err != nil { + // Write the message plaintext. + if err := rlp.Encode(&h.wbuf, msg); err != nil { return nil, err } - // pad with random amount of data. the amount needs to be at least 100 bytes to make + // Pad with random amount of data. the amount needs to be at least 100 bytes to make // the message distinguishable from pre-EIP-8 handshakes. - pad := padSpace[:mrand.Intn(len(padSpace)-100)+100] - buf.Write(pad) + h.wbuf.appendZero(mrand.Intn(100) + 100) + prefix := make([]byte, 2) - binary.BigEndian.PutUint16(prefix, uint16(buf.Len()+eciesOverhead)) + binary.BigEndian.PutUint16(prefix, uint16(len(h.wbuf.data)+eciesOverhead)) - enc, err := ecies.Encrypt(rand.Reader, h.remote, buf.Bytes(), nil, prefix) + enc, err := ecies.Encrypt(rand.Reader, h.remote, h.wbuf.data, nil, prefix) return append(prefix, enc...), err } -type plainDecoder interface { - decodePlain([]byte) -} - -func readHandshakeMsg(msg plainDecoder, plainSize int, prv *ecdsa.PrivateKey, r io.Reader) ([]byte, error) { - buf := make([]byte, plainSize) - if _, err := io.ReadFull(r, buf); err != nil { - return buf, err - } - // Attempt decoding pre-EIP-8 "plain" format. - key := ecies.ImportECDSA(prv) - if dec, err := key.Decrypt(buf, nil, nil); err == nil { - msg.decodePlain(dec) - return buf, nil - } - // Could be EIP-8 format, try that. - prefix := buf[:2] - size := binary.BigEndian.Uint16(prefix) - if size < uint16(plainSize) { - return buf, fmt.Errorf("size underflow, need at least %d bytes", plainSize) - } - buf = append(buf, make([]byte, size-uint16(plainSize)+2)...) - if _, err := io.ReadFull(r, buf[plainSize:]); err != nil { - return buf, err - } - dec, err := key.Decrypt(buf[2:], nil, prefix) - if err != nil { - return buf, err - } - // Can't use rlp.DecodeBytes here because it rejects - // trailing data (forward-compatibility). - s := rlp.NewStream(bytes.NewReader(dec), 0) - return buf, s.Decode(msg) -} - // importPublicKey unmarshals 512 bit public keys. func importPublicKey(pubKey []byte) (*ecies.PublicKey, error) { var pubKey65 []byte diff --git a/p2p/rlpx/rlpx_test.go b/p2p/rlpx/rlpx_test.go index 127a018164..28759f2b49 100644 --- a/p2p/rlpx/rlpx_test.go +++ b/p2p/rlpx/rlpx_test.go @@ -22,6 +22,7 @@ import ( "encoding/hex" "fmt" "io" + "math/rand" "net" "reflect" "strings" @@ -30,6 +31,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/ecies" + "github.com/ethereum/go-ethereum/p2p/simulations/pipes" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" ) @@ -124,7 +126,7 @@ func TestFrameReadWrite(t *testing.T) { IngressMAC: hash, EgressMAC: hash, }) - h := conn.handshake + h := conn.session golden := unhex(` 00828ddae471818bb0bfa6b551d1cb42 @@ -166,27 +168,11 @@ func (h fakeHash) Sum(b []byte) []byte { return append(b, h...) } type handshakeAuthTest struct { input string - isPlain bool wantVersion uint wantRest []rlp.RawValue } var eip8HandshakeAuthTests = []handshakeAuthTest{ - // (Auth₁) RLPx v4 plain encoding - { - input: ` - 048ca79ad18e4b0659fab4853fe5bc58eb83992980f4c9cc147d2aa31532efd29a3d3dc6a3d89eaf - 913150cfc777ce0ce4af2758bf4810235f6e6ceccfee1acc6b22c005e9e3a49d6448610a58e98744 - ba3ac0399e82692d67c1f58849050b3024e21a52c9d3b01d871ff5f210817912773e610443a9ef14 - 2e91cdba0bd77b5fdf0769b05671fc35f83d83e4d3b0b000c6b2a1b1bba89e0fc51bf4e460df3105 - c444f14be226458940d6061c296350937ffd5e3acaceeaaefd3c6f74be8e23e0f45163cc7ebd7622 - 0f0128410fd05250273156d548a414444ae2f7dea4dfca2d43c057adb701a715bf59f6fb66b2d1d2 - 0f2c703f851cbf5ac47396d9ca65b6260bd141ac4d53e2de585a73d1750780db4c9ee4cd4d225173 - a4592ee77e2bd94d0be3691f3b406f9bba9b591fc63facc016bfa8 - `, - isPlain: true, - wantVersion: 4, - }, // (Auth₂) EIP-8 encoding { input: ` @@ -233,18 +219,6 @@ type handshakeAckTest struct { } var eip8HandshakeRespTests = []handshakeAckTest{ - // (Ack₁) RLPx v4 plain encoding - { - input: ` - 049f8abcfa9c0dc65b982e98af921bc0ba6e4243169348a236abe9df5f93aa69d99cadddaa387662 - b0ff2c08e9006d5a11a278b1b3331e5aaabf0a32f01281b6f4ede0e09a2d5f585b26513cb794d963 - 5a57563921c04a9090b4f14ee42be1a5461049af4ea7a7f49bf4c97a352d39c8d02ee4acc416388c - 1c66cec761d2bc1c72da6ba143477f049c9d2dde846c252c111b904f630ac98e51609b3b1f58168d - dca6505b7196532e5f85b259a20c45e1979491683fee108e9660edbf38f3add489ae73e3dda2c71b - d1497113d5c755e942d1 - `, - wantVersion: 4, - }, // (Ack₂) EIP-8 encoding { input: ` @@ -287,10 +261,13 @@ var eip8HandshakeRespTests = []handshakeAckTest{ }, } +var ( + keyA, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + keyB, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") +) + func TestHandshakeForwardCompatibility(t *testing.T) { var ( - keyA, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - keyB, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") pubA = crypto.FromECDSAPub(&keyA.PublicKey)[1:] pubB = crypto.FromECDSAPub(&keyB.PublicKey)[1:] ephA, _ = crypto.HexToECDSA("869d6ecf5211f1cc60418a13b9d870b22959d0c16f02bec714c960dd2298a32d") @@ -304,7 +281,7 @@ func TestHandshakeForwardCompatibility(t *testing.T) { _ = authSignature ) makeAuth := func(test handshakeAuthTest) *authMsgV4 { - msg := &authMsgV4{Version: test.wantVersion, Rest: test.wantRest, gotPlain: test.isPlain} + msg := &authMsgV4{Version: test.wantVersion, Rest: test.wantRest} copy(msg.Signature[:], authSignature) copy(msg.InitiatorPubkey[:], pubA) copy(msg.Nonce[:], nonceA) @@ -319,9 +296,10 @@ func TestHandshakeForwardCompatibility(t *testing.T) { // check auth msg parsing for _, test := range eip8HandshakeAuthTests { + var h handshakeState r := bytes.NewReader(unhex(test.input)) msg := new(authMsgV4) - ciphertext, err := readHandshakeMsg(msg, encAuthMsgLen, keyB, r) + ciphertext, err := h.readMsg(msg, keyB, r) if err != nil { t.Errorf("error for input %x:\n %v", unhex(test.input), err) continue @@ -337,10 +315,11 @@ func TestHandshakeForwardCompatibility(t *testing.T) { // check auth resp parsing for _, test := range eip8HandshakeRespTests { + var h handshakeState input := unhex(test.input) r := bytes.NewReader(input) msg := new(authRespV4) - ciphertext, err := readHandshakeMsg(msg, encAuthRespLen, keyA, r) + ciphertext, err := h.readMsg(msg, keyA, r) if err != nil { t.Errorf("error for input %x:\n %v", input, err) continue @@ -356,14 +335,14 @@ func TestHandshakeForwardCompatibility(t *testing.T) { // check derivation for (Auth₂, Ack₂) on recipient side var ( - hs = &encHandshake{ + hs = &handshakeState{ initiator: false, respNonce: nonceB, randomPrivKey: ecies.ImportECDSA(ephB), } - authCiphertext = unhex(eip8HandshakeAuthTests[1].input) - authRespCiphertext = unhex(eip8HandshakeRespTests[1].input) - authMsg = makeAuth(eip8HandshakeAuthTests[1]) + authCiphertext = unhex(eip8HandshakeAuthTests[0].input) + authRespCiphertext = unhex(eip8HandshakeRespTests[0].input) + authMsg = makeAuth(eip8HandshakeAuthTests[0]) wantAES = unhex("80e8632c05fed6fc2a13b0f8d31a3cf645366239170ea067065aba8e28bac487") wantMAC = unhex("2ea74ec5dae199227dff1af715362700e989d889d7a493cb0639691efb8e5f98") wantFooIngressHash = unhex("0c7ec6340062cc46f5e9f1e3cf86f8c8c403c5a0964f5df0ebd34a75ddc86db5") @@ -388,6 +367,74 @@ func TestHandshakeForwardCompatibility(t *testing.T) { } } +func BenchmarkHandshakeRead(b *testing.B) { + var input = unhex(eip8HandshakeAuthTests[0].input) + + for i := 0; i < b.N; i++ { + var ( + h handshakeState + r = bytes.NewReader(input) + msg = new(authMsgV4) + ) + if _, err := h.readMsg(msg, keyB, r); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkThroughput(b *testing.B) { + pipe1, pipe2, err := pipes.TCPPipe() + if err != nil { + b.Fatal(err) + } + + var ( + conn1, conn2 = NewConn(pipe1, nil), NewConn(pipe2, &keyA.PublicKey) + handshakeDone = make(chan error, 1) + msgdata = make([]byte, 1024) + rand = rand.New(rand.NewSource(1337)) + ) + rand.Read(msgdata) + + // Server side. + go func() { + defer conn1.Close() + // Perform handshake. + _, err := conn1.Handshake(keyA) + handshakeDone <- err + if err != nil { + return + } + conn1.SetSnappy(true) + // Keep sending messages until connection closed. + for { + if _, err := conn1.Write(0, msgdata); err != nil { + return + } + } + }() + + // Set up client side. + defer conn2.Close() + if _, err := conn2.Handshake(keyB); err != nil { + b.Fatal("client handshake error:", err) + } + conn2.SetSnappy(true) + if err := <-handshakeDone; err != nil { + b.Fatal("server hanshake error:", err) + } + + // Read N messages. + b.SetBytes(int64(len(msgdata))) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _, _, _, err := conn2.Read() + if err != nil { + b.Fatal("read error:", err) + } + } +} + func unhex(str string) []byte { r := strings.NewReplacer("\t", "", " ", "", "\n", "") b, err := hex.DecodeString(r.Replace(str)) diff --git a/p2p/transport.go b/p2p/transport.go index 3f1cd7d64f..d594259866 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/rlpx" @@ -62,6 +63,10 @@ func (t *rlpxTransport) ReadMsg() (Msg, error) { t.conn.SetReadDeadline(time.Now().Add(frameReadTimeout)) code, data, wireSize, err := t.conn.Read() if err == nil { + // Protocol messages are dispatched to subprotocol handlers asynchronously, + // but package rlpx may reuse the returned 'data' buffer on the next call + // to Read. Copy the message data to avoid this being an issue. + data = common.CopyBytes(data) msg = Msg{ ReceivedAt: time.Now(), Code: code, diff --git a/rlp/raw.go b/rlp/raw.go index 3071e99cab..f355efc144 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -34,6 +34,14 @@ func ListSize(contentSize uint64) uint64 { return uint64(headsize(contentSize)) + contentSize } +// IntSize returns the encoded size of the integer x. +func IntSize(x uint64) int { + if x < 0x80 { + return 1 + } + return 1 + intsize(x) +} + // Split returns the content of first RLP value and any // bytes after the value as subslices of b. func Split(b []byte) (k Kind, content, rest []byte, err error) { diff --git a/rlp/raw_test.go b/rlp/raw_test.go index c976c4f734..185e269d07 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -263,6 +263,12 @@ func TestAppendUint64(t *testing.T) { if !bytes.Equal(x, unhex(test.output)) { t.Errorf("AppendUint64(%v, %d): got %x, want %s", test.slice, test.input, x, test.output) } + + // Check that IntSize returns the appended size. + length := len(x) - len(test.slice) + if s := IntSize(test.input); s != length { + t.Errorf("IntSize(%d): got %d, want %d", test.input, s, length) + } } } From d836ad141ef4842be494cd5156fc6bd25a13e463 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 27 May 2021 11:57:49 +0200 Subject: [PATCH 395/709] cmd/devp2p/internal/ethtest: add block hash announcement test (#22535) --- cmd/devp2p/internal/ethtest/helpers.go | 103 +++++++++++++++++++++++++ cmd/devp2p/internal/ethtest/suite.go | 20 +++++ 2 files changed, 123 insertions(+) diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index d99376124d..a9a213f337 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -633,3 +633,106 @@ func (s *Suite) maliciousStatus(conn *Conn) error { return fmt.Errorf("expected disconnect, got: %s", pretty.Sdump(msg)) } } + +func (s *Suite) hashAnnounce(isEth66 bool) error { + // create connections + sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + if err != nil { + return fmt.Errorf("failed to create connections: %v", err) + } + defer sendConn.Close() + defer recvConn.Close() + if err := sendConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + if err := recvConn.peer(s.chain, nil); err != nil { + return fmt.Errorf("peering failed: %v", err) + } + // create NewBlockHashes announcement + nextBlock := s.fullChain.blocks[s.chain.Len()] + newBlockHash := &NewBlockHashes{ + {Hash: nextBlock.Hash(), Number: nextBlock.Number().Uint64()}, + } + + if err := sendConn.Write(newBlockHash); err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + if isEth66 { + // expect GetBlockHeaders request, and respond + id, msg := sendConn.Read66() + switch msg := msg.(type) { + case GetBlockHeaders: + blockHeaderReq := msg + if blockHeaderReq.Amount != 1 { + return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) + } + if blockHeaderReq.Origin.Hash != nextBlock.Hash() { + return fmt.Errorf("unexpected block header requested: %v", pretty.Sdump(blockHeaderReq)) + } + resp := ð.BlockHeadersPacket66{ + RequestId: id, + BlockHeadersPacket: eth.BlockHeadersPacket{ + nextBlock.Header(), + }, + } + if err := sendConn.Write66(resp, BlockHeaders{}.Code()); err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + default: + return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) + } + } else { + // expect GetBlockHeaders request, and respond + switch msg := sendConn.Read().(type) { + case *GetBlockHeaders: + blockHeaderReq := *msg + if blockHeaderReq.Amount != 1 { + return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) + } + if blockHeaderReq.Origin.Hash != nextBlock.Hash() { + return fmt.Errorf("unexpected block header requested: %v", pretty.Sdump(blockHeaderReq)) + } + if err := sendConn.Write(&BlockHeaders{nextBlock.Header()}); err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + default: + return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) + } + } + // wait for block announcement + msg := recvConn.readAndServe(s.chain, timeout) + switch msg := msg.(type) { + case *NewBlockHashes: + hashes := *msg + if len(hashes) != 1 { + return fmt.Errorf("unexpected new block hash announcement: wanted 1 announcement, got %d", len(hashes)) + } + if nextBlock.Hash() != hashes[0].Hash { + return fmt.Errorf("unexpected block hash announcement, wanted %v, got %v", nextBlock.Hash(), + hashes[0].Hash) + } + case *NewBlock: + // node should only propagate NewBlock without having requested the body if the body is empty + nextBlockBody := nextBlock.Body() + if len(nextBlockBody.Transactions) != 0 || len(nextBlockBody.Uncles) != 0 { + return fmt.Errorf("unexpected non-empty new block propagated: %s", pretty.Sdump(msg)) + } + if msg.Block.Hash() != nextBlock.Hash() { + return fmt.Errorf("mismatched hash of propagated new block: wanted %v, got %v", + nextBlock.Hash(), msg.Block.Hash()) + } + // check to make sure header matches header that was sent to the node + if !reflect.DeepEqual(nextBlock.Header(), msg.Block.Header()) { + return fmt.Errorf("incorrect header received: wanted %v, got %v", nextBlock.Header(), msg.Block.Header()) + } + default: + return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) + } + // confirm node imported block + if err := s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + return fmt.Errorf("error waiting for node to import new block: %v", err) + } + // update the chain + s.chain.blocks = append(s.chain.blocks, nextBlock) + return nil +} diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index ad832dddd2..491bcda7e7 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -70,6 +70,8 @@ func (s *Suite) AllEthTests() []utesting.Test { {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, + {Name: "TestBlockHashAnnounce", Fn: s.TestBlockHashAnnounce}, + {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, // malicious handshakes + status {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, @@ -93,6 +95,7 @@ func (s *Suite) EthTests() []utesting.Test { {Name: "TestBroadcast", Fn: s.TestBroadcast}, {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, + {Name: "TestBlockHashAnnounce", Fn: s.TestBlockHashAnnounce}, {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, {Name: "TestTransaction", Fn: s.TestTransaction}, @@ -112,6 +115,7 @@ func (s *Suite) Eth66Tests() []utesting.Test { {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, + {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, {Name: "TestTransaction66", Fn: s.TestTransaction66}, @@ -580,6 +584,22 @@ func (s *Suite) TestOldAnnounce66(t *utesting.T) { } } +// TestBlockHashAnnounce sends a new block hash announcement and expects +// the node to perform a `GetBlockHeaders` request. +func (s *Suite) TestBlockHashAnnounce(t *utesting.T) { + if err := s.hashAnnounce(eth65); err != nil { + t.Fatalf("block hash announcement failed: %v", err) + } +} + +// TestBlockHashAnnounce66 sends a new block hash announcement and expects +// the node to perform a `GetBlockHeaders` request. +func (s *Suite) TestBlockHashAnnounce66(t *utesting.T) { + if err := s.hashAnnounce(eth66); err != nil { + t.Fatalf("block hash announcement failed: %v", err) + } +} + // TestMaliciousHandshake tries to send malicious data during the handshake. func (s *Suite) TestMaliciousHandshake(t *utesting.T) { if err := s.maliciousHandshakes(t, eth65); err != nil { From 0703ef62d388eafa177540ff722c3a0871c4979d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 27 May 2021 13:30:25 +0200 Subject: [PATCH 396/709] crypto/secp256k1: fix undefined behavior in BitCurve.Add (#22621) This commit changes the behavior of BitCurve.Add to be more inline with btcd. It fixes two different bugs: 1) When adding a point at infinity to another point, the other point should be returned. While this is undefined behavior, it is better to be more inline with the go standard library. Thus (0,0) + (a, b) = (a,b) 2) Adding the same point to itself produced the point at infinity. This is incorrect, now doubleJacobian is used to correctly calculate it. Thus (a,b) + (a,b) == 2* (a,b) and not (0,0) anymore. The change also adds a differential fuzzer for Add, testing it against btcd. Co-authored-by: Felix Lange --- crypto/secp256k1/curve.go | 53 +++++------------------- crypto/secp256k1/panic_cb.go | 2 + crypto/secp256k1/scalar_mult_cgo.go | 56 ++++++++++++++++++++++++++ crypto/secp256k1/scalar_mult_nocgo.go | 13 ++++++ crypto/secp256k1/secp256.go | 2 + oss-fuzz.sh | 1 + tests/fuzzers/secp256k1/secp_fuzzer.go | 50 +++++++++++++++++++++++ tests/fuzzers/secp256k1/secp_test.go | 8 ++++ 8 files changed, 143 insertions(+), 42 deletions(-) create mode 100644 crypto/secp256k1/scalar_mult_cgo.go create mode 100644 crypto/secp256k1/scalar_mult_nocgo.go create mode 100644 tests/fuzzers/secp256k1/secp_fuzzer.go create mode 100644 tests/fuzzers/secp256k1/secp_test.go diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index 8f83cccad9..fa1b199a34 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -35,15 +35,8 @@ package secp256k1 import ( "crypto/elliptic" "math/big" - "unsafe" ) -/* -#include "libsecp256k1/include/secp256k1.h" -extern int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, const unsigned char *point, const unsigned char *scalar); -*/ -import "C" - const ( // number of bits in a big.Word wordBits = 32 << (uint64(^big.Word(0)) >> 63) @@ -133,7 +126,18 @@ func (BitCurve *BitCurve) affineFromJacobian(x, y, z *big.Int) (xOut, yOut *big. // Add returns the sum of (x1,y1) and (x2,y2) func (BitCurve *BitCurve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) { + // If one point is at infinity, return the other point. + // Adding the point at infinity to any point will preserve the other point. + if x1.Sign() == 0 && y1.Sign() == 0 { + return x2, y2 + } + if x2.Sign() == 0 && y2.Sign() == 0 { + return x1, y1 + } z := new(big.Int).SetInt64(1) + if x1.Cmp(x2) == 0 && y1.Cmp(y2) == 0 { + return BitCurve.affineFromJacobian(BitCurve.doubleJacobian(x1, y1, z)) + } return BitCurve.affineFromJacobian(BitCurve.addJacobian(x1, y1, z, x2, y2, z)) } @@ -242,41 +246,6 @@ func (BitCurve *BitCurve) doubleJacobian(x, y, z *big.Int) (*big.Int, *big.Int, return x3, y3, z3 } -func (BitCurve *BitCurve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int, *big.Int) { - // Ensure scalar is exactly 32 bytes. We pad always, even if - // scalar is 32 bytes long, to avoid a timing side channel. - if len(scalar) > 32 { - panic("can't handle scalars > 256 bits") - } - // NOTE: potential timing issue - padded := make([]byte, 32) - copy(padded[32-len(scalar):], scalar) - scalar = padded - - // Do the multiplication in C, updating point. - point := make([]byte, 64) - readBits(Bx, point[:32]) - readBits(By, point[32:]) - - pointPtr := (*C.uchar)(unsafe.Pointer(&point[0])) - scalarPtr := (*C.uchar)(unsafe.Pointer(&scalar[0])) - res := C.secp256k1_ext_scalar_mul(context, pointPtr, scalarPtr) - - // Unpack the result and clear temporaries. - x := new(big.Int).SetBytes(point[:32]) - y := new(big.Int).SetBytes(point[32:]) - for i := range point { - point[i] = 0 - } - for i := range padded { - scalar[i] = 0 - } - if res != 1 { - return nil, nil - } - return x, y -} - // ScalarBaseMult returns k*G, where G is the base point of the group and k is // an integer in big-endian form. func (BitCurve *BitCurve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) { diff --git a/crypto/secp256k1/panic_cb.go b/crypto/secp256k1/panic_cb.go index 6d59a1d247..262846fd89 100644 --- a/crypto/secp256k1/panic_cb.go +++ b/crypto/secp256k1/panic_cb.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. +// +build !gofuzz cgo + package secp256k1 import "C" diff --git a/crypto/secp256k1/scalar_mult_cgo.go b/crypto/secp256k1/scalar_mult_cgo.go new file mode 100644 index 0000000000..34998ad1a4 --- /dev/null +++ b/crypto/secp256k1/scalar_mult_cgo.go @@ -0,0 +1,56 @@ +// Copyright 2015 Jeffrey Wilcke, Felix Lange, Gustav Simonsson. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build !gofuzz cgo + +package secp256k1 + +import ( + "math/big" + "unsafe" +) + +/* + +#include "libsecp256k1/include/secp256k1.h" + +extern int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, const unsigned char *point, const unsigned char *scalar); + +*/ +import "C" + +func (BitCurve *BitCurve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int, *big.Int) { + // Ensure scalar is exactly 32 bytes. We pad always, even if + // scalar is 32 bytes long, to avoid a timing side channel. + if len(scalar) > 32 { + panic("can't handle scalars > 256 bits") + } + // NOTE: potential timing issue + padded := make([]byte, 32) + copy(padded[32-len(scalar):], scalar) + scalar = padded + + // Do the multiplication in C, updating point. + point := make([]byte, 64) + readBits(Bx, point[:32]) + readBits(By, point[32:]) + + pointPtr := (*C.uchar)(unsafe.Pointer(&point[0])) + scalarPtr := (*C.uchar)(unsafe.Pointer(&scalar[0])) + res := C.secp256k1_ext_scalar_mul(context, pointPtr, scalarPtr) + + // Unpack the result and clear temporaries. + x := new(big.Int).SetBytes(point[:32]) + y := new(big.Int).SetBytes(point[32:]) + for i := range point { + point[i] = 0 + } + for i := range padded { + scalar[i] = 0 + } + if res != 1 { + return nil, nil + } + return x, y +} diff --git a/crypto/secp256k1/scalar_mult_nocgo.go b/crypto/secp256k1/scalar_mult_nocgo.go new file mode 100644 index 0000000000..55756b5be8 --- /dev/null +++ b/crypto/secp256k1/scalar_mult_nocgo.go @@ -0,0 +1,13 @@ +// Copyright 2015 Jeffrey Wilcke, Felix Lange, Gustav Simonsson. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build gofuzz !cgo + +package secp256k1 + +import "math/big" + +func (BitCurve *BitCurve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int, *big.Int) { + panic("ScalarMult is not available when secp256k1 is built without cgo") +} diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index 9a7c06d7ce..9e942ac6fe 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. +// +build !gofuzz cgo + // Package secp256k1 wraps the bitcoin secp256k1 C library. package secp256k1 diff --git a/oss-fuzz.sh b/oss-fuzz.sh index a9bac03257..081a8e1d5a 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -102,6 +102,7 @@ compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty compile_fuzzer tests/fuzzers/abi Fuzz fuzzAbi compile_fuzzer tests/fuzzers/les Fuzz fuzzLes +compile_fuzzer tests/fuzzers/secp265k1 Fuzz fuzzSecp256k1 compile_fuzzer tests/fuzzers/vflux FuzzClientPool fuzzClientPool compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add diff --git a/tests/fuzzers/secp256k1/secp_fuzzer.go b/tests/fuzzers/secp256k1/secp_fuzzer.go new file mode 100644 index 0000000000..53845b6433 --- /dev/null +++ b/tests/fuzzers/secp256k1/secp_fuzzer.go @@ -0,0 +1,50 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// build +gofuzz + +package secp256k1 + +import ( + "fmt" + + "github.com/btcsuite/btcd/btcec" + "github.com/ethereum/go-ethereum/crypto/secp256k1" + fuzz "github.com/google/gofuzz" +) + +func Fuzz(input []byte) int { + var ( + fuzzer = fuzz.NewFromGoFuzz(input) + curveA = secp256k1.S256() + curveB = btcec.S256() + dataP1 []byte + dataP2 []byte + ) + // first point + fuzzer.Fuzz(&dataP1) + x1, y1 := curveB.ScalarBaseMult(dataP1) + // second point + fuzzer.Fuzz(&dataP2) + x2, y2 := curveB.ScalarBaseMult(dataP2) + resAX, resAY := curveA.Add(x1, y1, x2, y2) + resBX, resBY := curveB.Add(x1, y1, x2, y2) + if resAX.Cmp(resBX) != 0 || resAY.Cmp(resBY) != 0 { + fmt.Printf("%s %s %s %s\n", x1, y1, x2, y2) + panic(fmt.Sprintf("Addition failed: geth: %s %s btcd: %s %s", resAX, resAY, resBX, resBY)) + } + return 0 +} diff --git a/tests/fuzzers/secp256k1/secp_test.go b/tests/fuzzers/secp256k1/secp_test.go new file mode 100644 index 0000000000..76bae87086 --- /dev/null +++ b/tests/fuzzers/secp256k1/secp_test.go @@ -0,0 +1,8 @@ +package secp256k1 + +import "testing" + +func TestFuzzer(t *testing.T) { + test := "00000000N0000000/R00000000000000000U0000S0000000mkhP000000000000000U" + Fuzz([]byte(test)) +} From 427175153c0e33a7f640f86bf2ce4c97f03ede72 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 27 May 2021 18:43:55 +0200 Subject: [PATCH 397/709] p2p/msgrate: return capacity as integer, clamp to max uint32 (#22943) * p2p/msgrate: return capacity as integer * eth/protocols/snap: remove conversions * p2p/msgrate: add overflow test * p2p/msgrate: make the capacity overflow test actually overflow * p2p/msgrate: clamp capacity to max int32 * p2p/msgrate: fix min/max confusion --- eth/downloader/peer.go | 48 +++++++++++++++++++------------------ eth/protocols/snap/sync.go | 30 +++++++++++------------ p2p/msgrate/msgrate.go | 14 ++++++++--- p2p/msgrate/msgrate_test.go | 28 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 41 deletions(-) create mode 100644 p2p/msgrate/msgrate_test.go diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index b9c7716941..066a366315 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -21,7 +21,6 @@ package downloader import ( "errors" - "math" "math/big" "sort" "sync" @@ -232,7 +231,7 @@ func (p *peerConnection) SetNodeDataIdle(delivered int, deliveryTime time.Time) // HeaderCapacity retrieves the peers header download allowance based on its // previously discovered throughput. func (p *peerConnection) HeaderCapacity(targetRTT time.Duration) int { - cap := int(math.Ceil(p.rates.Capacity(eth.BlockHeadersMsg, targetRTT))) + cap := p.rates.Capacity(eth.BlockHeadersMsg, targetRTT) if cap > MaxHeaderFetch { cap = MaxHeaderFetch } @@ -242,7 +241,7 @@ func (p *peerConnection) HeaderCapacity(targetRTT time.Duration) int { // BlockCapacity retrieves the peers block download allowance based on its // previously discovered throughput. func (p *peerConnection) BlockCapacity(targetRTT time.Duration) int { - cap := int(math.Ceil(p.rates.Capacity(eth.BlockBodiesMsg, targetRTT))) + cap := p.rates.Capacity(eth.BlockBodiesMsg, targetRTT) if cap > MaxBlockFetch { cap = MaxBlockFetch } @@ -252,7 +251,7 @@ func (p *peerConnection) BlockCapacity(targetRTT time.Duration) int { // ReceiptCapacity retrieves the peers receipt download allowance based on its // previously discovered throughput. func (p *peerConnection) ReceiptCapacity(targetRTT time.Duration) int { - cap := int(math.Ceil(p.rates.Capacity(eth.ReceiptsMsg, targetRTT))) + cap := p.rates.Capacity(eth.ReceiptsMsg, targetRTT) if cap > MaxReceiptFetch { cap = MaxReceiptFetch } @@ -262,7 +261,7 @@ func (p *peerConnection) ReceiptCapacity(targetRTT time.Duration) int { // NodeDataCapacity retrieves the peers state download allowance based on its // previously discovered throughput. func (p *peerConnection) NodeDataCapacity(targetRTT time.Duration) int { - cap := int(math.Ceil(p.rates.Capacity(eth.NodeDataMsg, targetRTT))) + cap := p.rates.Capacity(eth.NodeDataMsg, targetRTT) if cap > MaxStateFetch { cap = MaxStateFetch } @@ -411,7 +410,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.headerIdle) == 0 } - throughput := func(p *peerConnection) float64 { + throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) @@ -423,7 +422,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.blockIdle) == 0 } - throughput := func(p *peerConnection) float64 { + throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) @@ -435,7 +434,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.receiptIdle) == 0 } - throughput := func(p *peerConnection) float64 { + throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) @@ -447,7 +446,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.stateIdle) == 0 } - throughput := func(p *peerConnection) float64 { + throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) @@ -455,45 +454,48 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { // idlePeers retrieves a flat list of all currently idle peers satisfying the // protocol version constraints, using the provided function to check idleness. -// The resulting set of peers are sorted by their measure throughput. -func (ps *peerSet) idlePeers(minProtocol, maxProtocol uint, idleCheck func(*peerConnection) bool, throughput func(*peerConnection) float64) ([]*peerConnection, int) { +// The resulting set of peers are sorted by their capacity. +func (ps *peerSet) idlePeers(minProtocol, maxProtocol uint, idleCheck func(*peerConnection) bool, capacity func(*peerConnection) int) ([]*peerConnection, int) { ps.lock.RLock() defer ps.lock.RUnlock() - idle, total := make([]*peerConnection, 0, len(ps.peers)), 0 - tps := make([]float64, 0, len(ps.peers)) + var ( + total = 0 + idle = make([]*peerConnection, 0, len(ps.peers)) + tps = make([]int, 0, len(ps.peers)) + ) for _, p := range ps.peers { if p.version >= minProtocol && p.version <= maxProtocol { if idleCheck(p) { idle = append(idle, p) - tps = append(tps, throughput(p)) + tps = append(tps, capacity(p)) } total++ } } + // And sort them - sortPeers := &peerThroughputSort{idle, tps} + sortPeers := &peerCapacitySort{idle, tps} sort.Sort(sortPeers) return sortPeers.p, total } -// peerThroughputSort implements the Sort interface, and allows for -// sorting a set of peers by their throughput -// The sorted data is with the _highest_ throughput first -type peerThroughputSort struct { +// peerCapacitySort implements sort.Interface. +// It sorts peer connections by capacity (descending). +type peerCapacitySort struct { p []*peerConnection - tp []float64 + tp []int } -func (ps *peerThroughputSort) Len() int { +func (ps *peerCapacitySort) Len() int { return len(ps.p) } -func (ps *peerThroughputSort) Less(i, j int) bool { +func (ps *peerCapacitySort) Less(i, j int) bool { return ps.tp[i] > ps.tp[j] } -func (ps *peerThroughputSort) Swap(i, j int) { +func (ps *peerCapacitySort) Swap(i, j int) { ps.p[i], ps.p[j] = ps.p[j], ps.p[i] ps.tp[i], ps.tp[j] = ps.tp[j], ps.tp[i] } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index c57fcd71f6..646df03887 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -861,7 +861,7 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac // Sort the peers by download capacity to use faster ones if many available idlers := &capacitySort{ ids: make([]string, 0, len(s.accountIdlers)), - caps: make([]float64, 0, len(s.accountIdlers)), + caps: make([]int, 0, len(s.accountIdlers)), } targetTTL := s.rates.TargetTimeout() for id := range s.accountIdlers { @@ -958,7 +958,7 @@ func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan * // Sort the peers by download capacity to use faster ones if many available idlers := &capacitySort{ ids: make([]string, 0, len(s.bytecodeIdlers)), - caps: make([]float64, 0, len(s.bytecodeIdlers)), + caps: make([]int, 0, len(s.bytecodeIdlers)), } targetTTL := s.rates.TargetTimeout() for id := range s.bytecodeIdlers { @@ -1012,11 +1012,11 @@ func (s *Syncer) assignBytecodeTasks(success chan *bytecodeResponse, fail chan * if cap > maxCodeRequestCount { cap = maxCodeRequestCount } - hashes := make([]common.Hash, 0, int(cap)) + hashes := make([]common.Hash, 0, cap) for hash := range task.codeTasks { delete(task.codeTasks, hash) hashes = append(hashes, hash) - if len(hashes) >= int(cap) { + if len(hashes) >= cap { break } } @@ -1061,7 +1061,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st // Sort the peers by download capacity to use faster ones if many available idlers := &capacitySort{ ids: make([]string, 0, len(s.storageIdlers)), - caps: make([]float64, 0, len(s.storageIdlers)), + caps: make([]int, 0, len(s.storageIdlers)), } targetTTL := s.rates.TargetTimeout() for id := range s.storageIdlers { @@ -1120,7 +1120,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st if cap < minRequestSize { // Don't bother with peers below a bare minimum performance cap = minRequestSize } - storageSets := int(cap / 1024) + storageSets := cap / 1024 var ( accounts = make([]common.Hash, 0, storageSets) @@ -1217,7 +1217,7 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai // Sort the peers by download capacity to use faster ones if many available idlers := &capacitySort{ ids: make([]string, 0, len(s.trienodeHealIdlers)), - caps: make([]float64, 0, len(s.trienodeHealIdlers)), + caps: make([]int, 0, len(s.trienodeHealIdlers)), } targetTTL := s.rates.TargetTimeout() for id := range s.trienodeHealIdlers { @@ -1284,9 +1284,9 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai cap = maxTrieRequestCount } var ( - hashes = make([]common.Hash, 0, int(cap)) - paths = make([]trie.SyncPath, 0, int(cap)) - pathsets = make([]TrieNodePathSet, 0, int(cap)) + hashes = make([]common.Hash, 0, cap) + paths = make([]trie.SyncPath, 0, cap) + pathsets = make([]TrieNodePathSet, 0, cap) ) for hash, pathset := range s.healer.trieTasks { delete(s.healer.trieTasks, hash) @@ -1295,7 +1295,7 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai paths = append(paths, pathset) pathsets = append(pathsets, [][]byte(pathset)) // TODO(karalabe): group requests by account hash - if len(hashes) >= int(cap) { + if len(hashes) >= cap { break } } @@ -1341,7 +1341,7 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai // Sort the peers by download capacity to use faster ones if many available idlers := &capacitySort{ ids: make([]string, 0, len(s.bytecodeHealIdlers)), - caps: make([]float64, 0, len(s.bytecodeHealIdlers)), + caps: make([]int, 0, len(s.bytecodeHealIdlers)), } targetTTL := s.rates.TargetTimeout() for id := range s.bytecodeHealIdlers { @@ -1407,12 +1407,12 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai if cap > maxCodeRequestCount { cap = maxCodeRequestCount } - hashes := make([]common.Hash, 0, int(cap)) + hashes := make([]common.Hash, 0, cap) for hash := range s.healer.codeTasks { delete(s.healer.codeTasks, hash) hashes = append(hashes, hash) - if len(hashes) >= int(cap) { + if len(hashes) >= cap { break } } @@ -2852,7 +2852,7 @@ func estimateRemainingSlots(hashes int, last common.Hash) (uint64, error) { // of highest capacity being at the front. type capacitySort struct { ids []string - caps []float64 + caps []int } func (s *capacitySort) Len() int { diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 7cd172c566..5bfa27b433 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -19,6 +19,7 @@ package msgrate import ( "errors" + "math" "sort" "sync" "time" @@ -162,7 +163,7 @@ func NewTracker(caps map[uint64]float64, rtt time.Duration) *Tracker { // the load proportionally to the requested items, so fetching a bit more might // still take the same RTT. By forcefully overshooting by a small amount, we can // avoid locking into a lower-that-real capacity. -func (t *Tracker) Capacity(kind uint64, targetRTT time.Duration) float64 { +func (t *Tracker) Capacity(kind uint64, targetRTT time.Duration) int { t.lock.RLock() defer t.lock.RUnlock() @@ -171,7 +172,14 @@ func (t *Tracker) Capacity(kind uint64, targetRTT time.Duration) float64 { // Return an overestimation to force the peer out of a stuck minima, adding // +1 in case the item count is too low for the overestimator to dent - return 1 + capacityOverestimation*throughput + return roundCapacity(1 + capacityOverestimation*throughput) +} + +// roundCapacity gives the integer value of a capacity. +// The result fits int32, and is guaranteed to be positive. +func roundCapacity(cap float64) int { + const maxInt32 = float64(1<<31 - 1) + return int(math.Min(maxInt32, math.Max(1, math.Ceil(cap)))) } // Update modifies the peer's capacity values for a specific data type with a new @@ -435,7 +443,7 @@ func (t *Trackers) detune() { // Capacity is a helper function to access a specific tracker without having to // track it explicitly outside. -func (t *Trackers) Capacity(id string, kind uint64, targetRTT time.Duration) float64 { +func (t *Trackers) Capacity(id string, kind uint64, targetRTT time.Duration) int { t.lock.RLock() defer t.lock.RUnlock() diff --git a/p2p/msgrate/msgrate_test.go b/p2p/msgrate/msgrate_test.go new file mode 100644 index 0000000000..a5c8dd0518 --- /dev/null +++ b/p2p/msgrate/msgrate_test.go @@ -0,0 +1,28 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package msgrate + +import "testing" + +func TestCapacityOverflow(t *testing.T) { + tracker := NewTracker(nil, 1) + tracker.Update(1, 1, 100000) + cap := tracker.Capacity(1, 10000000) + if int32(cap) < 0 { + t.Fatalf("Negative: %v", int32(cap)) + } +} From 04cb5e2be30e1aa6c0cca657e10fec7239a6334f Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 27 May 2021 18:45:13 +0200 Subject: [PATCH 398/709] cmd/puppeth: remove outdated mist support (#22940) --- cmd/puppeth/module_dashboard.go | 64 +--------- cmd/puppeth/module_wallet.go | 201 -------------------------------- cmd/puppeth/wizard_dashboard.go | 8 +- cmd/puppeth/wizard_netstats.go | 8 -- cmd/puppeth/wizard_network.go | 9 +- cmd/puppeth/wizard_wallet.go | 113 ------------------ 6 files changed, 5 insertions(+), 398 deletions(-) delete mode 100644 cmd/puppeth/module_wallet.go delete mode 100644 cmd/puppeth/wizard_wallet.go diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index a76ee19a06..b238af0316 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -80,12 +80,10 @@ var dashboardContent = `

    {{if .EthstatsPage}}
  • Network Stats
    • {{end}} {{if .ExplorerPage}}
  • Block Explorer
    • {{end}} - {{if .WalletPage}}
  • Browser Wallet
    • {{end}} {{if .FaucetPage}}
  • Crypto Faucet
    • {{end}}
  • Connect Yourself @@ -186,58 +184,6 @@ var dashboardContent = ` -