From cccd675148f1c46f80ff606610dc6eb6f9e9b101 Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Wed, 24 Nov 2021 10:47:37 +0800 Subject: [PATCH 1/8] eth/protocols/snap, trie: better error-handling (#23657) (#582) Co-authored-by: Martin Holst Swende --- eth/protocols/snap/handler.go | 2 +- trie/trie.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 3d668a2ebb6f1..d9935f455f39f 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -469,7 +469,7 @@ func handleMessage(backend Backend, peer *Peer) error { // Storage slots requested, open the storage trie and retrieve from there account, err := snap.Account(common.BytesToHash(pathset[0])) loads++ // always account database reads, even for failures - if err != nil { + if err != nil || account == nil { break } stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) diff --git a/trie/trie.go b/trie/trie.go index 44de1374a4c60..1e6e8fd250211 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -174,6 +174,10 @@ func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) { } func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) { + // If non-existent path requested, abort + if origNode == nil { + return nil, nil, 0, nil + } // If we reached the requested path, return the current node if pos >= len(path) { // Although we most probably have the original node expanded, encoding @@ -193,10 +197,6 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new } // Path still needs to be traversed, descend into children switch n := (origNode).(type) { - case nil: - // Non-existent path requested, abort - return nil, nil, 0, nil - case valueNode: // Path prematurely ended, abort return nil, nil, 0, nil From c9f892cd6a05c8c733aa8f2626821b8d9bdc24eb Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Wed, 24 Nov 2021 14:23:54 +0800 Subject: [PATCH 2/8] go.mod: update goleveldb (#23417) (#578) Co-authored-by: ucwong --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4ea9fc689adcc..c58a45dfd25eb 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 - github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 + github.com/golang/snappy v0.0.4 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 @@ -65,7 +65,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 - github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tendermint/go-amino v0.14.1 github.com/tendermint/iavl v0.12.0 github.com/tendermint/tendermint v0.31.11 diff --git a/go.sum b/go.sum index e94241a414281..6ab7ca8c4540d 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -387,8 +387,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tendermint/go-amino v0.14.1 h1:o2WudxNfdLNBwMyl2dqOJxiro5rfrEaU0Ugs6offJMk= github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso= github.com/tendermint/iavl v0.12.0 h1:xcaFAr+ycqCj7WN1RzL2EfcBioRDOHcU1oWcg83K028= From 90fd01423ac2b7e61463e673df4732832fb7d435 Mon Sep 17 00:00:00 2001 From: KeefeL <90749943+KeefeL@users.noreply.github.com> Date: Thu, 25 Nov 2021 11:00:14 +0800 Subject: [PATCH 3/8] [R4R]reannounce local pending transactions (#570) * reannouce local pending transactions * add tests for tx_pool reannouce local pending transactions * add tests for handler reannounce local pending transactions --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 8 +++++ core/events.go | 3 ++ core/tx_pool.go | 72 ++++++++++++++++++++++++++++++++------- core/tx_pool_test.go | 41 ++++++++++++++++++++++ core/types/transaction.go | 5 +++ eth/handler.go | 51 +++++++++++++++++++++++++++ eth/handler_eth_test.go | 53 ++++++++++++++++++++++++++++ eth/handler_test.go | 23 +++++++++++-- eth/peerset.go | 16 +++++++++ 11 files changed, 259 insertions(+), 15 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 8be8d20bf4e7a..e896c7d659821 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -90,6 +90,7 @@ var ( utils.TxPoolAccountQueueFlag, utils.TxPoolGlobalQueueFlag, utils.TxPoolLifetimeFlag, + utils.TxPoolReannounceTimeFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, utils.GCModeFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 2a208c827b1d2..fba14530b58cc 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -108,6 +108,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.TxPoolAccountQueueFlag, utils.TxPoolGlobalQueueFlag, utils.TxPoolLifetimeFlag, + utils.TxPoolReannounceTimeFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 49ea0d1de157e..67632c031b52e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -398,6 +398,11 @@ var ( Usage: "Maximum amount of time non-executable transaction are queued", Value: ethconfig.Defaults.TxPool.Lifetime, } + TxPoolReannounceTimeFlag = cli.DurationFlag{ + Name: "txpool.reannouncetime", + Usage: "Duration for announcing local pending transactions again (default = 10 years, minimum = 1 minute)", + Value: ethconfig.Defaults.TxPool.ReannounceTime, + } // Performance tuning settings CacheFlag = cli.IntFlag{ Name: "cache", @@ -1410,6 +1415,9 @@ func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { if ctx.GlobalIsSet(TxPoolLifetimeFlag.Name) { cfg.Lifetime = ctx.GlobalDuration(TxPoolLifetimeFlag.Name) } + if ctx.GlobalIsSet(TxPoolReannounceTimeFlag.Name) { + cfg.ReannounceTime = ctx.GlobalDuration(TxPoolReannounceTimeFlag.Name) + } } func setEthash(ctx *cli.Context, cfg *ethconfig.Config) { diff --git a/core/events.go b/core/events.go index ac935a137f5f6..5e730a24a7b49 100644 --- a/core/events.go +++ b/core/events.go @@ -24,6 +24,9 @@ import ( // NewTxsEvent is posted when a batch of transactions enter the transaction pool. type NewTxsEvent struct{ Txs []*types.Transaction } +// ReannoTxsEvent is posted when a batch of local pending transactions exceed a specified duration. +type ReannoTxsEvent struct{ Txs []*types.Transaction } + // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } diff --git a/core/tx_pool.go b/core/tx_pool.go index d0304857c36f1..4c08bc72b37c4 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -49,6 +49,9 @@ const ( // more expensive to propagate; larger transactions also take more resources // to validate whether they fit into the pool or not. txMaxSize = 4 * txSlotSize // 128KB + + // txReannoMaxNum is the maximum number of transactions a reannounce action can include. + txReannoMaxNum = 1024 ) var ( @@ -88,6 +91,7 @@ var ( var ( evictionInterval = time.Minute // Time interval to check for evictable transactions statsReportInterval = 8 * time.Second // Time interval to report transaction pool stats + reannounceInterval = time.Minute // Time interval to check for reannounce transactions ) var ( @@ -152,7 +156,8 @@ type TxPoolConfig struct { AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts - Lifetime time.Duration // Maximum amount of time non-executable transaction are queued + Lifetime time.Duration // Maximum amount of time non-executable transaction are queued + ReannounceTime time.Duration // Duration for announcing local pending transactions again } // DefaultTxPoolConfig contains the default configurations for the transaction @@ -169,7 +174,8 @@ var DefaultTxPoolConfig = TxPoolConfig{ AccountQueue: 64, GlobalQueue: 1024, - Lifetime: 3 * time.Hour, + Lifetime: 3 * time.Hour, + ReannounceTime: 10 * 365 * 24 * time.Hour, } // sanitize checks the provided user configurations and changes anything that's @@ -208,6 +214,10 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { log.Warn("Sanitizing invalid txpool lifetime", "provided", conf.Lifetime, "updated", DefaultTxPoolConfig.Lifetime) conf.Lifetime = DefaultTxPoolConfig.Lifetime } + if conf.ReannounceTime < time.Minute { + log.Warn("Sanitizing invalid txpool reannounce time", "provided", conf.ReannounceTime, "updated", time.Minute) + conf.ReannounceTime = time.Minute + } return conf } @@ -219,14 +229,15 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { // current state) and future transactions. Transactions move between those // two states over time as they are received and processed. type TxPool struct { - config TxPoolConfig - chainconfig *params.ChainConfig - chain blockChain - gasPrice *big.Int - txFeed event.Feed - scope event.SubscriptionScope - signer types.Signer - mu sync.RWMutex + config TxPoolConfig + chainconfig *params.ChainConfig + chain blockChain + gasPrice *big.Int + txFeed event.Feed + reannoTxFeed event.Feed // Event feed for announcing transactions again + scope event.SubscriptionScope + signer types.Signer + mu sync.RWMutex istanbul bool // Fork indicator whether we are in the istanbul stage. eip2718 bool // Fork indicator whether we are using EIP-2718 type transactions. @@ -323,14 +334,16 @@ func (pool *TxPool) loop() { var ( prevPending, prevQueued, prevStales int // Start the stats reporting and transaction eviction tickers - report = time.NewTicker(statsReportInterval) - evict = time.NewTicker(evictionInterval) - journal = time.NewTicker(pool.config.Rejournal) + report = time.NewTicker(statsReportInterval) + evict = time.NewTicker(evictionInterval) + reannounce = time.NewTicker(reannounceInterval) + journal = time.NewTicker(pool.config.Rejournal) // Track the previous head headers for transaction reorgs head = pool.chain.CurrentBlock() ) defer report.Stop() defer evict.Stop() + defer reannounce.Stop() defer journal.Stop() for { @@ -378,6 +391,33 @@ func (pool *TxPool) loop() { } pool.mu.Unlock() + case <-reannounce.C: + pool.mu.RLock() + reannoTxs := func() []*types.Transaction { + txs := make([]*types.Transaction, 0) + for addr, list := range pool.pending { + if !pool.locals.contains(addr) { + continue + } + + for _, tx := range list.Flatten() { + // Default ReannounceTime is 10 years, won't announce by default. + if time.Since(tx.Time()) < pool.config.ReannounceTime { + break + } + txs = append(txs, tx) + if len(txs) >= txReannoMaxNum { + return txs + } + } + } + return txs + }() + pool.mu.RUnlock() + if len(reannoTxs) > 0 { + pool.reannoTxFeed.Send(ReannoTxsEvent{reannoTxs}) + } + // Handle local transaction journal rotation case <-journal.C: if pool.journal != nil { @@ -412,6 +452,12 @@ func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- NewTxsEvent) event.Subscripti return pool.scope.Track(pool.txFeed.Subscribe(ch)) } +// SubscribeReannoTxsEvent registers a subscription of ReannoTxsEvent and +// starts sending event to the given channel. +func (pool *TxPool) SubscribeReannoTxsEvent(ch chan<- ReannoTxsEvent) event.Subscription { + return pool.scope.Track(pool.reannoTxFeed.Subscribe(ch)) +} + // GasPrice returns the current gas price enforced by the transaction pool. func (pool *TxPool) GasPrice() *big.Int { pool.mu.RLock() diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 5d555f5a9cfd3..5f27631e15619 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -1933,6 +1933,47 @@ func TestTransactionSlotCount(t *testing.T) { } } +// Tests the local pending transaction announced again correctly. +func TestTransactionPendingReannouce(t *testing.T) { + t.Parallel() + + // Create the pool to test the limit enforcement with + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + + config := testTxPoolConfig + // This ReannounceTime will be modified to time.Minute when creating tx_pool. + config.ReannounceTime = time.Second + reannounceInterval = time.Second + + pool := NewTxPool(config, params.TestChainConfig, blockchain) + // Modify ReannounceTime to trigger quicker. + pool.config.ReannounceTime = time.Second + defer pool.Stop() + + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(account, big.NewInt(1000000)) + + events := make(chan ReannoTxsEvent, testTxPoolConfig.AccountQueue) + sub := pool.reannoTxFeed.Subscribe(events) + defer sub.Unsubscribe() + + // Generate a batch of transactions and add to tx_pool locally. + txs := make([]*types.Transaction, 0, testTxPoolConfig.AccountQueue) + for i := uint64(0); i < testTxPoolConfig.AccountQueue; i++ { + txs = append(txs, transaction(i, 100000, key)) + } + pool.AddLocals(txs) + + select { + case ev := <-events: + t.Logf("received reannouce event, txs length: %d", len(ev.Txs)) + case <-time.After(5 * time.Second): + t.Errorf("reannouce event not fired") + } +} + // Benchmarks the speed of validating the contents of the pending queue of the // transaction pool. func BenchmarkPendingDemotion100(b *testing.B) { benchmarkPendingDemotion(b, 100) } diff --git a/core/types/transaction.go b/core/types/transaction.go index b127cb2af648f..74c011544b872 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -82,6 +82,11 @@ type TxData interface { setSignatureValues(chainID, v, r, s *big.Int) } +// Time returns transaction's time +func (tx *Transaction) Time() time.Time { + return tx.time +} + // EncodeRLP implements rlp.Encoder func (tx *Transaction) EncodeRLP(w io.Writer) error { if tx.Type() == LegacyTxType { diff --git a/eth/handler.go b/eth/handler.go index f00f955b348bb..9eb448040bc19 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -73,6 +73,10 @@ type txPool interface { // SubscribeNewTxsEvent should return an event subscription of // NewTxsEvent and send events to the given channel. SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + + // SubscribeReannoTxsEvent should return an event subscription of + // ReannoTxsEvent and send events to the given channel. + SubscribeReannoTxsEvent(chan<- core.ReannoTxsEvent) event.Subscription } // handlerConfig is the collection of initialization parameters to create a full @@ -120,6 +124,8 @@ type handler struct { eventMux *event.TypeMux txsCh chan core.NewTxsEvent txsSub event.Subscription + reannoTxsCh chan core.ReannoTxsEvent + reannoTxsSub event.Subscription minedBlockSub *event.TypeMuxSubscription whitelist map[uint64]common.Hash @@ -432,6 +438,12 @@ func (h *handler) Start(maxPeers int) { h.txsSub = h.txpool.SubscribeNewTxsEvent(h.txsCh) go h.txBroadcastLoop() + // announce local pending transactions again + h.wg.Add(1) + h.reannoTxsCh = make(chan core.ReannoTxsEvent, txChanSize) + h.reannoTxsSub = h.txpool.SubscribeReannoTxsEvent(h.reannoTxsCh) + go h.txReannounceLoop() + // broadcast mined blocks h.wg.Add(1) h.minedBlockSub = h.eventMux.Subscribe(core.NewMinedBlockEvent{}) @@ -445,6 +457,7 @@ func (h *handler) Start(maxPeers int) { func (h *handler) Stop() { h.txsSub.Unsubscribe() // quits txBroadcastLoop + h.reannoTxsSub.Unsubscribe() // quits txReannounceLoop h.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop // Quit chainSync and txsync64. @@ -549,6 +562,31 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { "tx packs", directPeers, "broadcast txs", directCount) } +// ReannounceTransactions will announce a batch of local pending transactions +// to a square root of all peers. +func (h *handler) ReannounceTransactions(txs types.Transactions) { + var ( + annoCount int // Count of announcements made + annos = make(map[*ethPeer][]common.Hash) // Set peer->hash to announce + ) + + // Announce transactions hash to a batch of peers + peersCount := uint(math.Sqrt(float64(h.peers.len()))) + peers := h.peers.headPeers(peersCount) + for _, tx := range txs { + for _, peer := range peers { + annos[peer] = append(annos[peer], tx.Hash()) + } + } + + for peer, hashes := range annos { + annoCount += len(hashes) + peer.AsyncSendPooledTransactionHashes(hashes) + } + log.Debug("Transaction reannounce", "txs", len(txs), + "announce packs", peersCount, "announced hashes", annoCount) +} + // minedBroadcastLoop sends mined blocks to connected peers. func (h *handler) minedBroadcastLoop() { defer h.wg.Done() @@ -573,3 +611,16 @@ func (h *handler) txBroadcastLoop() { } } } + +// txReannounceLoop announces local pending transactions to connected peers again. +func (h *handler) txReannounceLoop() { + defer h.wg.Done() + for { + select { + case event := <-h.reannoTxsCh: + h.ReannounceTransactions(event.Txs) + case <-h.reannoTxsSub.Err(): + return + } + } +} diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 271bae07c7636..aad2c72b1b775 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -450,6 +450,59 @@ func testTransactionPropagation(t *testing.T, protocol uint) { } } +// Tests that local pending transactions get propagated to peers. +func TestTransactionPendingReannounce(t *testing.T) { + t.Parallel() + + // Create a source handler to announce transactions from and a sink handler + // to receive them. + source := newTestHandler() + defer source.close() + + sink := newTestHandler() + defer sink.close() + sink.handler.acceptTxs = 1 // mark synced to accept transactions + + sourcePipe, sinkPipe := p2p.MsgPipe() + defer sourcePipe.Close() + defer sinkPipe.Close() + + sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{0}, "", nil), sourcePipe, source.txpool) + sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeer(enode.ID{0}, "", nil), sinkPipe, sink.txpool) + defer sourcePeer.Close() + defer sinkPeer.Close() + + go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(source.handler), peer) + }) + go sink.handler.runEthPeer(sinkPeer, func(peer *eth.Peer) error { + return eth.Handle((*ethHandler)(sink.handler), peer) + }) + + // Subscribe transaction pools + txCh := make(chan core.NewTxsEvent, 1024) + sub := sink.txpool.SubscribeNewTxsEvent(txCh) + defer sub.Unsubscribe() + + txs := make([]*types.Transaction, 64) + for nonce := range txs { + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), nil) + tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + + txs[nonce] = tx + } + source.txpool.ReannouceTransactions(txs) + + for arrived := 0; arrived < len(txs); { + select { + case event := <-txCh: + arrived += len(event.Txs) + case <-time.NewTimer(time.Second).C: + t.Errorf("sink: transaction propagation timed out: have %d, want %d", arrived, len(txs)) + } + } +} + // Tests that post eth protocol handshake, clients perform a mutual checkpoint // challenge to validate each other's chains. Hash mismatches, or missing ones // during a fast sync should lead to the peer getting dropped. diff --git a/eth/handler_test.go b/eth/handler_test.go index a90ef5c348aa4..c3b7b769b27d5 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -48,8 +48,9 @@ var ( type testTxPool struct { pool map[common.Hash]*types.Transaction // Hash map of collected transactions - txFeed event.Feed // Notification feed to allow waiting for inclusion - lock sync.RWMutex // Protects the transaction pool + txFeed event.Feed // Notification feed to allow waiting for inclusion + reannoTxFeed event.Feed // Notification feed to trigger reannouce + lock sync.RWMutex // Protects the transaction pool } // newTestTxPool creates a mock transaction pool. @@ -90,6 +91,18 @@ func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { return make([]error, len(txs)) } +// ReannouceTransactions announce the transactions to some peers. +func (p *testTxPool) ReannouceTransactions(txs []*types.Transaction) []error { + p.lock.Lock() + defer p.lock.Unlock() + + for _, tx := range txs { + p.pool[tx.Hash()] = tx + } + p.reannoTxFeed.Send(core.ReannoTxsEvent{Txs: txs}) + return make([]error, len(txs)) +} + // Pending returns all the transactions known to the pool func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) { p.lock.RLock() @@ -112,6 +125,12 @@ func (p *testTxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subs return p.txFeed.Subscribe(ch) } +// SubscribeReannoTxsEvent should return an event subscription of ReannoTxsEvent and +// send events to the given channel. +func (p *testTxPool) SubscribeReannoTxsEvent(ch chan<- core.ReannoTxsEvent) event.Subscription { + return p.reannoTxFeed.Subscribe(ch) +} + // testHandler is a live implementation of the Ethereum protocol handler, just // preinitialized with some sane testing defaults and the transaction pool mocked // out. diff --git a/eth/peerset.go b/eth/peerset.go index f0955f34c6c88..220b01d832c32 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -266,6 +266,22 @@ func (ps *peerSet) peer(id string) *ethPeer { return ps.peers[id] } +// headPeers retrieves a specified number list of peers. +func (ps *peerSet) headPeers(num uint) []*ethPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + if num > uint(len(ps.peers)) { + num = uint(len(ps.peers)) + } + + list := make([]*ethPeer, 0, num) + for _, p := range ps.peers { + list = append(list, p) + } + return list +} + // peersWithoutBlock retrieves a list of peers that do not have a given block in // their set of known hashes so it might be propagated to them. func (ps *peerSet) peersWithoutBlock(hash common.Hash) []*ethPeer { From aff68c35a438aeb6b875737447a96930a5b03459 Mon Sep 17 00:00:00 2001 From: dylanhuang Date: Fri, 26 Nov 2021 18:48:56 +0800 Subject: [PATCH 4/8] [R4R] pre-release ci flow (#594) * ci: add pre release ci flow * ci: fix change log format --- .github/generate_change_log.sh | 2 +- .github/workflows/pre-release.yml | 183 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 6 +- 3 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/pre-release.yml diff --git a/.github/generate_change_log.sh b/.github/generate_change_log.sh index 09332b7911e59..ab8ffff689b14 100755 --- a/.github/generate_change_log.sh +++ b/.github/generate_change_log.sh @@ -15,7 +15,7 @@ while read line; do if [[ $line == *"$version_prefix"* ]] && [ $start == 1 ]; then break; fi - if [ $start == 1 ] && [[ $line != "" ]]; then + if [ $start == 1 ]; then CHANGE_LOG+="$line\n" fi done < ${change_log_file} diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml new file mode 100644 index 0000000000000..888fe900964b8 --- /dev/null +++ b/.github/workflows/pre-release.yml @@ -0,0 +1,183 @@ +name: Pre Release + +on: + push: + tags: + - 'pre-*' + +jobs: + build: + name: Build Release + strategy: + matrix: + go-version: [1.16.x] + os: [ubuntu-18.04, macos-11, windows-2019] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - uses: actions/cache@v2 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + # ============================== + # Linux/Macos/Windows Build + # ============================== + + - name: Build Binary for ${{matrix.os}} + run: make geth + + # ============================== + # Upload artifacts + # ============================== + + - name: Upload Linux Build + uses: actions/upload-artifact@v2 + if: matrix.os == 'ubuntu-18.04' + with: + name: linux + path: ./build/bin/geth + + - name: Upload MacOS Build + uses: actions/upload-artifact@v2 + if: matrix.os == 'macos-11' + with: + name: macos + path: ./build/bin/geth + + - name: Upload Windows Build + uses: actions/upload-artifact@v2 + if: matrix.os == 'windows-2019' + with: + name: windows + path: ./build/bin/geth.exe + + release: + name: Release + needs: build + runs-on: ubuntu-18.04 + steps: + - name: Set Env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Checkout Code + uses: actions/checkout@v2 + + # ============================== + # Download artifacts + # ============================== + + - name: Download Artifacts + uses: actions/download-artifact@v2 + with: + name: linux + path: ./linux + + - name: Download Artifacts + uses: actions/download-artifact@v2 + with: + name: macos + path: ./macos + + - name: Download Artifacts + uses: actions/download-artifact@v2 + with: + name: windows + path: ./windows + + - name: Download Config File + run: | + . ./.github/release.env + echo "mainnet.zip url: $MAINNET_FILE_URL" + echo "testnet.zip url: $TESTNET_FILE_URL" + curl -L $MAINNET_FILE_URL -o ./mainnet.zip + curl -L $TESTNET_FILE_URL -o ./testnet.zip + + # ============================== + # Create release + # ============================== + + - name: Create Release + id: create_release + uses: actions/create-release@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body: | + versing: ${{ env.RELEASE_VERSION}} + git commit: ${{ github.sha }} + draft: true + prerelease: true + + # Check downloaded files + - run: ls + + - name: Upload Release Asset - Linux + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./linux/geth + asset_name: geth_linux + asset_content_type: application/octet-stream + + - name: Upload Release Asset - MacOS + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./macos/geth + asset_name: geth_mac + asset_content_type: application/octet-stream + + - name: Upload Release Asset - Windows + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./windows/geth.exe + asset_name: geth_windows.exe + asset_content_type: application/octet-stream + + - name: Upload Release Asset - MAINNET.ZIP + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./mainnet.zip + asset_name: mainnet.zip + asset_content_type: application/zip + + - name: Upload Release Asset - TESTNET.ZIP + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./testnet.zip + asset_name: testnet.zip + asset_content_type: application/zip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1633752c23450..77cbbee7251e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,14 +15,14 @@ jobs: os: [ubuntu-18.04, macos-11, windows-2019] runs-on: ${{ matrix.os }} steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Install Go uses: actions/setup-go@v2 with: go-version: ${{ matrix.go-version }} - - name: Checkout Code - uses: actions/checkout@v2 - - uses: actions/cache@v2 with: # In order: From 0b575443c43f47f2f50ed5549d2bcaf7ccc39ab4 Mon Sep 17 00:00:00 2001 From: dylanhuang Date: Mon, 29 Nov 2021 10:59:36 +0800 Subject: [PATCH 5/8] prepare for release v.1.1.6 (#603) --- CHANGELOG.md | 11 +++++++++++ params/version.go | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0b345db0e90..babc9e3c222db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,15 @@ # Changelog + +## v1.1.6 +BUGFIX +* [\#582](https://github.com/binance-chain/bsc/pull/582) the DoS vulnerabilities fixed in go-ethereum v1.10.9 + +IMPROVEMENT +* [\#578](https://github.com/binance-chain/bsc/pull/578) reduce memory allocation and upgrade snappy version + +FEATURES +* [\#570](https://github.com/binance-chain/bsc/pull/570) reannounce local pending transactions + ## v1.1.5 BUGFIX * [\#509](https://github.com/binance-chain/bsc/pull/509) fix graceful shutdown bug diff --git a/params/version.go b/params/version.go index d15c2d94c56ef..1df331e96c5ad 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 1 // Minor version component of the current release - VersionPatch = 5 // Patch version component of the current release + VersionPatch = 6 // Patch version component of the current release VersionMeta = "" // Version metadata to append to the version string ) From a8618952fdd6eb213488c48ad291261f7d47be0a Mon Sep 17 00:00:00 2001 From: guagualvcha <296179868@qq.com> Date: Sun, 5 Dec 2021 01:13:56 +0800 Subject: [PATCH 6/8] fix state inconsistent between snapshot and MPT when doing diffsync --- core/blockchain_diff_test.go | 115 ++++++++++++++++++++++++++++++----- core/state_processor.go | 2 +- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index 8e04363d0ce7e..facd86b52b686 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -21,6 +21,8 @@ package core import ( + "bytes" + "encoding/hex" "math/big" "testing" "time" @@ -42,9 +44,43 @@ import ( var ( // testKey is a private key to use for funding a tester account. - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + contractCode, _ = hex.DecodeString("608060405260016000806101000a81548160ff02191690831515021790555034801561002a57600080fd5b506101688061003a6000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806389a2d8011461003b578063b0483f4814610059575b600080fd5b610043610075565b60405161005091906100f4565b60405180910390f35b610073600480360381019061006e91906100bc565b61008b565b005b60008060009054906101000a900460ff16905090565b806000806101000a81548160ff02191690831515021790555050565b6000813590506100b68161011b565b92915050565b6000602082840312156100ce57600080fd5b60006100dc848285016100a7565b91505092915050565b6100ee8161010f565b82525050565b600060208201905061010960008301846100e5565b92915050565b60008115159050919050565b6101248161010f565b811461012f57600080fd5b5056fea264697066735822122092f788b569bfc3786e90601b5dbec01cfc3d76094164fd66ca7d599c4239fc5164736f6c63430008000033") + contractAddr = common.HexToAddress("0xe74a3c7427cda785e0000d42a705b1f3fd371e09") + contractSlot = common.HexToHash("0x290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") + contractData1, _ = hex.DecodeString("b0483f480000000000000000000000000000000000000000000000000000000000000000") + contractData2, _ = hex.DecodeString("b0483f480000000000000000000000000000000000000000000000000000000000000001") + commonGas = 192138 // testAddr is the Ethereum address of the tester account. testAddr = crypto.PubkeyToAddress(testKey.PublicKey) + + checkBlocks = map[int]checkBlockParam{ + 12: { + txs: []checkTransactionParam{ + { + to: &contractAddr, + slot: contractSlot, + value: []byte{01}, + }, + }}, + + 13: { + txs: []checkTransactionParam{ + { + to: &contractAddr, + slot: contractSlot, + value: []byte{}, + }, + }}, + 14: { + txs: []checkTransactionParam{ + { + to: &contractAddr, + slot: contractSlot, + value: []byte{01}, + }, + }}, + } // testBlocks is the test parameters array for specific blocks. testBlocks = []testBlockParam{ { @@ -52,7 +88,7 @@ var ( blockNr: 11, txs: []testTransactionParam{ { - to: common.Address{0x01}, + to: &common.Address{0x01}, value: big.NewInt(1), gasPrice: big.NewInt(1), data: nil, @@ -63,51 +99,74 @@ var ( blockNr: 12, txs: []testTransactionParam{ { - to: common.Address{0x01}, + to: &common.Address{0x01}, value: big.NewInt(1), gasPrice: big.NewInt(1), data: nil, }, { - to: common.Address{0x02}, + to: &common.Address{0x02}, value: big.NewInt(2), gasPrice: big.NewInt(2), data: nil, }, + { + to: nil, + value: big.NewInt(0), + gasPrice: big.NewInt(2), + data: contractCode, + }, }, }, { blockNr: 13, txs: []testTransactionParam{ { - to: common.Address{0x01}, + to: &common.Address{0x01}, value: big.NewInt(1), gasPrice: big.NewInt(1), data: nil, }, { - to: common.Address{0x02}, + to: &common.Address{0x02}, value: big.NewInt(2), gasPrice: big.NewInt(2), data: nil, }, { - to: common.Address{0x03}, + to: &common.Address{0x03}, value: big.NewInt(3), gasPrice: big.NewInt(3), data: nil, }, + { + to: &contractAddr, + value: big.NewInt(0), + gasPrice: big.NewInt(3), + data: contractData1, + }, }, }, { blockNr: 14, + txs: []testTransactionParam{ + { + to: &contractAddr, + value: big.NewInt(0), + gasPrice: big.NewInt(3), + data: contractData2, + }, + }, + }, + { + blockNr: 15, txs: []testTransactionParam{}, }, } ) type testTransactionParam struct { - to common.Address + to *common.Address value *big.Int gasPrice *big.Int data []byte @@ -118,6 +177,16 @@ type testBlockParam struct { txs []testTransactionParam } +type checkTransactionParam struct { + to *common.Address + slot common.Hash + value []byte +} + +type checkBlockParam struct { + txs []checkTransactionParam +} + // testBackend is a mock implementation of the live Ethereum message handler. Its // purpose is to allow testing the request/reply workflows and wire serialization // in the `eth` protocol without actually doing any data processing. @@ -153,8 +222,15 @@ func newTestBackendWithGenerator(blocks int, lightProcess bool) *testBackend { // Specific block setting, the index in this generator has 1 diff from specified blockNr. if i+1 == testBlock.blockNr { for _, testTransaction := range testBlock.txs { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), testTransaction.to, - testTransaction.value, params.TxGas, testTransaction.gasPrice, testTransaction.data), signer, testKey) + var transaction *types.Transaction + if testTransaction.to == nil { + transaction = types.NewContractCreation(block.TxNonce(testAddr), + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } else { + transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } + tx, err := types.SignTx(transaction, signer, testKey) if err != nil { panic(err) } @@ -168,8 +244,8 @@ func newTestBackendWithGenerator(blocks int, lightProcess bool) *testBackend { // We want to simulate an empty middle block, having the same state as the // first one. The last is needs a state change again to force a reorg. for _, testTransaction := range testBlocks[0].txs { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), testTransaction.to, - testTransaction.value, params.TxGas, testTransaction.gasPrice, testTransaction.data), signer, testKey) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data), signer, testKey) if err != nil { panic(err) } @@ -241,6 +317,14 @@ func TestProcessDiffLayer(t *testing.T) { lightBackend.Chain().HandleDiffLayer(diff, "testpid", true) } _, err := lightBackend.chain.insertChain([]*types.Block{block}, true) + if checks, exist := checkBlocks[i]; exist { + for _, check := range checks.txs { + s, _ := lightBackend.Chain().Snapshots().Snapshot(block.Root()).Storage(crypto.Keccak256Hash((*check.to)[:]), check.slot) + if !bytes.Equal(s, check.value) { + t.Fatalf("Expected value %x, get %x", check.value, s) + } + } + } if err != nil { t.Errorf("failed to insert block %v", err) } @@ -385,13 +469,14 @@ func TestGetDiffAccounts(t *testing.T) { t.Errorf("the diff accounts does't include addr: %v", testAddr) } } - for _, transaction := range testBlock.txs { + if transaction.to == nil || len(transaction.data) > 0 { + continue + } for idx, account := range accounts { - if transaction.to == account { + if *transaction.to == account { break } - if idx == len(accounts)-1 { t.Errorf("the diff accounts does't include addr: %v", transaction.to) } diff --git a/core/state_processor.go b/core/state_processor.go index 9f2a09bd369dd..6ca99f2eec9ca 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -253,7 +253,7 @@ func (p *LightStateProcessor) LightProcess(diffLayer *types.DiffLayer, block *ty //update storage latestRoot := common.BytesToHash(latestAccount.Root) - if latestRoot != previousAccount.Root && latestRoot != types.EmptyRootHash { + if latestRoot != previousAccount.Root { accountTrie, err := statedb.Database().OpenStorageTrie(addrHash, previousAccount.Root) if err != nil { errChan <- err From 2bf7d669ea47806ab7989f00e7470308e2a0dd71 Mon Sep 17 00:00:00 2001 From: guagualvcha <296179868@qq.com> Date: Sun, 5 Dec 2021 11:55:59 +0800 Subject: [PATCH 7/8] change the release env --- .github/release.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/release.env b/.github/release.env index c65f709a262b3..37739d157e04e 100644 --- a/.github/release.env +++ b/.github/release.env @@ -1,2 +1,2 @@ -MAINNET_FILE_URL="https://github.com/binance-chain/bsc/releases/download/v1.1.2/mainnet.zip" -TESTNET_FILE_URL="https://github.com/binance-chain/bsc/releases/download/v1.1.2/testnet.zip" +MAINNET_FILE_URL="https://github.com/binance-chain/bsc/releases/download/v1.1.6/mainnet.zip" +TESTNET_FILE_URL="https://github.com/binance-chain/bsc/releases/download/v1.1.6/testnet.zip" From d7e48bf8243ae397d1a807ceb496f662aa0fd463 Mon Sep 17 00:00:00 2001 From: guagualvcha <296179868@qq.com> Date: Sun, 5 Dec 2021 11:58:07 +0800 Subject: [PATCH 8/8] add change logs && prelease v1.1.7 --- CHANGELOG.md | 5 +++++ params/version.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index babc9e3c222db..4d5bf0877f8f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v1.1.7 + +BUGFIX +* [\#628](https://github.com/binance-chain/bsc/pull/628) fix state inconsistent when doing diffsync + ## v1.1.6 BUGFIX * [\#582](https://github.com/binance-chain/bsc/pull/582) the DoS vulnerabilities fixed in go-ethereum v1.10.9 diff --git a/params/version.go b/params/version.go index 1df331e96c5ad..0cab1d8822cfa 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 1 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release + VersionPatch = 7 // Patch version component of the current release VersionMeta = "" // Version metadata to append to the version string )