From 8e59f128b1edcb282af23d0f4797bec0597fecb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 23 Aug 2021 23:50:24 +0200 Subject: [PATCH] eth/gasprice: feeHistory improvements (#23422) * eth/gasprice: cache feeHistory results * eth/gasprice: changed feeHistory block count limitation * eth/gasprice: do not use embedded struct in blockFees * eth/gasprice: fee processing logic cleanup * eth/gasprice: purge feeHistory cache at chain reorgs --- eth/ethconfig/config.go | 4 +- eth/gasprice/feehistory.go | 105 +++++++++++++++++--------------- eth/gasprice/feehistory_test.go | 22 +++---- eth/gasprice/gasprice.go | 20 ++++++ eth/gasprice/gasprice_test.go | 5 ++ 5 files changed, 95 insertions(+), 61 deletions(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 23ccf2484310b..89cdb75597e08 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -43,8 +43,8 @@ import ( var FullNodeGPO = gasprice.Config{ Blocks: 20, Percentile: 60, - MaxHeaderHistory: 0, - MaxBlockHistory: 0, + MaxHeaderHistory: 1024, + MaxBlockHistory: 1024, MaxPrice: gasprice.DefaultMaxPrice, IgnorePrice: gasprice.DefaultIgnorePrice, } diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 2c4486c6f4c2d..970dfd4467a57 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -18,8 +18,10 @@ package gasprice import ( "context" + "encoding/binary" "errors" "fmt" + "math" "math/big" "sort" "sync/atomic" @@ -37,10 +39,6 @@ var ( ) const ( - // maxFeeHistory is the maximum number of blocks that can be retrieved for a - // fee history request. - maxFeeHistory = 1024 - // maxBlockFetchers is the max number of goroutines to spin up to pull blocks // for the fee history calculation (mostly relevant for LES). maxBlockFetchers = 4 @@ -54,10 +52,15 @@ type blockFees struct { block *types.Block // only set if reward percentiles are requested receipts types.Receipts // filled by processBlock + results processedFees + err error +} + +// processedFees contains the results of a processed block and is also used for caching +type processedFees struct { reward []*big.Int baseFee, nextBaseFee *big.Int gasUsedRatio float64 - err error } // txGasAndReward is sorted in ascending order based on reward @@ -82,15 +85,15 @@ func (s sortGasAndReward) Less(i, j int) bool { // fills in the rest of the fields. func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { chainconfig := oracle.backend.ChainConfig() - if bf.baseFee = bf.header.BaseFee; bf.baseFee == nil { - bf.baseFee = new(big.Int) + if bf.results.baseFee = bf.header.BaseFee; bf.results.baseFee == nil { + bf.results.baseFee = new(big.Int) } if chainconfig.IsLondon(big.NewInt(int64(bf.blockNumber + 1))) { - bf.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header) + bf.results.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header) } else { - bf.nextBaseFee = new(big.Int) + bf.results.nextBaseFee = new(big.Int) } - bf.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) + bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) if len(percentiles) == 0 { // rewards were not requested, return null return @@ -100,11 +103,11 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { return } - bf.reward = make([]*big.Int, len(percentiles)) + bf.results.reward = make([]*big.Int, len(percentiles)) if len(bf.block.Transactions()) == 0 { // return an all zero row if there are no transactions to gather data from - for i := range bf.reward { - bf.reward[i] = new(big.Int) + for i := range bf.results.reward { + bf.results.reward[i] = new(big.Int) } return } @@ -125,7 +128,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { txIndex++ sumGasUsed += sorter[txIndex].gasUsed } - bf.reward[i] = sorter[txIndex].reward + bf.results.reward[i] = sorter[txIndex].reward } } @@ -134,7 +137,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( headBlock rpc.BlockNumber pendingBlock *types.Block @@ -167,17 +170,6 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block } else if pendingBlock == nil && lastBlock > headBlock { return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) } - if maxHistory != 0 { - // limit retrieval to the given number of latest blocks - if tooOldCount := int64(headBlock) - int64(maxHistory) - int64(lastBlock) + int64(blocks); tooOldCount > 0 { - // tooOldCount is the number of requested blocks that are too old to be served - if int64(blocks) > tooOldCount { - blocks -= int(tooOldCount) - } else { - return nil, nil, 0, 0, nil - } - } - } // ensure not trying to retrieve before genesis if rpc.BlockNumber(blocks) > lastBlock+1 { blocks = int(lastBlock + 1) @@ -202,6 +194,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast if blocks < 1 { return common.Big0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks } + maxFeeHistory := oracle.maxHeaderHistory + if len(rewardPercentiles) != 0 { + maxFeeHistory = oracle.maxBlockHistory + } if blocks > maxFeeHistory { log.Warn("Sanitizing fee history length", "requested", blocks, "truncated", maxFeeHistory) blocks = maxFeeHistory @@ -214,17 +210,12 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } - // Only process blocks if reward percentiles were requested - maxHistory := oracle.maxHeaderHistory - if len(rewardPercentiles) != 0 { - maxHistory = oracle.maxBlockHistory - } var ( pendingBlock *types.Block pendingReceipts []*types.Receipt err error ) - pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks, maxHistory) + pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks) if err != nil || blocks == 0 { return common.Big0, nil, nil, nil, err } @@ -234,6 +225,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast next = oldestBlock results = make(chan *blockFees, blocks) ) + percentileKey := make([]byte, 8*len(rewardPercentiles)) + for i, p := range rewardPercentiles { + binary.LittleEndian.PutUint64(percentileKey[i*8:(i+1)*8], math.Float64bits(p)) + } for i := 0; i < maxBlockFetchers && i < blocks; i++ { go func() { for { @@ -246,24 +241,38 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast fees := &blockFees{blockNumber: blockNumber} if pendingBlock != nil && blockNumber >= pendingBlock.NumberU64() { fees.block, fees.receipts = pendingBlock, pendingReceipts + fees.header = fees.block.Header() + oracle.processBlock(fees, rewardPercentiles) + results <- fees } else { - if len(rewardPercentiles) != 0 { - fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) - if fees.block != nil && fees.err == nil { - fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) - } + cacheKey := struct { + number uint64 + percentiles string + }{blockNumber, string(percentileKey)} + + if p, ok := oracle.historyCache.Get(cacheKey); ok { + fees.results = p.(processedFees) + results <- fees } else { - fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + if len(rewardPercentiles) != 0 { + fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) + if fees.block != nil && fees.err == nil { + fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) + fees.header = fees.block.Header() + } + } else { + fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + } + if fees.header != nil && fees.err == nil { + oracle.processBlock(fees, rewardPercentiles) + if fees.err == nil { + oracle.historyCache.Add(cacheKey, fees.results) + } + } + // send to results even if empty to guarantee that blocks items are sent in total + results <- fees } } - if fees.block != nil { - fees.header = fees.block.Header() - } - if fees.header != nil { - oracle.processBlock(fees, rewardPercentiles) - } - // send to results even if empty to guarantee that blocks items are sent in total - results <- fees } }() } @@ -279,8 +288,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast return common.Big0, nil, nil, nil, fees.err } i := int(fees.blockNumber - oldestBlock) - if fees.header != nil { - reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.reward, fees.baseFee, fees.nextBaseFee, fees.gasUsedRatio + if fees.results.baseFee != nil { + reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.results.reward, fees.results.baseFee, fees.results.nextBaseFee, fees.results.gasUsedRatio } else { // getting no block and no error means we are requesting into the future (might happen because of a reorg) if i < firstMissing { diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index 16c74b7db46a4..c259eb0acf762 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -36,20 +36,20 @@ func TestFeeHistory(t *testing.T) { expCount int expErr error }{ - {false, 0, 0, 10, 30, nil, 21, 10, nil}, - {false, 0, 0, 10, 30, []float64{0, 10}, 21, 10, nil}, - {false, 0, 0, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile}, - {false, 0, 0, 1000000000, 30, nil, 0, 31, nil}, - {false, 0, 0, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil}, - {false, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead}, - {true, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead}, + {false, 1000, 1000, 10, 30, nil, 21, 10, nil}, + {false, 1000, 1000, 10, 30, []float64{0, 10}, 21, 10, nil}, + {false, 1000, 1000, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile}, + {false, 1000, 1000, 1000000000, 30, nil, 0, 31, nil}, + {false, 1000, 1000, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil}, + {false, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead}, + {true, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead}, {false, 20, 2, 100, rpc.LatestBlockNumber, nil, 13, 20, nil}, {false, 20, 2, 100, rpc.LatestBlockNumber, []float64{0, 10}, 31, 2, nil}, {false, 20, 2, 100, 32, []float64{0, 10}, 31, 2, nil}, - {false, 0, 0, 1, rpc.PendingBlockNumber, nil, 0, 0, nil}, - {false, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, - {true, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, - {true, 0, 0, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 1, rpc.PendingBlockNumber, nil, 0, 0, nil}, + {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, + {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, + {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 407eeaa2899cc..8feb5ef24ba4c 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -23,10 +23,13 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -53,6 +56,7 @@ type OracleBackend interface { GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) PendingBlockAndReceipts() (*types.Block, types.Receipts) ChainConfig() *params.ChainConfig + SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } // Oracle recommends gas prices based on the content of recent @@ -68,6 +72,7 @@ type Oracle struct { checkBlocks, percentile int maxHeaderHistory, maxBlockHistory int + historyCache *lru.Cache } // NewOracle returns a new gasprice oracle which can recommend suitable @@ -99,6 +104,20 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { } else if ignorePrice.Int64() > 0 { log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice) } + + cache, _ := lru.New(2048) + headEvent := make(chan core.ChainHeadEvent, 1) + backend.SubscribeChainHeadEvent(headEvent) + go func() { + var lastHead common.Hash + for ev := range headEvent { + if ev.Block.ParentHash() != lastHead { + cache.Purge() + } + lastHead = ev.Block.Hash() + } + }() + return &Oracle{ backend: backend, lastPrice: params.Default, @@ -108,6 +127,7 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { percentile: percent, maxHeaderHistory: params.MaxHeaderHistory, maxBlockHistory: params.MaxBlockHistory, + historyCache: cache, } } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index dea8fea95ae3d..feecfddec7308 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -90,6 +91,10 @@ func (b *testBackend) ChainConfig() *params.ChainConfig { return b.chain.Config() } +func (b *testBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} + func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBackend { var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")