Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

eth/gasprice: feeHistory improvements #23422

Merged
merged 5 commits into from Aug 23, 2021
Merged

eth/gasprice: feeHistory improvements #23422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0c4a098
eth/gasprice: cache feeHistory results
zsfelfoldi Aug 19, 2021
aebca7f
eth/gasprice: changed feeHistory block count limitation
zsfelfoldi Aug 19, 2021
ac4e010
eth/gasprice: do not use embedded struct in blockFees
zsfelfoldi Aug 23, 2021
77d6825
eth/gasprice: fee processing logic cleanup
zsfelfoldi Aug 23, 2021
27ef400
eth/gasprice: purge feeHistory cache at chain reorgs
zsfelfoldi Aug 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions eth/ethconfig/config.go
View file
Open in desktop
Expand Up @@ -43,8 +43,8 @@ import (
var FullNodeGPO = gasprice.Config{
Blocks: 20,
Percentile: 60,
MaxHeaderHistory: 0,
MaxBlockHistory: 0,
MaxHeaderHistory: 1024,
MaxBlockHistory: 1024,
MaxPrice: gasprice.DefaultMaxPrice,
IgnorePrice: gasprice.DefaultIgnorePrice,
}
Expand Down
105 changes: 57 additions & 48 deletions eth/gasprice/feehistory.go
View file
Open in desktop
Expand Up @@ -18,8 +18,10 @@ package gasprice

import (
"context"
"encoding/binary"
"errors"
"fmt"
"math"
"math/big"
"sort"
"sync/atomic"
Expand All @@ -37,10 +39,6 @@ var (
)

const (
// maxFeeHistory is the maximum number of blocks that can be retrieved for a
// fee history request.
maxFeeHistory = 1024

// maxBlockFetchers is the max number of goroutines to spin up to pull blocks
// for the fee history calculation (mostly relevant for LES).
maxBlockFetchers = 4
Expand All @@ -54,10 +52,15 @@ type blockFees struct {
block *types.Block // only set if reward percentiles are requested
receipts types.Receipts
// filled by processBlock
results processedFees
err error
}

// processedFees contains the results of a processed block and is also used for caching
type processedFees struct {
reward []*big.Int
baseFee, nextBaseFee *big.Int
gasUsedRatio float64
err error
}

// txGasAndReward is sorted in ascending order based on reward
Expand All @@ -82,15 +85,15 @@ func (s sortGasAndReward) Less(i, j int) bool {
// fills in the rest of the fields.
func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
chainconfig := oracle.backend.ChainConfig()
if bf.baseFee = bf.header.BaseFee; bf.baseFee == nil {
bf.baseFee = new(big.Int)
if bf.results.baseFee = bf.header.BaseFee; bf.results.baseFee == nil {
bf.results.baseFee = new(big.Int)
}
if chainconfig.IsLondon(big.NewInt(int64(bf.blockNumber + 1))) {
bf.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header)
bf.results.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header)
} else {
bf.nextBaseFee = new(big.Int)
bf.results.nextBaseFee = new(big.Int)
}
bf.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit)
bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit)
if len(percentiles) == 0 {
// rewards were not requested, return null
return
Expand All @@ -100,11 +103,11 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
return
}

bf.reward = make([]*big.Int, len(percentiles))
bf.results.reward = make([]*big.Int, len(percentiles))
if len(bf.block.Transactions()) == 0 {
// return an all zero row if there are no transactions to gather data from
for i := range bf.reward {
bf.reward[i] = new(big.Int)
for i := range bf.results.reward {
bf.results.reward[i] = new(big.Int)
}
return
}
Expand All @@ -125,7 +128,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
txIndex++
sumGasUsed += sorter[txIndex].gasUsed
}
bf.reward[i] = sorter[txIndex].reward
bf.results.reward[i] = sorter[txIndex].reward
}
}

Expand All @@ -134,7 +137,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
// also returned if requested and available.
// Note: an error is only returned if retrieving the head header has failed. If there are no
// retrievable blocks in the specified range then zero block count is returned with no error.
func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, uint64, int, error) {
func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) {
var (
headBlock rpc.BlockNumber
pendingBlock *types.Block
Expand Down Expand Up @@ -167,17 +170,6 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block
} else if pendingBlock == nil && lastBlock > headBlock {
return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock)
}
if maxHistory != 0 {
// limit retrieval to the given number of latest blocks
if tooOldCount := int64(headBlock) - int64(maxHistory) - int64(lastBlock) + int64(blocks); tooOldCount > 0 {
// tooOldCount is the number of requested blocks that are too old to be served
if int64(blocks) > tooOldCount {
blocks -= int(tooOldCount)
} else {
return nil, nil, 0, 0, nil
}
}
}
// ensure not trying to retrieve before genesis
if rpc.BlockNumber(blocks) > lastBlock+1 {
blocks = int(lastBlock + 1)
Expand All @@ -202,6 +194,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
if blocks < 1 {
return common.Big0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks
}
maxFeeHistory := oracle.maxHeaderHistory
if len(rewardPercentiles) != 0 {
maxFeeHistory = oracle.maxBlockHistory
}
if blocks > maxFeeHistory {
log.Warn("Sanitizing fee history length", "requested", blocks, "truncated", maxFeeHistory)
blocks = maxFeeHistory
Expand All @@ -214,17 +210,12 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p)
}
}
// Only process blocks if reward percentiles were requested
maxHistory := oracle.maxHeaderHistory
if len(rewardPercentiles) != 0 {
maxHistory = oracle.maxBlockHistory
}
var (
pendingBlock *types.Block
pendingReceipts []*types.Receipt
err error
)
pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks, maxHistory)
pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks)
if err != nil || blocks == 0 {
return common.Big0, nil, nil, nil, err
}
Expand All @@ -234,6 +225,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
next = oldestBlock
results = make(chan *blockFees, blocks)
)
percentileKey := make([]byte, 8*len(rewardPercentiles))
for i, p := range rewardPercentiles {
binary.LittleEndian.PutUint64(percentileKey[i*8:(i+1)*8], math.Float64bits(p))
}
for i := 0; i < maxBlockFetchers && i < blocks; i++ {
go func() {
for {
Expand All @@ -246,24 +241,38 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
fees := &blockFees{blockNumber: blockNumber}
if pendingBlock != nil && blockNumber >= pendingBlock.NumberU64() {
fees.block, fees.receipts = pendingBlock, pendingReceipts
fees.header = fees.block.Header()
oracle.processBlock(fees, rewardPercentiles)
results <- fees
} else {
if len(rewardPercentiles) != 0 {
fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber))
if fees.block != nil && fees.err == nil {
fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash())
}
cacheKey := struct {
number uint64
percentiles string
}{blockNumber, string(percentileKey)}

if p, ok := oracle.historyCache.Get(cacheKey); ok {
fees.results = p.(processedFees)
results <- fees
} else {
fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber))
if len(rewardPercentiles) != 0 {
fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber))
if fees.block != nil && fees.err == nil {
fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash())
fees.header = fees.block.Header()
}
} else {
fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber))
}
if fees.header != nil && fees.err == nil {
oracle.processBlock(fees, rewardPercentiles)
if fees.err == nil {
oracle.historyCache.Add(cacheKey, fees.results)
}
}
// send to results even if empty to guarantee that blocks items are sent in total
results <- fees
}
}
if fees.block != nil {
fees.header = fees.block.Header()
}
if fees.header != nil {
oracle.processBlock(fees, rewardPercentiles)
}
// send to results even if empty to guarantee that blocks items are sent in total
results <- fees
}
}()
}
Expand All @@ -279,8 +288,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
return common.Big0, nil, nil, nil, fees.err
}
i := int(fees.blockNumber - oldestBlock)
if fees.header != nil {
reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.reward, fees.baseFee, fees.nextBaseFee, fees.gasUsedRatio
if fees.results.baseFee != nil {
reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.results.reward, fees.results.baseFee, fees.results.nextBaseFee, fees.results.gasUsedRatio
} else {
// getting no block and no error means we are requesting into the future (might happen because of a reorg)
if i < firstMissing {
Expand Down
22 changes: 11 additions & 11 deletions eth/gasprice/feehistory_test.go
View file
Open in desktop
Expand Up @@ -36,20 +36,20 @@ func TestFeeHistory(t *testing.T) {
expCount int
expErr error
}{
{false, 0, 0, 10, 30, nil, 21, 10, nil},
{false, 0, 0, 10, 30, []float64{0, 10}, 21, 10, nil},
{false, 0, 0, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile},
{false, 0, 0, 1000000000, 30, nil, 0, 31, nil},
{false, 0, 0, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil},
{false, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead},
{true, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead},
{false, 1000, 1000, 10, 30, nil, 21, 10, nil},
{false, 1000, 1000, 10, 30, []float64{0, 10}, 21, 10, nil},
{false, 1000, 1000, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile},
{false, 1000, 1000, 1000000000, 30, nil, 0, 31, nil},
{false, 1000, 1000, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil},
{false, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead},
{true, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead},
{false, 20, 2, 100, rpc.LatestBlockNumber, nil, 13, 20, nil},
{false, 20, 2, 100, rpc.LatestBlockNumber, []float64{0, 10}, 31, 2, nil},
{false, 20, 2, 100, 32, []float64{0, 10}, 31, 2, nil},
{false, 0, 0, 1, rpc.PendingBlockNumber, nil, 0, 0, nil},
{false, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 1, nil},
{true, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 2, nil},
{true, 0, 0, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil},
{false, 1000, 1000, 1, rpc.PendingBlockNumber, nil, 0, 0, nil},
{false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil},
{true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil},
{true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil},
}
for i, c := range cases {
config := Config{
Expand Down
20 changes: 20 additions & 0 deletions eth/gasprice/gasprice.go
View file
Open in desktop
Expand Up @@ -23,10 +23,13 @@ import (
"sync"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
lru "github.com/hashicorp/golang-lru"
)

const sampleNumber = 3 // Number of transactions sampled in a block
Expand All @@ -53,6 +56,7 @@ type OracleBackend interface {
GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
PendingBlockAndReceipts() (*types.Block, types.Receipts)
ChainConfig() *params.ChainConfig
SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
}

// Oracle recommends gas prices based on the content of recent
Expand All @@ -68,6 +72,7 @@ type Oracle struct {

checkBlocks, percentile int
maxHeaderHistory, maxBlockHistory int
historyCache *lru.Cache
}

// NewOracle returns a new gasprice oracle which can recommend suitable
Expand Down Expand Up @@ -99,6 +104,20 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
} else if ignorePrice.Int64() > 0 {
log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice)
}

cache, _ := lru.New(2048)
headEvent := make(chan core.ChainHeadEvent, 1)
backend.SubscribeChainHeadEvent(headEvent)
go func() {
var lastHead common.Hash
for ev := range headEvent {
if ev.Block.ParentHash() != lastHead {
cache.Purge()
}
lastHead = ev.Block.Hash()
}
}()

return &Oracle{
backend: backend,
lastPrice: params.Default,
Expand All @@ -108,6 +127,7 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
percentile: percent,
maxHeaderHistory: params.MaxHeaderHistory,
maxBlockHistory: params.MaxBlockHistory,
historyCache: cache,
}
}

Expand Down
5 changes: 5 additions & 0 deletions eth/gasprice/gasprice_test.go
View file
Open in desktop
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
)
Expand Down Expand Up @@ -90,6 +91,10 @@ func (b *testBackend) ChainConfig() *params.ChainConfig {
return b.chain.Config()
}

func (b *testBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription {
return nil
}

func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBackend {
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
Expand Down