diff --git a/README.fh.md b/README.fh.md index 519a1803a149e..bded088cdd65a 100644 --- a/README.fh.md +++ b/README.fh.md @@ -7,7 +7,7 @@ This is our Firehose instrumented fork of [ethereum/go-ethereum](https://github. The Firehose instrumentation have a protocol version for the messages exchanges with Firehose on Ethereum binary (`fireeth`). The protocols we currently develop are: -- Protocol `fh2.3` using the `firehose-fh2.3` branch and `fh2.3` tag(s) suffix +- Protocol `fh2.4` using the `firehose-fh2.4` branch and `fh2.4` tag(s) suffix - Protocol `fh3.0` using the `firehose-fh3.0` branch and `fh3.0` tag(s) suffix > [!NOTE] @@ -23,12 +23,12 @@ the same names and settings. ``` cd ~/work -git clone --branch="firehose-fh2.3" git@github.com:streamingfast/go-ethereum.git +git clone --branch="firehose-fh2.4" git@github.com:streamingfast/go-ethereum.git cd go-ethereum git remote rename origin sf -git checkout firehose-fh2.3 +git checkout firehose-fh2.4 git remote add origin https://github.com/ethereum/go-ethereum.git git remote add polygon https://github.com/maticnetwork/bor.git @@ -38,9 +38,9 @@ git fetch origin git fetch polygon git fetch bsc -git checkout release/geth-1.x-fh2.3 -git checkout release/bsc-1.x-fh2.2 -git checkout release/polygon-0.x-fh2.3 +git checkout release/geth-1.x-fh2.4 +git checkout release/bsc-1.x-fh2.5 +git checkout release/polygon-1.x-fh2.4 ``` ##### Assumptions @@ -60,7 +60,7 @@ for Firehose consumption. We use merging of the branches into one another to make that work manageable. The first and foremost important rule is that we always put new development -in the `firehose-fh2.3` branch. +in the `firehose-fh2.4` branch. This branch must always be tracking the lowest supported version of all. Indeed, this is our "work" branch for our patches, **new development must go there**. If you @@ -75,37 +75,37 @@ for us, versions that we will manages and deploy. Currently supported forks & version and the release branch -- `firehose-fh2.3` - Default branch with all Firehose commits in it, based on Geth `1.10.1`. -- [release/geth-1.x-fh2.3](https://github.com/streamingfast/go-ethereum/tree/release/geth-1.x-fh2.3) - Ethereum Geth, latest update for this branch is `1.10.7` ([https://github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum)). -- [release/polygon-0.x-fh2.3](https://github.com/streamingfast/go-ethereum/tree/release/polygon-0.x-fh2.3) - Polygon fork (a.k.a Matic), based on Geth `1.10.3`, latest update for this branch is `v0.3.2` ([https://github.com/maticnetwork/bor](https://github.com/maticnetwork/bor)). -- [release/bsc-1.x-fh2.2](https://github.com/streamingfast/go-ethereum/tree/release/bsc-1.x-fh2.2) - BSC fork (Binance), based on Geth `1.10.22`, latest update for this branch is `v1.1.18` ([https://github.com/binance-chain/bsc](https://github.com/binance-chain/bsc)). +- `firehose-fh2.4` - Default branch with all Firehose commits in it, based on Geth `1.10.1`. +- [release/geth-1.x-fh2.4](https://github.com/streamingfast/go-ethereum/tree/release/geth-1.x-fh2.4) - Ethereum Geth, latest update for this branch is `1.13.14` ([https://github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum)). +- [release/polygon-1.x-fh2.4](https://github.com/streamingfast/go-ethereum/tree/release/polygon-1.x-fh2.4) - Polygon fork (a.k.a Matic), based on Geth `1.13.5`, latest update for this branch is `v1.3.0` ([https://github.com/maticnetwork/bor](https://github.com/maticnetwork/bor)). +- [release/bsc-1.x-fh2.5](https://github.com/streamingfast/go-ethereum/tree/release/bsc-1.x-fh2.5) - BSC fork (Binance), based on Geth `1.13.5`, latest update for this branch is `v1.4.5` ([https://github.com/binance-chain/bsc](https://github.com/binance-chain/bsc)). > **Note** To find on which Geth version a particular fork is, you can do `git merge-base sf/release/geth-1.x-fh2.3 origin/master` where `origin/master` is the `master` branch of the original Geth repository (https://github.com/ethereum/go-ethereum). #### Making New Firehose Changes -Making new changes should be performed on the `firehose-fh2.3` branch. When happy -with the changes, simply merge the `firehose-fh2.3` branch in all the release branches we track +Making new changes should be performed on the `firehose-fh2.4` branch. When happy +with the changes, simply merge the `firehose-fh2.4` branch in all the release branches we track and support. - git checkout firehose-fh2.3 + git checkout firehose-fh2.4 git pull -p # Perform necessary changes, tests and commit(s) - git checkout release/geth-1.x-fh2.3 + git checkout release/geth-1.x-fh2.4 git pull -p - git merge firehose-fh2.3 + git merge firehose-fh2.4 - git checkout release/polygon-0.x-fh2.3 + git checkout release/polygon-1.x-fh2.4 git pull -p - git merge firehose-fh2.3 + git merge firehose-fh2.4 - git checkout release/bsc-1.x-fh2.3 + git checkout release/bsc-1.x-fh2.5 git pull -p - git merge firehose-fh2.3 + git merge firehose-fh2.4 - git push sf firehose-fh2.3 release/geth-1.x-fh2.3 release/polygon-0.x-fh2.3 release/bsc-1.x-fh2.3 + git push sf firehose-fh2.4 release/geth-1.x-fh2.4 release/polygon-1.x-fh2.4 release/bsc-1.x-fh2.4 ### Update to New Upstream Version @@ -120,7 +120,7 @@ those with your own values. First step is to checkout the release branch of the series you are currently updating to: - git checkout release/geth-1.x-fh2.3 + git checkout release/geth-1.x-fh2.4 git pull -p You first fetch the origin repository new data from Git: @@ -134,18 +134,18 @@ Then apply the update Solve conflicts if any. Once all conflicts have been resolved, commit then create a tag with release - git tag geth-v1.10.18-fh2.3 + git tag geth-v1.10.18-fh2.4 Then push all that to the repository: - git push sf release/geth-1.x-fh2.3 geth-v1.10.18-fh2.3 + git push sf release/geth-1.x-fh2.4 geth-v1.10.18-fh2.4 > [!NOTE] -> If you need to issue a Firehose bug fix for an existing version of upstream, for example a Firehose fix on `v1.10.8`, you append `-N` at the end where `N` is 1 then increments further is newer revisions are needed, so you would got tag `geth-v1.10.18-fh2.3-1` +> If you need to issue a Firehose bug fix for an existing version of upstream, for example a Firehose fix on `v1.10.8`, you append `-N` at the end where `N` is 1 then increments further is newer revisions are needed, so you would got tag `geth-v1.10.18-fh2.4-1` ### Development -All the *new* development should happen in the `firehose-fh2.3` branch, this is our own branch +All the *new* development should happen in the `firehose-fh2.4` branch, this is our own branch containing our commits. ##### Build Locally @@ -161,7 +161,7 @@ containing our commits. **Important** To correctly work, you need to use the right base branch, otherwise, it will be screwed up. The `firehose-v2` branch was based on `v1.10.1` at time of writing. -* From `gitk`: `gitk --first-parent v1.10.1..firehose-fh2.3` -* From terminal: `git log --decorate --pretty=oneline --abbrev-commit --first-parent=v1.10.1..firehose-fh2.3` -* From `GitHub`: [https://github.com/streamingfast/go-ethereum/compare/v1.10.1...firehose-fh2.3](https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-fh2.3) -* Modified files in our fork: `git diff --name-status v1.10.1..firehose-fh2.3 | grep -E "^M" | cut -d $'\t' -f 2` +* From `gitk`: `gitk --first-parent v1.10.1..firehose-fh2.4` +* From terminal: `git log --decorate --pretty=oneline --abbrev-commit --first-parent=v1.10.1..firehose-fh2.4` +* From `GitHub`: [https://github.com/streamingfast/go-ethereum/compare/v1.10.1...firehose-fh2.4](https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-fh2.4) +* Modified files in our fork: `git diff --name-status v1.10.1..firehose-fh2.4 | grep -E "^M" | cut -d $'\t' -f 2` diff --git a/core/blockchain.go b/core/blockchain.go index da3c6937ddc48..6aae75bd767eb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -633,7 +633,7 @@ func (bc *BlockChain) ProcessBlock(block *types.Block, parent *types.Header) (ty go func() { firehoseContext := firehose.NoOpContext if firehose.Enabled { - firehoseContext = firehose.NewSpeculativeExecutionContextWithBuffer(firehose.ParallelBlockSyncBuffer) + firehoseContext = firehose.NewBlockContextWithBuffer(firehose.ParallelBlockSyncBuffer) } parallelStatedb.StartPrefetcher("chain") @@ -655,7 +655,7 @@ func (bc *BlockChain) ProcessBlock(block *types.Block, parent *types.Header) (ty firehoseContext := firehose.NoOpContext if firehose.Enabled { - firehoseContext = firehose.NewSpeculativeExecutionContextWithBuffer(firehose.BlockSyncBuffer) + firehoseContext = firehose.NewBlockContextWithBuffer(firehose.BlockSyncBuffer) } receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig, ctx, firehoseContext) diff --git a/core/state_processor.go b/core/state_processor.go index f0b5349acdcdc..99bb4f78bd871 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -80,7 +80,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg txFirehoseContext := firehoseContext if txFirehoseContext.Enabled() { - txFirehoseContext = firehose.NewSpeculativeExecutionContextWithBuffer(firehose.TxSyncBuffer) + txFirehoseContext = firehose.NewTransactionContextWithBuffer(firehose.TxSyncBuffer) } var ( context = NewEVMBlockContext(header, p.bc, nil) diff --git a/firehose/context.go b/firehose/context.go index 4bc37cf6e257a..971aac5fd4682 100644 --- a/firehose/context.go +++ b/firehose/context.go @@ -49,14 +49,14 @@ func SyncContext() *Context { return syncContext } -func NewContext(printer Printer, speculative bool) *Context { +func NewContext(printer Printer, transactionScopedContext bool) *Context { ctx := &Context{ printer: printer, - isSpeculativeContext: speculative, - inBlock: atomic.NewBool(false), - inTransaction: atomic.NewBool(false), - totalOrderingCounter: atomic.NewUint64(0), + transactionScopedContext: transactionScopedContext, + inBlock: atomic.NewBool(false), + inTransaction: atomic.NewBool(false), + totalOrderingCounter: atomic.NewUint64(0), } ctx.resetBlock() @@ -73,8 +73,8 @@ type Context struct { printer Printer // Global state - isSpeculativeContext bool - flushTxLock sync.Mutex + transactionScopedContext bool + flushTxLock sync.Mutex // Block state inBlock *atomic.Bool @@ -117,8 +117,16 @@ func NewSpeculativeExecutionContext(initialAllocationInBytes int) *Context { return NewContext(NewToBufferPrinter(initialAllocationInBytes), true) } -// NewSpeculativeExecutionContextWithBuffer takes a buffer and cleans it before usage -func NewSpeculativeExecutionContextWithBuffer(buffer *bytes.Buffer) *Context { +// NewBlockContextWithBuffer creates a new block context with a buffer to accumulate the +// firehose logs. This should be used when tracing a block. +func NewBlockContextWithBuffer(buffer *bytes.Buffer) *Context { + return NewContext(NewToBufferPrinterWithBuffer(buffer), false) +} + +// NewTransactionContextWithBuffer creates a new transaction context with a buffer to accumulate the +// firehose logs. This should be used when tracing a standalone transaction that should later be +// either emitted or flushed to a block context. +func NewTransactionContextWithBuffer(buffer *bytes.Buffer) *Context { return NewContext(NewToBufferPrinterWithBuffer(buffer), true) } @@ -159,10 +167,11 @@ func (ctx *Context) RecordGenesisBlock(block *types.Block, recordGenesisAlloc fu ctx.EndTransaction(&types.Receipt{PostState: root[:]}) ctx.FinalizeBlock(block) ctx.EndBlock(block, block.Difficulty()) + ctx.FlushBlock() } func (ctx *Context) StartBlock(block *types.Block) { - if !ctx.inBlock.CAS(false, true) { + if !ctx.inBlock.CompareAndSwap(false, true) { panic("entering a block while already in a block scope") } @@ -207,7 +216,7 @@ func (ctx *Context) FlushBlock() { } // exitBlock is used when an abnormal condition is encountered while processing -// transactions and we must end the block processing right away, resetting the start +// transactions and we must end the block processing right away, resetting the state // along the way. func (ctx *Context) exitBlock() { if !ctx.inBlock.Load() { @@ -220,26 +229,6 @@ func (ctx *Context) exitBlock() { ctx.resetTransaction() } -// CancelBlock emit a Firehose CANCEL_BLOCK event that tells the console reader to discard any -// accumulated block's data and start over. This happens on certains error conditions where the block -// is actually invalid and will be re-processed by the chain so we should not record it. -func (ctx *Context) CancelBlock(block *types.Block, err error) { - if ctx == nil { - return - } - - // There is some particular runtime code path that could trigger a CANCEL_BLOCK without having started - // one, it's ok, the reader is resistant to such and here, we simply don't call `ExitBlock`. - if ctx.inBlock.Load() { - ctx.exitBlock() - } - - ctx.printer.Print("CANCEL_BLOCK", - Uint64(block.NumberU64()), - err.Error(), - ) -} - func (ctx *Context) StartSystemCall() { if ctx == nil { return @@ -249,7 +238,7 @@ func (ctx *Context) StartSystemCall() { panic("starting system call while not already within a block scope") } - if !ctx.inTransaction.CAS(false, true) { + if !ctx.inTransaction.CompareAndSwap(false, true) { panic("entering a system call while already in a transaction scope") } @@ -265,7 +254,7 @@ func (ctx *Context) EndSystemCall() { panic("ending system call while not already within a block scope") } - if !ctx.inTransaction.CAS(true, false) { + if !ctx.inTransaction.CompareAndSwap(true, false) { panic("ending a system call while not in a transaction scope") } @@ -363,17 +352,15 @@ func (ctx *Context) StartTransactionRaw( maxPriorityFeePerGas *big.Int, txType uint8, txIndex uint, - // The blob data gas used is statically computed for the transaction and there is no execution, - // so it's known already at transaction's start. - blobDataGasUsed uint64, - maxFeePerDataGas *big.Int, + blobGas uint64, + blobGasFeeCap *big.Int, blobHashes []common.Hash, ) { if ctx == nil { return } - if !ctx.inTransaction.CAS(false, true) { + if !ctx.inTransaction.CompareAndSwap(false, true) { panic("entering a transaction while already in a transaction scope") } @@ -393,9 +380,9 @@ func (ctx *Context) StartTransactionRaw( maxPriorityFeePerGasAsString = Hex(maxPriorityFeePerGas.Bytes()) } - maxFeePerDataGasAsString := "." - if maxFeePerDataGas != nil { - maxFeePerDataGasAsString = Hex(maxFeePerDataGas.Bytes()) + blobGasFeeCapAsString := "." + if blobGasFeeCap != nil { + blobGasFeeCapAsString = Hex(blobGasFeeCap.Bytes()) } blobHashesAsString := "." @@ -408,9 +395,6 @@ func (ctx *Context) StartTransactionRaw( blobHashesAsString = strings.Join(stringHashses, ",") } - // Fork is not active yet, so let's not modify the instrumentation just yet - _, _, _ = blobDataGasUsed, maxFeePerDataGasAsString, blobHashesAsString - ctx.printer.Print("BEGIN_APPLY_TRX", Hash(hash), toAsString, @@ -428,6 +412,9 @@ func (ctx *Context) StartTransactionRaw( Uint8(txType), Uint64(ctx.totalOrderingCounter.Inc()), Uint(txIndex), + Uint64(blobGas), + blobGasFeeCapAsString, + blobHashesAsString, ) } @@ -505,6 +492,8 @@ func (ctx *Context) EndTransaction(receipt *types.Receipt) { Uint64(receipt.CumulativeGasUsed), Hex(receipt.Bloom[:]), Uint64(ctx.totalOrderingCounter.Inc()), + Uint64(receipt.BlobGasUsed), + BigInt(receipt.BlobGasPrice), JSON(logItems), ) @@ -544,7 +533,7 @@ func (ctx *Context) openCall() string { } func (ctx *Context) callIndex() string { - if !ctx.isSpeculativeContext && !ctx.inBlock.Load() { + if !ctx.transactionScopedContext && !ctx.inBlock.Load() { debug.PrintStack() panic("should have been call in a block or in speculative context, something is deeply wrong") } @@ -818,45 +807,6 @@ func (ctx *Context) RecordNonceChange(addr common.Address, oldNonce, newNonce ui ) } -// Mempool methods - -func (ctx *Context) RecordTrxPool(eventType string, tx *types.Transaction, err error) { - if ctx == nil { - return - } - - signer := types.LatestSignerForChainID(tx.ChainId()) - - fromAsString := "." - from, err := types.Sender(signer, tx) - if err == nil { - fromAsString = Addr(from) - } - - toAsString := "." - if tx.To() != nil { - toAsString = Addr(*tx.To()) - } - - v, r, s := tx.RawSignatureValues() - - //todo: handle error message - ctx.printer.Print( - eventType, - Hash(tx.Hash()), - fromAsString, - toAsString, - Hex(tx.Value().Bytes()), - Hex(v.Bytes()), - Hex(r.Bytes()), - Hex(s.Bytes()), - Uint64(tx.Gas()), - Hex(tx.GasPrice().Bytes()), - Uint64(tx.Nonce()), - Hex(tx.Data()), - ) -} - type AccessList types.AccessList // marshal in a binary format that will be printed as hex in firehose and read on the console reader diff --git a/firehose/printer.go b/firehose/printer.go index e0980eec2cddf..396d8819744df 100644 --- a/firehose/printer.go +++ b/firehose/printer.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math/big" "os" "strconv" @@ -67,7 +66,7 @@ func flushToFirehose(in []byte, writer io.Writer) { } errstr := fmt.Sprintf("\nFIREHOSE FAILED WRITING %dx: %s\n", loops, err) - ioutil.WriteFile("/tmp/firehose_writer_failed_print.log", []byte(errstr), 0644) + os.WriteFile("/tmp/firehose_writer_failed_print.log", []byte(errstr), 0644) fmt.Fprint(writer, errstr) } diff --git a/params/version.go b/params/version.go index 4dd30f9afbfd0..7b6b50aca1d03 100644 --- a/params/version.go +++ b/params/version.go @@ -24,10 +24,10 @@ const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 3 // Minor version component of the current release VersionPatch = 0 // Patch version component of the current release - VersionMeta = "fh2.3" // Version metadata to append to the version string + VersionMeta = "fh2.4" // Version metadata to append to the version string FirehoseVersionMajor = 2 - FirehoseVersionMinor = 3 + FirehoseVersionMinor = 4 Variant = "polygon" )