From 4235d7fd3101b9a230f723eaa4423296beae53bd Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 17 Apr 2024 11:07:20 -0400 Subject: [PATCH 1/2] Updated branching information --- README.fh.md | 60 +++++++++++++++++++++++------------------------ params/version.go | 4 ++-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.fh.md b/README.fh.md index 519a1803a149e..bded088cdd65a 100644 --- a/README.fh.md +++ b/README.fh.md @@ -7,7 +7,7 @@ This is our Firehose instrumented fork of [ethereum/go-ethereum](https://github. The Firehose instrumentation have a protocol version for the messages exchanges with Firehose on Ethereum binary (`fireeth`). The protocols we currently develop are: -- Protocol `fh2.3` using the `firehose-fh2.3` branch and `fh2.3` tag(s) suffix +- Protocol `fh2.4` using the `firehose-fh2.4` branch and `fh2.4` tag(s) suffix - Protocol `fh3.0` using the `firehose-fh3.0` branch and `fh3.0` tag(s) suffix > [!NOTE] @@ -23,12 +23,12 @@ the same names and settings. ``` cd ~/work -git clone --branch="firehose-fh2.3" git@github.com:streamingfast/go-ethereum.git +git clone --branch="firehose-fh2.4" git@github.com:streamingfast/go-ethereum.git cd go-ethereum git remote rename origin sf -git checkout firehose-fh2.3 +git checkout firehose-fh2.4 git remote add origin https://github.com/ethereum/go-ethereum.git git remote add polygon https://github.com/maticnetwork/bor.git @@ -38,9 +38,9 @@ git fetch origin git fetch polygon git fetch bsc -git checkout release/geth-1.x-fh2.3 -git checkout release/bsc-1.x-fh2.2 -git checkout release/polygon-0.x-fh2.3 +git checkout release/geth-1.x-fh2.4 +git checkout release/bsc-1.x-fh2.5 +git checkout release/polygon-1.x-fh2.4 ``` ##### Assumptions @@ -60,7 +60,7 @@ for Firehose consumption. We use merging of the branches into one another to make that work manageable. The first and foremost important rule is that we always put new development -in the `firehose-fh2.3` branch. +in the `firehose-fh2.4` branch. This branch must always be tracking the lowest supported version of all. Indeed, this is our "work" branch for our patches, **new development must go there**. If you @@ -75,37 +75,37 @@ for us, versions that we will manages and deploy. Currently supported forks & version and the release branch -- `firehose-fh2.3` - Default branch with all Firehose commits in it, based on Geth `1.10.1`. -- [release/geth-1.x-fh2.3](https://github.com/streamingfast/go-ethereum/tree/release/geth-1.x-fh2.3) - Ethereum Geth, latest update for this branch is `1.10.7` ([https://github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum)). -- [release/polygon-0.x-fh2.3](https://github.com/streamingfast/go-ethereum/tree/release/polygon-0.x-fh2.3) - Polygon fork (a.k.a Matic), based on Geth `1.10.3`, latest update for this branch is `v0.3.2` ([https://github.com/maticnetwork/bor](https://github.com/maticnetwork/bor)). -- [release/bsc-1.x-fh2.2](https://github.com/streamingfast/go-ethereum/tree/release/bsc-1.x-fh2.2) - BSC fork (Binance), based on Geth `1.10.22`, latest update for this branch is `v1.1.18` ([https://github.com/binance-chain/bsc](https://github.com/binance-chain/bsc)). +- `firehose-fh2.4` - Default branch with all Firehose commits in it, based on Geth `1.10.1`. +- [release/geth-1.x-fh2.4](https://github.com/streamingfast/go-ethereum/tree/release/geth-1.x-fh2.4) - Ethereum Geth, latest update for this branch is `1.13.14` ([https://github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum)). +- [release/polygon-1.x-fh2.4](https://github.com/streamingfast/go-ethereum/tree/release/polygon-1.x-fh2.4) - Polygon fork (a.k.a Matic), based on Geth `1.13.5`, latest update for this branch is `v1.3.0` ([https://github.com/maticnetwork/bor](https://github.com/maticnetwork/bor)). +- [release/bsc-1.x-fh2.5](https://github.com/streamingfast/go-ethereum/tree/release/bsc-1.x-fh2.5) - BSC fork (Binance), based on Geth `1.13.5`, latest update for this branch is `v1.4.5` ([https://github.com/binance-chain/bsc](https://github.com/binance-chain/bsc)). > **Note** To find on which Geth version a particular fork is, you can do `git merge-base sf/release/geth-1.x-fh2.3 origin/master` where `origin/master` is the `master` branch of the original Geth repository (https://github.com/ethereum/go-ethereum). #### Making New Firehose Changes -Making new changes should be performed on the `firehose-fh2.3` branch. When happy -with the changes, simply merge the `firehose-fh2.3` branch in all the release branches we track +Making new changes should be performed on the `firehose-fh2.4` branch. When happy +with the changes, simply merge the `firehose-fh2.4` branch in all the release branches we track and support. - git checkout firehose-fh2.3 + git checkout firehose-fh2.4 git pull -p # Perform necessary changes, tests and commit(s) - git checkout release/geth-1.x-fh2.3 + git checkout release/geth-1.x-fh2.4 git pull -p - git merge firehose-fh2.3 + git merge firehose-fh2.4 - git checkout release/polygon-0.x-fh2.3 + git checkout release/polygon-1.x-fh2.4 git pull -p - git merge firehose-fh2.3 + git merge firehose-fh2.4 - git checkout release/bsc-1.x-fh2.3 + git checkout release/bsc-1.x-fh2.5 git pull -p - git merge firehose-fh2.3 + git merge firehose-fh2.4 - git push sf firehose-fh2.3 release/geth-1.x-fh2.3 release/polygon-0.x-fh2.3 release/bsc-1.x-fh2.3 + git push sf firehose-fh2.4 release/geth-1.x-fh2.4 release/polygon-1.x-fh2.4 release/bsc-1.x-fh2.4 ### Update to New Upstream Version @@ -120,7 +120,7 @@ those with your own values. First step is to checkout the release branch of the series you are currently updating to: - git checkout release/geth-1.x-fh2.3 + git checkout release/geth-1.x-fh2.4 git pull -p You first fetch the origin repository new data from Git: @@ -134,18 +134,18 @@ Then apply the update Solve conflicts if any. Once all conflicts have been resolved, commit then create a tag with release - git tag geth-v1.10.18-fh2.3 + git tag geth-v1.10.18-fh2.4 Then push all that to the repository: - git push sf release/geth-1.x-fh2.3 geth-v1.10.18-fh2.3 + git push sf release/geth-1.x-fh2.4 geth-v1.10.18-fh2.4 > [!NOTE] -> If you need to issue a Firehose bug fix for an existing version of upstream, for example a Firehose fix on `v1.10.8`, you append `-N` at the end where `N` is 1 then increments further is newer revisions are needed, so you would got tag `geth-v1.10.18-fh2.3-1` +> If you need to issue a Firehose bug fix for an existing version of upstream, for example a Firehose fix on `v1.10.8`, you append `-N` at the end where `N` is 1 then increments further is newer revisions are needed, so you would got tag `geth-v1.10.18-fh2.4-1` ### Development -All the *new* development should happen in the `firehose-fh2.3` branch, this is our own branch +All the *new* development should happen in the `firehose-fh2.4` branch, this is our own branch containing our commits. ##### Build Locally @@ -161,7 +161,7 @@ containing our commits. **Important** To correctly work, you need to use the right base branch, otherwise, it will be screwed up. The `firehose-v2` branch was based on `v1.10.1` at time of writing. -* From `gitk`: `gitk --first-parent v1.10.1..firehose-fh2.3` -* From terminal: `git log --decorate --pretty=oneline --abbrev-commit --first-parent=v1.10.1..firehose-fh2.3` -* From `GitHub`: [https://github.com/streamingfast/go-ethereum/compare/v1.10.1...firehose-fh2.3](https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-fh2.3) -* Modified files in our fork: `git diff --name-status v1.10.1..firehose-fh2.3 | grep -E "^M" | cut -d $'\t' -f 2` +* From `gitk`: `gitk --first-parent v1.10.1..firehose-fh2.4` +* From terminal: `git log --decorate --pretty=oneline --abbrev-commit --first-parent=v1.10.1..firehose-fh2.4` +* From `GitHub`: [https://github.com/streamingfast/go-ethereum/compare/v1.10.1...firehose-fh2.4](https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-fh2.4) +* Modified files in our fork: `git diff --name-status v1.10.1..firehose-fh2.4 | grep -E "^M" | cut -d $'\t' -f 2` diff --git a/params/version.go b/params/version.go index 29350f0f4e93c..cb84a72fbdff6 100644 --- a/params/version.go +++ b/params/version.go @@ -24,10 +24,10 @@ const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release VersionPatch = 1 // Patch version component of the current release - VersionMeta = "fh2.3" // Version metadata to append to the version string + VersionMeta = "fh2.4" // Version metadata to append to the version string FirehoseVersionMajor = 2 - FirehoseVersionMinor = 3 + FirehoseVersionMinor = 4 Variant = "geth" ) From 2da73413e1aea94bffc569baa238d489936901e8 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 17 Apr 2024 12:53:28 -0400 Subject: [PATCH 2/2] Added missing 2.4 Firehose methods (system_call_start/end), renamed some unclear calls --- core/blockchain.go | 2 +- core/state_processor.go | 2 +- firehose/context.go | 67 +++++++++++++++++++++++++++-------------- firehose/printer.go | 3 +- 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a9aefe07c4e2f..9d7d4ddace94f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1972,7 +1972,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // Process block using the parent state as reference point firehoseContext := firehose.NoOpContext if firehose.Enabled { - firehoseContext = firehose.NewSpeculativeExecutionContextWithBuffer(firehose.BlockSyncBuffer) + firehoseContext = firehose.NewBlockContextWithBuffer(firehose.BlockSyncBuffer) } substart := time.Now() diff --git a/core/state_processor.go b/core/state_processor.go index b2574638cc058..32103a0a156f5 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -76,7 +76,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg txFirehoseContext := firehoseContext if txFirehoseContext.Enabled() { - txFirehoseContext = firehose.NewSpeculativeExecutionContextWithBuffer(firehose.TxSyncBuffer) + txFirehoseContext = firehose.NewTransactionContextWithBuffer(firehose.TxSyncBuffer) } blockContext := NewEVMBlockContext(header, p.bc, nil) diff --git a/firehose/context.go b/firehose/context.go index 3e65e6cfd1c69..c01869103c017 100644 --- a/firehose/context.go +++ b/firehose/context.go @@ -48,14 +48,14 @@ func SyncContext() *Context { return syncContext } -func NewContext(printer Printer, speculative bool) *Context { +func NewContext(printer Printer, transactionScopedContext bool) *Context { ctx := &Context{ printer: printer, - isSpeculativeContext: speculative, - inBlock: atomic.NewBool(false), - inTransaction: atomic.NewBool(false), - totalOrderingCounter: atomic.NewUint64(0), + transactionScopedContext: transactionScopedContext, + inBlock: atomic.NewBool(false), + inTransaction: atomic.NewBool(false), + totalOrderingCounter: atomic.NewUint64(0), } ctx.resetBlock() @@ -72,8 +72,8 @@ type Context struct { printer Printer // Global state - isSpeculativeContext bool - flushTxLock sync.Mutex + transactionScopedContext bool + flushTxLock sync.Mutex // Block state inBlock *atomic.Bool @@ -112,7 +112,16 @@ func NewSpeculativeExecutionContext(initialAllocationInBytes int) *Context { return NewContext(NewToBufferPrinter(initialAllocationInBytes), true) } -func NewSpeculativeExecutionContextWithBuffer(buffer *bytes.Buffer) *Context { +// NewBlockContextWithBuffer creates a new block context with a buffer to accumulate the +// firehose logs. This should be used when tracing a block. +func NewBlockContextWithBuffer(buffer *bytes.Buffer) *Context { + return NewContext(NewToBufferPrinterWithBuffer(buffer), false) +} + +// NewTransactionContextWithBuffer creates a new transaction context with a buffer to accumulate the +// firehose logs. This should be used when tracing a standalone transaction that should later be +// either emitted or flushed to a block context. +func NewTransactionContextWithBuffer(buffer *bytes.Buffer) *Context { return NewContext(NewToBufferPrinterWithBuffer(buffer), true) } @@ -153,6 +162,7 @@ func (ctx *Context) RecordGenesisBlock(block *types.Block, recordGenesisAlloc fu ctx.EndTransaction(&types.Receipt{PostState: root[:]}) ctx.FinalizeBlock(block) ctx.EndBlock(block, block.Difficulty()) + ctx.FlushBlock() } func (ctx *Context) StartBlock(block *types.Block) { @@ -198,7 +208,7 @@ func (ctx *Context) FlushBlock() { } // exitBlock is used when an abnormal condition is encountered while processing -// transactions and we must end the block processing right away, resetting the start +// transactions and we must end the block processing right away, resetting the state // along the way. func (ctx *Context) exitBlock() { if !ctx.inBlock.Load() { @@ -211,24 +221,37 @@ func (ctx *Context) exitBlock() { ctx.resetTransaction() } -// CancelBlock emit a Firehose CANCEL_BLOCK event that tells the console reader to discard any -// accumulated block's data and start over. This happens on certains error conditions where the block -// is actually invalid and will be re-processed by the chain so we should not record it. -func (ctx *Context) CancelBlock(block *types.Block, err error) { +func (ctx *Context) StartSystemCall() { if ctx == nil { return } - // There is some particular runtime code path that could trigger a CANCEL_BLOCK without having started - // one, it's ok, the reader is resistant to such and here, we simply don't call `ExitBlock`. - if ctx.inBlock.Load() { - ctx.exitBlock() + if !ctx.inBlock.Load() { + panic("starting system call while not already within a block scope") } - ctx.printer.Print("CANCEL_BLOCK", - Uint64(block.NumberU64()), - err.Error(), - ) + if !ctx.inTransaction.CAS(false, true) { + panic("entering a system call while already in a transaction scope") + } + + ctx.printer.Print("SYSTEM_CALL_START") +} + +func (ctx *Context) EndSystemCall() { + if ctx == nil { + return + } + + if !ctx.inBlock.Load() { + panic("ending system call while not already within a block scope") + } + + if !ctx.inTransaction.CAS(true, false) { + panic("ending a system call while not in a transaction scope") + } + + ctx.resetTransaction() + ctx.printer.Print("SYSTEM_CALL_END") } // Transaction methods @@ -437,7 +460,7 @@ func (ctx *Context) openCall() string { } func (ctx *Context) callIndex() string { - if !ctx.isSpeculativeContext && !ctx.inBlock.Load() { + if !ctx.transactionScopedContext && !ctx.inBlock.Load() { debug.PrintStack() panic("should have been call in a block or in speculative context, something is deeply wrong") } diff --git a/firehose/printer.go b/firehose/printer.go index bd6a0df2cbb26..c540331342d90 100644 --- a/firehose/printer.go +++ b/firehose/printer.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math/big" "os" "strconv" @@ -67,7 +66,7 @@ func flushToFirehose(in []byte, writer io.Writer) { } errstr := fmt.Sprintf("\nFIREHOSE FAILED WRITING %dx: %s\n", loops, err) - ioutil.WriteFile("/tmp/firehose_writer_failed_print.log", []byte(errstr), 0644) + os.WriteFile("/tmp/firehose_writer_failed_print.log", []byte(errstr), 0644) fmt.Fprint(writer, errstr) }