From e111830bc653116100d3052dfbd85cd606cbba89 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 15 Dec 2022 15:07:10 +0100 Subject: [PATCH 1/5] eth/tracers: exec block serially for native tracer Co-authored-by: Martin Holst Swende --- eth/tracers/api.go | 58 +++++++++++++++++++++++++++++++++++++----- eth/tracers/js/goja.go | 3 +++ eth/tracers/tracers.go | 6 +++++ 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 4436d13961f63..a15ec4366de9c 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -593,6 +593,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") } + // Prepare base state parent, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(block.NumberU64()-1), block.ParentHash()) if err != nil { return nil, err @@ -606,24 +607,68 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac return nil, err } defer release() + // JS tracers have high overhead. In this case run a parallel + // process that generates states in one thread and traces txes + // in separate worker threads. + if config.Tracer != nil && *config.Tracer != "" { + t, err := New(*config.Tracer, nil, nil) + if err != nil { + return nil, err + } + if _, ok := t.(JSTracer); ok { + return api.traceBlockParallel(ctx, block, statedb, config) + } + } + // Native tracers have low overhead + var ( + txs = block.Transactions() + blockHash = block.Hash() + is158 = api.backend.ChainConfig().IsEIP158(block.Number()) + blockCtx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) + results = make([]*txTraceResult, len(txs)) + ) + for i, tx := range txs { + // Generate the next state snapshot fast without tracing + msg, _ := tx.AsMessage(signer, block.BaseFee()) + txctx := &Context{ + BlockHash: blockHash, + TxIndex: i, + TxHash: tx.Hash(), + } + res, err := api.traceTx(ctx, msg, txctx, blockCtx, statedb, config) + if err != nil { + return nil, err + } + results[i] = &txTraceResult{Result: res} + // Finalize the state so any modifications are written to the trie + // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect + statedb.Finalise(is158) + } + return results, nil +} +// traceBlockParallel is for tracers that have a high overhead (read JS tracers). One thread +// runs along and executes txes without tracing enabled to generate their prestate. +// Worker threads take the tasks and the prestate and trace them. +func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, statedb *state.StateDB, config *TraceConfig) ([]*txTraceResult, error) { // Execute all the transaction contained within the block concurrently var ( - signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) - txs = block.Transactions() - results = make([]*txTraceResult, len(txs)) - pend sync.WaitGroup + txs = block.Transactions() + blockHash = block.Hash() + blockCtx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) + results = make([]*txTraceResult, len(txs)) + pend sync.WaitGroup ) threads := runtime.NumCPU() if threads > len(txs) { threads = len(txs) } jobs := make(chan *txTraceTask, threads) - blockHash := block.Hash() for th := 0; th < threads; th++ { pend.Add(1) go func() { - blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) defer pend.Done() // Fetch and execute the next transaction trace tasks for task := range jobs { @@ -645,7 +690,6 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // Feed the transactions into the tracers and return var failed error - blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) txloop: for i, tx := range txs { // Send the trace task over for execution diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index 9adfca9fb62a6..ba7f97914e042 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -350,6 +350,9 @@ func (t *jsTracer) Stop(err error) { t.vm.Interrupt(err) } +// IsJS returns whether this tracer evaluates JS code. +func (t *jsTracer) IsJS() bool { return true } + // onError is called anytime the running JS code is interrupted // and returns an error. It in turn pings the EVM to cancel its // execution. diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index 3d2d1256c0914..c84dc758556ba 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -42,6 +42,12 @@ type Tracer interface { Stop(err error) } +// JSTracer is implemented by tracers evaluating JS code. +type JSTracer interface { + Tracer + IsJS() bool +} + type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) var ( From 33d1555fb691ecc66a92d1b1bc1041c20f89d269 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 15 Dec 2022 16:33:59 +0100 Subject: [PATCH 2/5] minor fix --- eth/tracers/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index a15ec4366de9c..7b985cc95cce0 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -610,7 +610,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // JS tracers have high overhead. In this case run a parallel // process that generates states in one thread and traces txes // in separate worker threads. - if config.Tracer != nil && *config.Tracer != "" { + if config != nil && config.Tracer != nil && *config.Tracer != "" { t, err := New(*config.Tracer, nil, nil) if err != nil { return nil, err From 2c2b36ace3f1e8f5892662c0c8dcafd8757f59d3 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 19 Dec 2022 19:21:24 +0300 Subject: [PATCH 3/5] refactor tracers registeration --- core/vm/runtime/runtime_test.go | 6 +- eth/tracers/api.go | 8 +- .../internal/tracetest/calltrace_test.go | 6 +- .../internal/tracetest/prestate_test.go | 2 +- eth/tracers/js/goja.go | 21 +++-- eth/tracers/native/4byte.go | 2 +- eth/tracers/native/call.go | 2 +- eth/tracers/native/mux.go | 4 +- eth/tracers/native/noop.go | 2 +- eth/tracers/native/prestate.go | 2 +- eth/tracers/native/tracer.go | 79 ------------------- eth/tracers/tracers.go | 69 ++++++++++------ 12 files changed, 72 insertions(+), 131 deletions(-) delete mode 100644 eth/tracers/native/tracer.go diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index ab77e284df35f..868d41e503488 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -333,7 +333,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas if len(tracerCode) > 0 { - tracer, err := tracers.New(tracerCode, new(tracers.Context), nil) + tracer, err := tracers.DefaultDirectory.New(tracerCode, new(tracers.Context), nil) if err != nil { b.Fatal(err) } @@ -832,7 +832,7 @@ func TestRuntimeJSTracer(t *testing.T) { statedb.SetCode(common.HexToAddress("0xee"), calleeCode) statedb.SetCode(common.HexToAddress("0xff"), depressedCode) - tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) + tracer, err := tracers.DefaultDirectory.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -868,7 +868,7 @@ func TestJSTracerCreateTx(t *testing.T) { code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) + tracer, err := tracers.DefaultDirectory.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 7b985cc95cce0..ed5435140acd5 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -611,11 +611,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // process that generates states in one thread and traces txes // in separate worker threads. if config != nil && config.Tracer != nil && *config.Tracer != "" { - t, err := New(*config.Tracer, nil, nil) - if err != nil { - return nil, err - } - if _, ok := t.(JSTracer); ok { + if isJS := DefaultDirectory.IsJS(*config.Tracer); isJS { return api.traceBlockParallel(ctx, block, statedb, config) } } @@ -967,7 +963,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex // Default tracer is the struct logger tracer = logger.NewStructLogger(config.Config) if config.Tracer != nil { - tracer, err = New(*config.Tracer, txctx, config.TracerConfig) + tracer, err = DefaultDirectory.New(*config.Tracer, txctx, config.TracerConfig) if err != nil { return nil, err } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 0827d3b40e415..5cfb5b33c1b08 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -140,7 +140,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context), test.TracerConfig) + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -243,7 +243,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - tracer, err := tracers.New(tracerName, new(tracers.Context), nil) + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil) if err != nil { b.Fatalf("failed to create call tracer: %v", err) } @@ -309,7 +309,7 @@ func TestZeroValueToNotExitCall(t *testing.T) { } _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := tracers.New("callTracer", nil, nil) + tracer, err := tracers.DefaultDirectory.New("callTracer", nil, nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 9227aff9453da..2fee7d6fb7ed4 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -110,7 +110,7 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context), test.TracerConfig) + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index ba7f97914e042..9aff77fbf6830 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -45,7 +45,16 @@ func init() { if err != nil { panic(err) } - tracers.RegisterLookup(true, newJsTracer) + type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) + lookup := func(code string) ctorFn { + return func(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + return newJsTracer(code, ctx, cfg) + } + } + for name, code := range assetTracers { + tracers.DefaultDirectory.Register(name, lookup(code), true) + } + tracers.DefaultDirectory.RegisterJSEval(newJsTracer) } // bigIntProgram is compiled once and the exported function mostly invoked to convert @@ -122,16 +131,14 @@ type jsTracer struct { frameResultValue goja.Value } -// newJsTracer instantiates a new JS tracer instance. code is either -// the name of a built-in JS tracer or a Javascript snippet which -// evaluates to an expression returning an object with certain methods. +// newJsTracer instantiates a new JS tracer instance. code is a +// Javascript snippet which evaluates to an expression returning +// an object with certain methods: +// // The methods `result` and `fault` are required to be present. // The methods `step`, `enter`, and `exit` are optional, but note that // `enter` and `exit` always go together. func newJsTracer(code string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { - if c, ok := assetTracers[code]; ok { - code = c - } vm := goja.New() // By default field names are exported to JS as is, i.e. capitalized. vm.SetFieldNameMapper(goja.UncapFieldNameMapper()) diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index 00cd5fe77b8bf..c9747f1ad016c 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -28,7 +28,7 @@ import ( ) func init() { - register("4byteTracer", newFourByteTracer) + tracers.DefaultDirectory.Register("4byteTracer", newFourByteTracer, false) } // fourByteTracer searches for 4byte-identifiers, and collects them for post-processing. diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 24fd406398bb1..5bf49e744dec8 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -32,7 +32,7 @@ import ( //go:generate go run github.com/fjl/gencodec -type callFrame -field-override callFrameMarshaling -out gen_callframe_json.go func init() { - register("callTracer", newCallTracer) + tracers.DefaultDirectory.Register("callTracer", newCallTracer, false) } type callLog struct { diff --git a/eth/tracers/native/mux.go b/eth/tracers/native/mux.go index 878e2dc9d6d7a..db8ddd64380db 100644 --- a/eth/tracers/native/mux.go +++ b/eth/tracers/native/mux.go @@ -26,7 +26,7 @@ import ( ) func init() { - register("muxTracer", newMuxTracer) + tracers.DefaultDirectory.Register("muxTracer", newMuxTracer, false) } // muxTracer is a go implementation of the Tracer interface which @@ -47,7 +47,7 @@ func newMuxTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, er objects := make([]tracers.Tracer, 0, len(config)) names := make([]string, 0, len(config)) for k, v := range config { - t, err := tracers.New(k, ctx, v) + t, err := tracers.DefaultDirectory.New(k, ctx, v) if err != nil { return nil, err } diff --git a/eth/tracers/native/noop.go b/eth/tracers/native/noop.go index c1035bd1b7c6e..3beecd8abfed7 100644 --- a/eth/tracers/native/noop.go +++ b/eth/tracers/native/noop.go @@ -26,7 +26,7 @@ import ( ) func init() { - register("noopTracer", newNoopTracer) + tracers.DefaultDirectory.Register("noopTracer", newNoopTracer, false) } // noopTracer is a go implementation of the Tracer interface which diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 10008699bda3f..948d09ef767c1 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -32,7 +32,7 @@ import ( //go:generate go run github.com/fjl/gencodec -type account -field-override accountMarshaling -out gen_account_json.go func init() { - register("prestateTracer", newPrestateTracer) + tracers.DefaultDirectory.Register("prestateTracer", newPrestateTracer, false) } type state = map[common.Address]*account diff --git a/eth/tracers/native/tracer.go b/eth/tracers/native/tracer.go deleted file mode 100644 index f70d4b2af1ae4..0000000000000 --- a/eth/tracers/native/tracer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package native is a collection of tracers written in go. -// -// In order to add a native tracer and have it compiled into the binary, a new -// file needs to be added to this folder, containing an implementation of the -// `eth.tracers.Tracer` interface. -// -// Aside from implementing the tracer, it also needs to register itself, using the -// `register` method -- and this needs to be done in the package initialization. -// -// Example: -// -// func init() { -// register("noopTracerNative", newNoopTracer) -// } -package native - -import ( - "encoding/json" - "errors" - - "github.com/ethereum/go-ethereum/eth/tracers" -) - -// init registers itself this packages as a lookup for tracers. -func init() { - tracers.RegisterLookup(false, lookup) -} - -// ctorFn is the constructor signature of a native tracer. -type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) - -/* -ctors is a map of package-local tracer constructors. - -We cannot be certain about the order of init-functions within a package, -The go spec (https://golang.org/ref/spec#Package_initialization) says - -> To ensure reproducible initialization behavior, build systems -> are encouraged to present multiple files belonging to the same -> package in lexical file name order to a compiler. - -Hence, we cannot make the map in init, but must make it upon first use. -*/ -var ctors map[string]ctorFn - -// register is used by native tracers to register their presence. -func register(name string, ctor ctorFn) { - if ctors == nil { - ctors = make(map[string]ctorFn) - } - ctors[name] = ctor -} - -// lookup returns a tracer, if one can be matched to the given name. -func lookup(name string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { - if ctors == nil { - ctors = make(map[string]ctorFn) - } - if ctor, ok := ctors[name]; ok { - return ctor(ctx, cfg) - } - return nil, errors.New("no tracer found") -} diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index c84dc758556ba..b93f7db6f5706 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -19,7 +19,6 @@ package tracers import ( "encoding/json" - "errors" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/vm" @@ -42,37 +41,55 @@ type Tracer interface { Stop(err error) } -// JSTracer is implemented by tracers evaluating JS code. -type JSTracer interface { - Tracer - IsJS() bool +type ctorFn func(*Context, json.RawMessage) (Tracer, error) +type jsCtorFn func(string, *Context, json.RawMessage) (Tracer, error) + +type elem struct { + ctor ctorFn + isJS bool } -type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) +// DefaultDirectory is the collection of tracers bundled by default. +var DefaultDirectory = directory{elems: make(map[string]elem)} -var ( - lookups []lookupFunc -) +// directory provides functionality to lookup a tracer by name +// and a function to instantiate it. It falls back to a JS code evaluator +// if no tracer of the given name exists. +type directory struct { + elems map[string]elem + jsEval jsCtorFn +} -// RegisterLookup registers a method as a lookup for tracers, meaning that -// users can invoke a named tracer through that lookup. If 'wildcard' is true, -// then the lookup will be placed last. This is typically meant for interpreted -// engines (js) which can evaluate dynamic user-supplied code. -func RegisterLookup(wildcard bool, lookup lookupFunc) { - if wildcard { - lookups = append(lookups, lookup) - } else { - lookups = append([]lookupFunc{lookup}, lookups...) - } +// Register registers a method as a lookup for tracers, meaning that +// users can invoke a named tracer through that lookup. +func (d *directory) Register(name string, f ctorFn, isJS bool) { + d.elems[name] = elem{ctor: f, isJS: isJS} +} + +// RegisterJSEval registers a tracer that is able to parse +// dynamic user-provided JS code. +func (d *directory) RegisterJSEval(f jsCtorFn) { + d.jsEval = f } // New returns a new instance of a tracer, by iterating through the -// registered lookups. -func New(code string, ctx *Context, cfg json.RawMessage) (Tracer, error) { - for _, lookup := range lookups { - if tracer, err := lookup(code, ctx, cfg); err == nil { - return tracer, nil - } +// registered lookups. Name is either name of an existing tracer +// or an arbitrary JS code. +func (d *directory) New(name string, ctx *Context, cfg json.RawMessage) (Tracer, error) { + if elem, ok := d.elems[name]; ok { + return elem.ctor(ctx, cfg) + } + // Assume JS code + return d.jsEval(name, ctx, cfg) +} + +// IsJS will return true if the given tracer will evaluate +// JS code. Because code evaluation has high overhead, this +// info will be used in determining fast and slow code paths. +func (d *directory) IsJS(name string) bool { + if elem, ok := d.elems[name]; ok { + return elem.isJS } - return nil, errors.New("tracer not found") + // JS eval will execute JS code + return true } From 04c5b8101780e4c3a5887cde17ca72a7c928d54e Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 19 Dec 2022 20:01:23 +0100 Subject: [PATCH 4/5] minor --- eth/tracers/js/goja.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index 9aff77fbf6830..a83a1d829d0db 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -357,9 +357,6 @@ func (t *jsTracer) Stop(err error) { t.vm.Interrupt(err) } -// IsJS returns whether this tracer evaluates JS code. -func (t *jsTracer) IsJS() bool { return true } - // onError is called anytime the running JS code is interrupted // and returns an error. It in turn pings the EVM to cancel its // execution. From f1b29d307c5480d106d052754355c3e1d53f29f8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 21 Dec 2022 10:09:36 +0100 Subject: [PATCH 5/5] add blank line --- eth/tracers/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index ed5435140acd5..5a34d9d4a6abf 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -607,6 +607,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac return nil, err } defer release() + // JS tracers have high overhead. In this case run a parallel // process that generates states in one thread and traces txes // in separate worker threads.