optimism: historical Bedrock geth rollup changes

This commit squashes the op-geth fork history into a more maintainable
diff for rebasing upon upstream geth.

reference-optimistic-geth changes (origins of op-geth in early Bedrock
development stage):
- Deposit TX Type
- Enable deposit tx in EVM/tx pool
- Change deposit nonce to not be the max nonce
- Extend PayloadAttributesV1 with a Transactions field
- Force deposits at the start of each L2 block
- Fix height check
- noTxPool flag, reproduce block in verifier mode without tx pool interference
- Fix RPC json marshalling (ref op-geth PR 4)
- Deposit txs block height check in block body validation (ref op-geth PR 5)
- core: do not try to reinject deposit txs into tx-pool (ref-op-geth PR 6)
- deposit source hash field instead of L2 block height and tx index combination
- Include invalid deposits, rewind state, but always persist mint (ethereum#10)
- Provide gas to Call/Create in deposit transactions (ethereum#12)
- Add docker builds (ref-op-geth PR 16, 17)
- Don't panic on deposit transaction signature values or chain ID (ref-op-geth PR 18)
- core: Add version to DepositTx (ref-op-geth PR 19)
- Enable Geth build/lint/test in CircleCI (ref-op-geth PR 23)
- core: Include guaranteed gas in the gas pool (ref-op-geth PR 21)
- core: handle base fee, l1 availability fee, tx fee (ref-op-geth PR 27)
- fix: deposit tx hash
- fix l1 fee cache, rpc, tracing and tx pool
- core: remove deposit-tx sub-type (a.k.a. deposit version byte)
- eth/catalyst: allow engine user to reorg own chain
- miner: restore ability to reorg deep as block builder
- params: print Optimism consensus type in banner
- core/types: remove unused protected() method, see upstream PR 23376
- core: do not mutate original balance value in tx pool l1 cost adjustment
- core: subtract deposit gas from pool, so other txs do not use the same gas. And fail tx processing if deposits reach gas limit
- core/types: deposits do not tip, avoid basefee subtraction
- Unmeter the L1 Attributes Transaction
- miner: handle force tx errors as critical, clean up diff
- ci: Switch branch
- eth,miner: return STATUS_INVALID when failing to process forced transactions in request (ref-op-geth PR 40)
- verifier: forward tx to sequencer based on flag
- txpool: add flag to disable tx gossip (ref-op-geth PR 42)
- Add op-geth version in addition to geth version (ref-op-geth PR 43)
- ci: CircleCI improvements (ref-op-geth PR 44)
- Rename to op-geth
- Build latest tag on optimism branch

op-geth changes:
- Expose cache config in simulated backend (ethereum#2)
- Add EIP-1559 parameters
- eth/catalyst: update payload id computation (#1)
- make eip1559 configurable (ethereum#4)
- post-merge network should not log warnings about missing transition information (ethereum#5)
- Make the simulator more configurable (ethereum#6)
- fix OPB-6 - IsDepositTx check instead of artificial nonce value check (ethereum#7)
- Simulated backend - enable proof of stake consensus type and fix performance issue (ethereum#8)
- accounts: simulated backend consensus engine option and immediate tx indexing
- consensus/beacon: recognize all blocks as reached TTD with 0 TTD in chain config
- Add --rollup.historicalhttp CLI flag and fix backend iface
- Flags and interfaces for historical RPC requests (ethereum#12)
- Redirect historical RPC requests (ethereum#13)
- Use the pre-existing ethereum.NotFound error (ethereum#18)
- Add historical endpoint to TraceBlockByNumber and TraceBlockByHash (ethereum#19)
- Add historical endpoint to TraceTransaction (ethereum#20)
- Add historical endpoint to TraceCall (ethereum#21)
- optimism: fee params from info txi, update l1 cost func GPO params read (ethereum#15)
- add hardcoded addresses for fee payouts (ethereum#23)
- dynamic gas limit via engine API (ethereum#22)

Co-authored-by: Matthew Slipper <me@matthewslipper.com>
Co-authored-by: Joshua Gutow <jgutow@oplabs.co>
Co-authored-by: protolambda <proto@protolambda.com>
Co-authored-by: Mark Tyneway <mark.tyneway@gmail.com>
Co-authored-by: Maurelian <maurelian@protonmail.ch>

.circleci/config.yml

@@ -0,0 +1,71 @@
+version: 2.1
+
+jobs:
+  build-geth:
+    docker:
+      - image: cimg/go:1.18
+    resource_class: medium
+    steps:
+      - checkout
+      - run:
+          command: go run build/ci.go install
+  unit-test:
+    resource_class: medium
+    docker:
+      - image: cimg/go:1.18
+    steps:
+      - checkout
+      - run:
+          command: go run build/ci.go test
+  lint-geth:
+    resource_class: medium
+    docker:
+      - image: cimg/go:1.18
+    steps:
+      - checkout
+      - run:
+          command: go run build/ci.go lint
+
+  push-geth:
+    docker:
+      - image: cimg/base:2022.04
+    steps:
+      - when:
+          condition:
+            or:
+              - equal: [ optimism, <<pipeline.git.branch>> ]
+              - equal: [ optimism-history, <<pipeline.git.branch>> ]
+          steps:
+            - checkout
+            - setup_remote_docker:
+                version: 20.10.12
+            - run:
+                name: Build and push
+                command: |
+                  echo "$DOCKER_PASS" | docker login -u "$DOCKER_USERNAME" --password-stdin
+                  docker build -t "ethereumoptimism/op-geth:$CIRCLE_SHA1" -f Dockerfile .
+                  docker tag "ethereumoptimism/op-geth:$CIRCLE_SHA1" "ethereumoptimism/op-geth:$CIRCLE_BRANCH"
+                  docker push "ethereumoptimism/op-geth:$CIRCLE_SHA1"
+                  docker push "ethereumoptimism/op-geth:$CIRCLE_BRANCH"
+
+                  if [ $CIRCLE_BRANCH = "optimism" ]
+                  then
+                    docker tag "ethereumoptimism/op-geth:$CIRCLE_SHA1" "ethereumoptimism/op-geth:latest"
+                    docker push "ethereumoptimism/op-geth:latest"
+                  pwd
+                  fi
+      # Below step is required to prevent CircleCI from barfing on a
+      # job with no steps
+      - run: echo 0
+
+workflows:
+  main:
+    jobs:
+      - build-geth:
+          name: Build geth
+      - unit-test:
+          name: Run unit tests for geth
+      - lint-geth:
+          name: Run linter over geth
+      - push-geth:
+          name: Push geth

accounts/abi/bind/backends/simulated.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/bloombits"
@@ -63,6 +64,8 @@ type SimulatedBackend struct {
 	database   ethdb.Database   // In memory database to store our testing data
 	blockchain *core.BlockChain // Ethereum blockchain to handle the consensus
 
+	consensus consensus.Engine
+
 	mu              sync.Mutex
 	pendingBlock    *types.Block   // Currently pending block that will be imported on request
 	pendingState    *state.StateDB // Currently pending state that will be the active on request
@@ -78,34 +81,100 @@ type SimulatedBackend struct {
 // and uses a simulated blockchain for testing purposes.
 // A simulated backend always uses chainID 1337.
 func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
-	genesis := core.Genesis{
-		Config:   params.AllEthashProtocolChanges,
-		GasLimit: gasLimit,
-		Alloc:    alloc,
+	return NewSimulatedBackendWithOpts(WithDatabase(database), WithAlloc(alloc), WithGasLimit(gasLimit))
+}
+
+// NewSimulatedBackend creates a new binding backend using a simulated blockchain
+// for testing purposes.
+// A simulated backend always uses chainID 1337.
+func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
+	return NewSimulatedBackendWithOpts(WithGasLimit(gasLimit), WithAlloc(alloc))
+}
+
+type simulatedBackendConfig struct {
+	genesis     core.Genesis
+	cacheConfig *core.CacheConfig
+	database    ethdb.Database
+	vmConfig    vm.Config
+	consensus   consensus.Engine
+}
+
+type SimulatedBackendOpt func(s *simulatedBackendConfig)
+
+func WithDatabase(database ethdb.Database) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.database = database
+	}
+}
+
+func WithGasLimit(gasLimit uint64) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.genesis.GasLimit = gasLimit
+	}
+}
+
+func WithAlloc(alloc core.GenesisAlloc) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.genesis.Alloc = alloc
+	}
+}
+
+func WithCacheConfig(cacheConfig *core.CacheConfig) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.cacheConfig = cacheConfig
 	}
-	blockchain, _ := core.NewBlockChain(database, nil, &genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
+}
+
+func WithGenesis(genesis core.Genesis) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.genesis = genesis
+	}
+}
+
+func WithVMConfig(vmConfig vm.Config) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.vmConfig = vmConfig
+	}
+}
+
+func WithConsensus(consensus consensus.Engine) SimulatedBackendOpt {
+	return func(s *simulatedBackendConfig) {
+		s.consensus = consensus
+	}
+}
+
+// NewSimulatedBackendWithOpts creates a new binding backend based on the given database
+// and uses a simulated blockchain for testing purposes. It exposes additional configuration
+// options that are useful to
+func NewSimulatedBackendWithOpts(opts ...SimulatedBackendOpt) *SimulatedBackend {
+	config := &simulatedBackendConfig{
+		genesis:   core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: 100000000, Alloc: make(core.GenesisAlloc)},
+		database:  rawdb.NewMemoryDatabase(),
+		consensus: ethash.NewFaker(),
+	}
+
+	for _, opt := range opts {
+		opt(config)
+	}
+
+	config.genesis.MustCommit(config.database)
+	blockchain, _ := core.NewBlockChain(config.database, config.cacheConfig, &config.genesis, nil, config.consensus, config.vmConfig, nil, nil)
 
 	backend := &SimulatedBackend{
-		database:   database,
+		database:   config.database,
 		blockchain: blockchain,
-		config:     genesis.Config,
+		config:     config.genesis.Config,
+		consensus:  config.consensus,
 	}
 
-	filterBackend := &filterBackend{database, blockchain, backend}
+	filterBackend := &filterBackend{config.database, blockchain, backend}
 	backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{})
 	backend.events = filters.NewEventSystem(backend.filterSystem, false)
 
 	backend.rollback(blockchain.CurrentBlock())
 	return backend
 }
 
-// NewSimulatedBackend creates a new binding backend using a simulated blockchain
-// for testing purposes.
-// A simulated backend always uses chainID 1337.
-func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
-	return NewSimulatedBackendWithDatabase(rawdb.NewMemoryDatabase(), alloc, gasLimit)
-}
-
 // Close terminates the underlying blockchain's update loop.
 func (b *SimulatedBackend) Close() error {
 	b.blockchain.Stop()
@@ -121,6 +190,8 @@ func (b *SimulatedBackend) Commit() common.Hash {
 	if _, err := b.blockchain.InsertChain([]*types.Block{b.pendingBlock}); err != nil {
 		panic(err) // This cannot happen unless the simulator is wrong, fail in that case
 	}
+	// Don't wait for the async tx indexing
+	rawdb.WriteTxLookupEntriesByBlock(b.database, b.pendingBlock)
 	blockHash := b.pendingBlock.Hash()
 
 	// Using the last inserted block here makes it possible to build on a side
@@ -139,7 +210,7 @@ func (b *SimulatedBackend) Rollback() {
 }
 
 func (b *SimulatedBackend) rollback(parent *types.Block) {
-	blocks, _ := core.GenerateChain(b.config, parent, ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {})
+	blocks, _ := core.GenerateChain(b.config, parent, b.consensus, b.database, 1, func(int, *core.BlockGen) {})
 
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), b.blockchain.StateCache(), nil)
@@ -646,6 +717,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 
 	txContext := core.NewEVMTxContext(msg)
 	evmContext := core.NewEVMBlockContext(block.Header(), b.blockchain, nil)
+	evmContext.L1CostFunc = types.NewL1CostFunc(b.config, stateDB)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
 	vmEnv := vm.NewEVM(evmContext, txContext, stateDB, b.config, vm.Config{NoBaseFee: true})
@@ -675,7 +747,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		return fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce)
 	}
 	// Include tx in chain
-	blocks, receipts := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
+	blocks, receipts := core.GenerateChain(b.config, block, b.consensus, b.database, 1, func(number int, block *core.BlockGen) {
 		for _, tx := range b.pendingBlock.Transactions() {
 			block.AddTxWithChain(b.blockchain, tx)
 		}
@@ -799,7 +871,7 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 		return fmt.Errorf("could not find parent")
 	}
 
-	blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
+	blocks, _ := core.GenerateChain(b.config, block, b.consensus, b.database, 1, func(number int, block *core.BlockGen) {
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
 	stateDB, _ := b.blockchain.State()
@@ -831,6 +903,10 @@ func (m callMsg) Gas() uint64                  { return m.CallMsg.Gas }
 func (m callMsg) Value() *big.Int              { return m.CallMsg.Value }
 func (m callMsg) Data() []byte                 { return m.CallMsg.Data }
 func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList }
+func (m callMsg) IsSystemTx() bool             { return false }
+func (m callMsg) IsDepositTx() bool            { return false }
+func (m callMsg) Mint() *big.Int               { return nil }
+func (m callMsg) RollupDataGas() uint64        { return 0 }
 
 // filterBackend implements filters.Backend to support filtering for logs without
 // taking bloom-bits acceleration structures into account.

beacon/engine/types.go

@@ -34,12 +34,23 @@ type PayloadAttributes struct {
 	Timestamp             uint64              `json:"timestamp"             gencodec:"required"`
 	Random                common.Hash         `json:"prevRandao"            gencodec:"required"`
 	SuggestedFeeRecipient common.Address      `json:"suggestedFeeRecipient" gencodec:"required"`
-	Withdrawals           []*types.Withdrawal `json:"withdrawals"`
+	Withdrawals           []*types.Withdrawal `json:"withdrawals,omitempty" gencodec:"optional"`
+
+	// Transactions is a field for rollups: the transactions list is forced into the block
+	Transactions [][]byte `json:"transactions,omitempty"  gencodec:"optional"`
+	// NoTxPool is a field for rollups: if true, the no transactions are taken out of the tx-pool,
+	// only transactions from the above Transactions list will be included.
+	NoTxPool bool `json:"noTxPool,omitempty" gencodec:"optional"`
+	// GasLimit is a field for rollups: if set, this sets the exact gas limit the block produced with.
+	GasLimit *uint64 `json:"gasLimit,omitempty" gencodec:"optional"`
 }
 
 // JSON type overrides for PayloadAttributes.
 type payloadAttributesMarshaling struct {
 	Timestamp hexutil.Uint64
+
+	Transactions []hexutil.Bytes
+	GasLimit     *hexutil.Uint64
 }
 
 //go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go

build/ci.go

@@ -512,7 +512,7 @@ func doDocker(cmdline []string) {
 	case env.Branch == "master":
 		tags = []string{"latest"}
 	case strings.HasPrefix(env.Tag, "v1."):
-		tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version}
+		tags = []string{"stable", fmt.Sprintf("release-1.%d", params.OPVersionMinor), "v" + params.Version}
 	}
 	// If architecture specific image builds are requested, build and push them
 	if *image {

cmd/geth/main.go

@@ -149,6 +149,9 @@ var (
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
 		utils.MinerNotifyFullFlag,
+		utils.RollupSequencerHTTPFlag,
+		utils.RollupHistoricalRPCFlag,
+		utils.RollupDisableTxPoolGossipFlag,
 		configFileFlag,
 	}, utils.NetworkFlags, utils.DatabasePathFlags)
 

cmd/geth/misccmd.go

@@ -137,6 +137,7 @@ func printVersion(ctx *cli.Context) error {
 	if git.Date != "" {
 		fmt.Println("Git Commit Date:", git.Date)
 	}
+	fmt.Println("Upstream Version:", params.GethVersionWithMeta)
 	fmt.Println("Architecture:", runtime.GOARCH)
 	fmt.Println("Go Version:", runtime.Version())
 	fmt.Println("Operating System:", runtime.GOOS)