Merge pull request #83 from vulcanize/marshall-binary

Update RPC api and backend to support the EIP-2930 and EIP-1559.

db/migrations/00014_create_stored_functions.sql

@@ -1,14 +1,14 @@
 -- +goose Up
 -- +goose StatementBegin
--- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
+-- returns if a storage node at the provided path was removed in the range >= the provided height and <= the provided block hash
 CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
 AS $$
 SELECT exists(SELECT 1
               FROM eth.storage_cids
                        INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
                        INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
               WHERE storage_path = path
-                AND block_number > height
+                AND block_number >= height
                 AND block_number <= (SELECT block_number
                                      FROM eth.header_cids
                                      WHERE block_hash = hash)

go.mod

@@ -4,17 +4,17 @@ go 1.13
 
 require (
 	github.com/ethereum/go-ethereum v1.9.25
-	github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e
+	github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.8.0
-	github.com/machinebox/graphql v0.2.2 // indirect
+	github.com/lib/pq v1.10.2
+	github.com/machinebox/graphql v0.2.2
 	github.com/multiformats/go-multihash v0.0.14
-	github.com/onsi/ginkgo v1.15.0
+	github.com/onsi/ginkgo v1.16.2
 	github.com/onsi/gomega v1.10.1
 	github.com/prometheus/client_golang v1.5.1
 	github.com/shirou/gopsutil v3.21.5+incompatible // indirect
@@ -24,9 +24,9 @@ require (
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
 	github.com/vulcanize/gap-filler v0.3.1
 	github.com/vulcanize/ipfs-ethdb v0.0.2-alpha
-	github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha
+	github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha.0.20210805022537-b4692fa49849
 )
 
-replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15
+replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.10.4-statediff-0.0.25
 
-replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
+replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha

pkg/eth/api.go

@@ -23,18 +23,17 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"math/big"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
@@ -393,7 +392,12 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Contex
 func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) {
 	tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash)
 	if tx != nil && err == nil {
-		return NewRPCTransaction(tx, blockHash, blockNumber, index), nil
+		header, err := pea.B.HeaderByHash(ctx, blockHash)
+		if err != nil {
+			return nil, err
+		}
+
+		return NewRPCTransaction(tx, blockHash, blockNumber, index, header.BaseFee), nil
 	}
 	if pea.rpc != nil {
 		var tx *RPCTransaction
@@ -879,13 +883,47 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 	if args.Gas != nil {
 		gas = uint64(*args.Gas)
 	}
+
 	if globalGasCap != nil && globalGasCap.Uint64() < gas {
 		logrus.Warnf("Caller gas above allowance, capping; requested: %d, cap: %d", gas, globalGasCap)
 		gas = globalGasCap.Uint64()
 	}
-	gasPrice := new(big.Int).SetUint64(params.GWei)
-	if args.GasPrice != nil {
-		gasPrice = args.GasPrice.ToInt()
+
+	var (
+		gasPrice  *big.Int
+		gasFeeCap *big.Int
+		gasTipCap *big.Int
+	)
+
+	if header.BaseFee == nil {
+		// If there's no basefee, then it must be a non-1559 execution
+		gasPrice = new(big.Int)
+		if args.GasPrice != nil {
+			gasPrice = args.GasPrice.ToInt()
+		}
+		gasFeeCap, gasTipCap = gasPrice, gasPrice
+	} else {
+		// A basefee is provided, necessitating 1559-type execution
+		if args.GasPrice != nil {
+			// User specified the legacy gas field, convert to 1559 gas typing
+			gasPrice = args.GasPrice.ToInt()
+			gasFeeCap, gasTipCap = gasPrice, gasPrice
+		} else {
+			// User specified 1559 gas feilds (or none), use those
+			gasFeeCap = new(big.Int)
+			if args.MaxFeePerGas != nil {
+				gasFeeCap = args.MaxFeePerGas.ToInt()
+			}
+			gasTipCap = new(big.Int)
+			if args.MaxPriorityFeePerGas != nil {
+				gasTipCap = args.MaxPriorityFeePerGas.ToInt()
+			}
+			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
+			gasPrice = new(big.Int)
+			if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 {
+				gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, header.BaseFee), gasFeeCap)
+			}
+		}
 	}
 
 	value := new(big.Int)
@@ -894,12 +932,19 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 	}
 
 	var data []byte
-	if args.Data != nil {
-		data = []byte(*args.Data)
+	if args.Input != nil {
+		data = *args.Input
+	} else if args.Data != nil {
+		data = *args.Data
+	}
+
+	var accessList types.AccessList
+	if args.AccessList != nil {
+		accessList = *args.AccessList
 	}
 
 	// Create new call message
-	msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, data, false)
+	msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, false)
 
 	// Setup context so it may be cancelled the call has completed
 	// or, in case of unmetered gas, setup a context with a timeout.

pkg/eth/api_test.go

@@ -42,13 +42,16 @@ import (
 )
 
 var (
-	randomAddr    = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
-	randomHash    = crypto.Keccak256Hash(randomAddr.Bytes())
-	number        = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
-	wrongNumber   = rpc.BlockNumber(number + 1)
-	blockHash     = test_helpers.MockBlock.Header().Hash()
-	ctx           = context.Background()
-	expectedBlock = map[string]interface{}{
+	randomAddr      = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomHash      = crypto.Keccak256Hash(randomAddr.Bytes())
+	number          = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
+	londonBlockNum  = rpc.BlockNumber(test_helpers.LondonBlockNum.Int64())
+	wrongNumber     = rpc.BlockNumber(number + 1)
+	blockHash       = test_helpers.MockBlock.Header().Hash()
+	londonBlockHash = test_helpers.MockLondonBlock.Header().Hash()
+	baseFee         = test_helpers.MockLondonBlock.BaseFee()
+	ctx             = context.Background()
+	expectedBlock   = map[string]interface{}{
 		"number":           (*hexutil.Big)(test_helpers.MockBlock.Number()),
 		"hash":             test_helpers.MockBlock.Hash(),
 		"parentHash":       test_helpers.MockBlock.ParentHash(),
@@ -128,13 +131,14 @@ var (
 		"receiptsRoot":     test_helpers.MockUncles[1].ReceiptHash,
 		"uncles":           []common.Hash{},
 	}
-	expectedTransaction  = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0)
-	expectedTransaction2 = eth.NewRPCTransaction(test_helpers.MockTransactions[1], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 1)
-	expectedTransaction3 = eth.NewRPCTransaction(test_helpers.MockTransactions[2], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 2)
-	expectRawTx, _       = rlp.EncodeToBytes(test_helpers.MockTransactions[0])
-	expectRawTx2, _      = rlp.EncodeToBytes(test_helpers.MockTransactions[1])
-	expectRawTx3, _      = rlp.EncodeToBytes(test_helpers.MockTransactions[2])
-	expectedReceipt      = map[string]interface{}{
+	expectedTransaction       = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0, test_helpers.MockBlock.BaseFee())
+	expectedTransaction2      = eth.NewRPCTransaction(test_helpers.MockTransactions[1], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 1, test_helpers.MockBlock.BaseFee())
+	expectedTransaction3      = eth.NewRPCTransaction(test_helpers.MockTransactions[2], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 2, test_helpers.MockBlock.BaseFee())
+	expectedLondonTransaction = eth.NewRPCTransaction(test_helpers.MockLondonTransactions[0], test_helpers.MockLondonBlock.Hash(), test_helpers.MockLondonBlock.NumberU64(), 0, test_helpers.MockLondonBlock.BaseFee())
+	expectRawTx, _            = rlp.EncodeToBytes(test_helpers.MockTransactions[0])
+	expectRawTx2, _           = rlp.EncodeToBytes(test_helpers.MockTransactions[1])
+	expectRawTx3, _           = rlp.EncodeToBytes(test_helpers.MockTransactions[2])
+	expectedReceipt           = map[string]interface{}{
 		"blockHash":         blockHash,
 		"blockNumber":       hexutil.Uint64(uint64(number.Int64())),
 		"transactionHash":   expectedTransaction.Hash,
@@ -208,7 +212,11 @@ var _ = Describe("API", func() {
 			uncleHashes[i] = uncle.Hash()
 		}
 		expectedBlock["uncles"] = uncleHashes
+
+		err = indexAndPublisher.Publish(test_helpers.MockConvertedLondonPayload)
+		Expect(err).ToNot(HaveOccurred())
 	})
+
 	// Single test db tear down at end of all tests
 	defer It("test teardown", func() { eth.TearDownDB(db) })
 	/*
@@ -250,7 +258,7 @@ var _ = Describe("API", func() {
 			bn := api.BlockNumber()
 			ubn := (uint64)(bn)
 			subn := strconv.FormatUint(ubn, 10)
-			Expect(subn).To(Equal(test_helpers.BlockNumber.String()))
+			Expect(subn).To(Equal(test_helpers.LondonBlockNum.String()))
 		})
 	})
 
@@ -284,6 +292,16 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(block).To(BeNil())
 		})
+		It("Fetch BaseFee from london block by block number, returns `nil` for legacy block", func() {
+			block, err := api.GetBlockByNumber(ctx, number, false)
+			Expect(err).ToNot(HaveOccurred())
+			_, ok := block["baseFee"]
+			Expect(ok).To(Equal(false))
+
+			block, err = api.GetBlockByNumber(ctx, londonBlockNum, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block["baseFee"].(*big.Int)).To(Equal(baseFee))
+		})
 	})
 
 	Describe("eth_getBlockByHash", func() {
@@ -316,6 +334,16 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(block).To(BeZero())
 		})
+		It("Fetch BaseFee from london block by block hash, returns `nil` for legacy block", func() {
+			block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, ok := block["baseFee"]
+			Expect(ok).To(Equal(false))
+
+			block, err = api.GetBlockByHash(ctx, londonBlockHash, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block["baseFee"].(*big.Int)).To(Equal(baseFee))
+		})
 	})
 
 	/*
@@ -437,6 +465,13 @@ var _ = Describe("API", func() {
 			Expect(tx).ToNot(BeNil())
 			Expect(tx).To(Equal(expectedTransaction3))
 		})
+		It("Retrieves the GasFeeCap and GasTipCap for dynamic transaction from the london block hash", func() {
+			tx := api.GetTransactionByBlockNumberAndIndex(ctx, londonBlockNum, 0)
+			Expect(tx).ToNot(BeNil())
+			Expect(tx.GasFeeCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasFeeCap())))
+			Expect(tx.GasTipCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasTipCap())))
+			Expect(tx).To(Equal(expectedLondonTransaction))
+		})
 	})
 
 	Describe("eth_getTransactionByBlockHashAndIndex", func() {
@@ -453,6 +488,14 @@ var _ = Describe("API", func() {
 			Expect(tx).ToNot(BeNil())
 			Expect(tx).To(Equal(expectedTransaction3))
 		})
+
+		It("Retrieves the GasFeeCap and GasTipCap for dynamic transaction from the london block hash", func() {
+			tx := api.GetTransactionByBlockHashAndIndex(ctx, londonBlockHash, 0)
+			Expect(tx).ToNot(BeNil())
+			Expect(tx.GasFeeCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasFeeCap())))
+			Expect(tx.GasTipCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasTipCap())))
+			Expect(tx).To(Equal(expectedLondonTransaction))
+		})
 	})
 
 	Describe("eth_getRawTransactionByBlockNumberAndIndex", func() {

pkg/eth/backend.go

@@ -336,7 +336,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	var transactions []*types.Transaction
 	for _, txIPLD := range txIPLDs {
 		var transaction types.Transaction
-		err = rlp.DecodeBytes(txIPLD.Data, &transaction)
+		err = transaction.UnmarshalBinary(txIPLD.Data)
 		if err != nil {
 			return nil, err
 		}
@@ -351,7 +351,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		err = rlp.DecodeBytes(rctIPLD.Data, &receipt)
+		err = receipt.UnmarshalBinary(rctIPLD.Data)
 		if err != nil {
 			return nil, err
 		}
@@ -433,7 +433,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var transactions []*types.Transaction
 	for _, txIPLD := range txIPLDs {
 		var transaction types.Transaction
-		err = rlp.DecodeBytes(txIPLD.Data, &transaction)
+		err = transaction.UnmarshalBinary(txIPLD.Data)
 		if err != nil {
 			return nil, err
 		}
@@ -451,7 +451,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		err = rlp.DecodeBytes(rctIPLD.Data, &receipt)
+		err = receipt.UnmarshalBinary(rctIPLD.Data)
 		if err != nil {
 			return nil, err
 		}
@@ -474,7 +474,7 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
 		return nil, common.Hash{}, 0, 0, err
 	}
 	var transaction types.Transaction
-	if err := rlp.DecodeBytes(tempTxStruct.Data, &transaction); err != nil {
+	if err := transaction.UnmarshalBinary(tempTxStruct.Data); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
 	return &transaction, common.HexToHash(tempTxStruct.BlockHash), tempTxStruct.BlockNumber, tempTxStruct.Index, nil
@@ -489,7 +489,7 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece
 	rcts := make(types.Receipts, len(receiptBytes))
 	for i, rctBytes := range receiptBytes {
 		rct := new(types.Receipt)
-		if err := rlp.DecodeBytes(rctBytes, rct); err != nil {
+		if err := rct.UnmarshalBinary(rctBytes); err != nil {
 			return nil, err
 		}
 		rct.TxHash = txs[i]