Add initial integration test with circuits (#257)

Add an integration tests for the evm and state circuits, using the input
generated by the circuit input builder and data queried from geth.

bus-mapping: Introduce `BuilderClient` which wraps a GethClient and
contains methods to generate the complete circuit inputs by doing all
the necessary steps: query geth, build partial state_db, etc.  Use this
`BuilderClient` in both the circuits and circuit_input_builder
integration tests.

.github/workflows/integration.yml

@@ -1,8 +1,6 @@
 name: Integration Tests
 
 on:
-  pull_request:
-    types: [synchronize, opened, reopened, ready_for_review]
   push:
     branches:
       - main
@@ -30,11 +28,16 @@ jobs:
           wget -q https://github.com/ethereum/solidity/releases/download/v0.8.10/solc-static-linux -O $HOME/bin/solc
           chmod u+x "$HOME/bin/solc"
           solc --version
-      # Run an initial build in a sepparate step to split the build time from execution time
-      - name: Build gendata bin
+      # Run an initial build in a separate step to split the build time from execution time
+      - name: Build bins
         run: cargo build --bin gen_blockchain_data
+      - name: Build tests
+        run: for testname in rpc circuit_input_builder circuits; do cargo test --profile release --test $testname --features $testname --no-run; done
       - run: ./run.sh --steps "setup"
       - run: ./run.sh --steps "gendata"
       - run: ./run.sh --steps "tests" --tests "rpc"
       - run: ./run.sh --steps "tests" --tests "circuit_input_builder"
+      # Uncomment once the evm and state circuits tests pass for the block
+      # where Greeter.sol is deployed.
+      # - run: ./run.sh --steps "tests" --tests "circuits"
       - run: ./run.sh --steps "cleanup"

bus-mapping/src/circuit_input_builder.rs

@@ -12,12 +12,15 @@ use crate::exec_trace::OperationRef;
 use crate::geth_errors::*;
 use crate::operation::container::OperationContainer;
 use crate::operation::{MemoryOp, Op, Operation, StackOp, RW};
-use crate::state_db::{CodeDB, StateDB};
+use crate::state_db::{self, CodeDB, StateDB};
 use crate::Error;
 use core::fmt::Debug;
 use ethers_core::utils::{get_contract_address, get_create2_address};
 use std::collections::{hash_map::Entry, HashMap, HashSet};
 
+use crate::rpc::GethClient;
+use ethers_providers::JsonRpcClient;
+
 /// Out of Gas errors by opcode
 #[derive(Debug, PartialEq)]
 pub enum OogError {
@@ -248,19 +251,19 @@ impl TryFrom<OpcodeId> for CallKind {
 #[derive(Debug)]
 pub struct Call {
     /// Unique call identifier within the Block.
-    call_id: usize,
+    pub call_id: usize,
     /// Type of call
-    kind: CallKind,
+    pub kind: CallKind,
     /// This call is being executed without write access (STATIC)
-    is_static: bool,
+    pub is_static: bool,
     /// This call generated implicity by a Transaction.
-    is_root: bool,
+    pub is_root: bool,
     /// Address where this call is being executed
     pub address: Address,
     /// Code Source
-    code_source: CodeSource,
+    pub code_source: CodeSource,
     /// Code Hash
-    code_hash: Hash,
+    pub code_hash: Hash,
 }
 
 impl Call {
@@ -1193,6 +1196,164 @@ pub fn gen_state_access_trace<TX>(
     Ok(accs)
 }
 
+type EthBlock = eth_types::Block<eth_types::Transaction>;
+
+/// Struct that wraps a GethClient and contains methods to perform all the steps
+/// necessary to generate the circuit inputs for a block by querying geth for
+/// the necessary information and using the CircuitInputBuilder.
+pub struct BuilderClient<P: JsonRpcClient> {
+    cli: GethClient<P>,
+    constants: ChainConstants,
+}
+
+impl<P: JsonRpcClient> BuilderClient<P> {
+    /// Create a new BuilderClient
+    pub async fn new(client: GethClient<P>) -> Result<Self, Error> {
+        let constants = ChainConstants {
+            coinbase: client.get_coinbase().await?,
+            chain_id: client.get_chain_id().await?,
+        };
+        Ok(Self {
+            cli: client,
+            constants,
+        })
+    }
+
+    /// Step 1. Query geth for Block, Txs and TxExecTraces
+    pub async fn get_block(
+        &self,
+        block_num: u64,
+    ) -> Result<(EthBlock, Vec<eth_types::GethExecTrace>), Error> {
+        let eth_block = self.cli.get_block_by_number(block_num.into()).await?;
+        let geth_traces =
+            self.cli.trace_block_by_number(block_num.into()).await?;
+        Ok((eth_block, geth_traces))
+    }
+
+    /// Step 2. Get State Accesses from TxExecTraces
+    pub fn get_state_accesses(
+        &self,
+        eth_block: &EthBlock,
+        geth_traces: &[eth_types::GethExecTrace],
+    ) -> Result<AccessSet, Error> {
+        let mut block_access_trace = Vec::new();
+        for (tx_index, tx) in eth_block.transactions.iter().enumerate() {
+            let geth_trace = &geth_traces[tx_index];
+            let tx_access_trace =
+                gen_state_access_trace(eth_block, tx, geth_trace)?;
+            block_access_trace.extend(tx_access_trace);
+        }
+
+        Ok(AccessSet::from(block_access_trace))
+    }
+
+    /// Step 3. Query geth for all accounts, storage keys, and codes from
+    /// Accesses
+    pub async fn get_state(
+        &self,
+        block_num: u64,
+        access_set: AccessSet,
+    ) -> Result<
+        (
+            Vec<eth_types::EIP1186ProofResponse>,
+            HashMap<Address, Vec<u8>>,
+        ),
+        Error,
+    > {
+        let mut proofs = Vec::new();
+        for (address, key_set) in access_set.state {
+            let mut keys: Vec<Word> = key_set.iter().cloned().collect();
+            keys.sort();
+            let proof = self
+                .cli
+                .get_proof(address, keys, (block_num - 1).into())
+                .await
+                .unwrap();
+            proofs.push(proof);
+        }
+        let mut codes: HashMap<Address, Vec<u8>> = HashMap::new();
+        for address in access_set.code {
+            let code = self
+                .cli
+                .get_code(address, (block_num - 1).into())
+                .await
+                .unwrap();
+            codes.insert(address, code);
+        }
+        Ok((proofs, codes))
+    }
+
+    /// Step 4. Build a partial StateDB from step 3
+    pub fn build_state_code_db(
+        &self,
+        proofs: Vec<eth_types::EIP1186ProofResponse>,
+        codes: HashMap<Address, Vec<u8>>,
+    ) -> (StateDB, CodeDB) {
+        let mut sdb = StateDB::new();
+        for proof in proofs {
+            let mut storage = HashMap::new();
+            for storage_proof in proof.storage_proof {
+                storage.insert(storage_proof.key, storage_proof.value);
+            }
+            sdb.set_account(
+                &proof.address,
+                state_db::Account {
+                    nonce: proof.nonce,
+                    balance: proof.balance,
+                    storage,
+                    code_hash: proof.code_hash,
+                },
+            )
+        }
+
+        let mut code_db = CodeDB::new();
+        for (_address, code) in codes {
+            code_db.insert(code.clone());
+        }
+        (sdb, code_db)
+    }
+
+    /// Step 5. For each step in TxExecTraces, gen the associated ops and state
+    /// circuit inputs
+    pub fn gen_inputs_from_state(
+        &self,
+        sdb: StateDB,
+        code_db: CodeDB,
+        eth_block: &EthBlock,
+        geth_traces: &[eth_types::GethExecTrace],
+    ) -> Result<CircuitInputBuilder, Error> {
+        let mut builder = CircuitInputBuilder::new(
+            sdb,
+            code_db,
+            eth_block,
+            self.constants.clone(),
+        );
+        for (tx_index, tx) in eth_block.transactions.iter().enumerate() {
+            let geth_trace = &geth_traces[tx_index];
+            builder.handle_tx(tx, geth_trace)?;
+        }
+        Ok(builder)
+    }
+
+    /// Perform all the steps to generate the circuit inputs
+    pub async fn gen_inputs(
+        &self,
+        block_num: u64,
+    ) -> Result<CircuitInputBuilder, Error> {
+        let (eth_block, geth_traces) = self.get_block(block_num).await?;
+        let access_set = self.get_state_accesses(&eth_block, &geth_traces)?;
+        let (proofs, codes) = self.get_state(block_num, access_set).await?;
+        let (state_db, code_db) = self.build_state_code_db(proofs, codes);
+        let builder = self.gen_inputs_from_state(
+            state_db,
+            code_db,
+            &eth_block,
+            &geth_traces,
+        )?;
+        Ok(builder)
+    }
+}
+
 #[cfg(test)]
 mod tracer_tests {
     use super::*;

bus-mapping/src/eth_types.rs

@@ -188,7 +188,7 @@ struct GethExecStepInternal {
     pc: ProgramCounter,
     op: OpcodeId,
     gas: Gas,
-    #[serde(alias = "gasCost")]
+    #[serde(rename = "gasCost")]
     gas_cost: GasCost,
     depth: u16,
     error: Option<String>,
@@ -310,7 +310,7 @@ pub struct GethExecTraceInternal {
     pub gas: Gas,
     pub failed: bool,
     // return_value is a hex encoded byte array
-    #[serde(alias = "structLogs")]
+    #[serde(rename = "structLogs")]
     pub struct_logs: Vec<GethExecStep>,
 }
 

bus-mapping/src/external_tracer.rs

@@ -150,6 +150,6 @@ mod trace_test {
 
         let block =
             mock::BlockData::new_single_tx_trace_code_at_start(&code).unwrap();
-        assert_eq!(block.geth_trace.struct_logs[2].memory.0.len(), 0)
+        assert_eq!(block.geth_trace.struct_logs[2].memory.0.len(), 0);
     }
 }

bus-mapping/src/operation.rs

@@ -166,7 +166,7 @@ impl fmt::Debug for StackOp {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str("StackOp { ")?;
         f.write_fmt(format_args!(
-            "{:?}, addr: {:?}, val: {:?}",
+            "{:?}, addr: {:?}, val: 0x{:x}",
             self.rw, self.address, self.value
         ))?;
         f.write_str(" }")
@@ -250,7 +250,7 @@ impl fmt::Debug for StorageOp {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str("StorageOp { ")?;
         f.write_fmt(format_args!(
-            "{:?}, addr: {:?}, key: {:?}, val_prev: {:?}, val: {:?}",
+            "{:?}, addr: {:?}, key: {:?}, val_prev: 0x{:x}, val: 0x{:x}",
             self.rw, self.address, self.key, self.value_prev, self.value,
         ))?;
         f.write_str(" }")
@@ -389,7 +389,7 @@ impl fmt::Debug for TxAccessListAccountStorageOp {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str("TxAccessListAccountStorageOp { ")?;
         f.write_fmt(format_args!(
-            "tx_id: {:?}, addr: {:?}, key: {:?}, val_prev: {:?}, val: {:?}",
+            "tx_id: {:?}, addr: {:?}, key: 0x{:x}, val_prev: {:?}, val: {:?}",
             self.tx_id, self.address, self.key, self.value_prev, self.value
         ))?;
         f.write_str(" }")
@@ -437,7 +437,7 @@ impl fmt::Debug for TxRefundOp {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str("TxRefundOp { ")?;
         f.write_fmt(format_args!(
-            "{:?} tx_id: {:?}, val_prev: {:?}, val: {:?}",
+            "{:?} tx_id: {:?}, val_prev: 0x{:x}, val: 0x{:x}",
             self.rw, self.tx_id, self.value_prev, self.value
         ))?;
         f.write_str(" }")
@@ -495,7 +495,7 @@ impl fmt::Debug for AccountOp {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str("AccountOp { ")?;
         f.write_fmt(format_args!(
-            "{:?}, addr: {:?}, field: {:?}, val_prev: {:?}, val: {:?}",
+            "{:?}, addr: {:?}, field: {:?}, val_prev: 0x{:x}, val: 0x{:x}",
             self.rw, self.address, self.field, self.value_prev, self.value
         ))?;
         f.write_str(" }")

bus-mapping/src/rpc.rs

@@ -8,6 +8,7 @@ use crate::eth_types::{
 use crate::Error;
 pub use ethers_core::types::BlockNumber;
 use ethers_providers::JsonRpcClient;
+use serde::Serialize;
 
 /// Serialize a type.
 ///
@@ -18,6 +19,34 @@ pub fn serialize<T: serde::Serialize>(t: &T) -> serde_json::Value {
     serde_json::to_value(t).expect("Types never fail to serialize.")
 }
 
+#[derive(Serialize)]
+#[doc(hidden)]
+pub(crate) struct GethLoggerConfig {
+    /// enable memory capture
+    #[serde(rename = "EnableMemory")]
+    enable_memory: bool,
+    /// disable stack capture
+    #[serde(rename = "DisableStack")]
+    disable_stack: bool,
+    /// disable storage capture
+    #[serde(rename = "DisableStorage")]
+    disable_storage: bool,
+    /// enable return data capture
+    #[serde(rename = "EnableReturnData")]
+    enable_return_data: bool,
+}
+
+impl Default for GethLoggerConfig {
+    fn default() -> Self {
+        Self {
+            enable_memory: true,
+            disable_stack: false,
+            disable_storage: false,
+            enable_return_data: true,
+        }
+    }
+}
+
 /// Placeholder structure designed to contain the methods that the BusMapping
 /// needs in order to enable Geth queries.
 pub struct GethClient<P: JsonRpcClient>(pub P);
@@ -83,9 +112,10 @@ impl<P: JsonRpcClient> GethClient<P> {
         hash: Hash,
     ) -> Result<Vec<GethExecTrace>, Error> {
         let hash = serialize(&hash);
+        let cfg = serialize(&GethLoggerConfig::default());
         let resp: ResultGethExecTraces = self
             .0
-            .request("debug_traceBlockByHash", [hash])
+            .request("debug_traceBlockByHash", [hash, cfg])
             .await
             .map_err(|e| Error::JSONRpcError(e.into()))?;
         Ok(resp.0.into_iter().map(|step| step.result).collect())
@@ -99,9 +129,10 @@ impl<P: JsonRpcClient> GethClient<P> {
         block_num: BlockNumber,
     ) -> Result<Vec<GethExecTrace>, Error> {
         let num = serialize(&block_num);
+        let cfg = serialize(&GethLoggerConfig::default());
         let resp: ResultGethExecTraces = self
             .0
-            .request("debug_traceBlockByNumber", [num])
+            .request("debug_traceBlockByNumber", [num, cfg])
             .await
             .map_err(|e| Error::JSONRpcError(e.into()))?;
         Ok(resp.0.into_iter().map(|step| step.result).collect())