Add initial integration test with circuits

Add an integration tests for the evm and state circuits, using the input
generated by the circuit input builder and data queried from geth.

bus-mapping: Introduce `BuilderClient` which wraps a GethClient and
contains methods to generate the complete circuit inputs by doing all
the necessary steps: query geth, build partial state_db, etc.  Use this
`BuilderClient` in both the circuits and circuit_input_builder
integration tests.

.github/workflows/integration.yml

@@ -31,10 +31,13 @@ jobs:
           chmod u+x "$HOME/bin/solc"
           solc --version
       # Run an initial build in a sepparate step to split the build time from execution time
-      - name: Build gendata bin
-        run: cargo build --bin gen_blockchain_data
+      - name: Build
+        run: |
+          cargo build --bin gen_blockchain_data
+          for testname in "rpc circuit_input_builder circuits"; do cargo test --profile release --test $testname --features $testname --no-run; done
       - run: ./run.sh --steps "setup"
       - run: ./run.sh --steps "gendata"
       - run: ./run.sh --steps "tests" --tests "rpc"
       - run: ./run.sh --steps "tests" --tests "circuit_input_builder"
+      - run: ./run.sh --steps "tests" --tests "circuits"
       - run: ./run.sh --steps "cleanup"

bus-mapping/src/circuit_input_builder.rs

@@ -12,12 +12,15 @@ use crate::exec_trace::OperationRef;
 use crate::geth_errors::*;
 use crate::operation::container::OperationContainer;
 use crate::operation::{MemoryOp, Op, Operation, StackOp, RW};
-use crate::state_db::{CodeDB, StateDB};
+use crate::state_db::{self, CodeDB, StateDB};
 use crate::Error;
 use core::fmt::Debug;
 use ethers_core::utils::{get_contract_address, get_create2_address};
 use std::collections::{hash_map::Entry, HashMap, HashSet};
 
+use crate::rpc::GethClient;
+use ethers_providers::JsonRpcClient;
+
 /// Out of Gas errors by opcode
 #[derive(Debug, PartialEq)]
 pub enum OogError {
@@ -248,19 +251,19 @@ impl TryFrom<OpcodeId> for CallKind {
 #[derive(Debug)]
 pub struct Call {
     /// Unique call identifier within the Block.
-    call_id: usize,
+    pub call_id: usize,
     /// Type of call
-    kind: CallKind,
+    pub kind: CallKind,
     /// This call is being executed without write access (STATIC)
-    is_static: bool,
+    pub is_static: bool,
     /// This call generated implicity by a Transaction.
-    is_root: bool,
+    pub is_root: bool,
     /// Address where this call is being executed
     pub address: Address,
     /// Code Source
-    code_source: CodeSource,
+    pub code_source: CodeSource,
     /// Code Hash
-    code_hash: Hash,
+    pub code_hash: Hash,
 }
 
 impl Call {
@@ -1193,6 +1196,164 @@ pub fn gen_state_access_trace<TX>(
     Ok(accs)
 }
 
+type EthBlock = eth_types::Block<eth_types::Transaction>;
+
+/// Struct that wraps a GethClient and contains methods to perform all the steps
+/// necessary to generate the circuit inputs for a block by querying geth for
+/// the necessary information and using the CircuitInputBuilder.
+pub struct BuilderClient<P: JsonRpcClient> {
+    cli: GethClient<P>,
+    constants: ChainConstants,
+}
+
+impl<P: JsonRpcClient> BuilderClient<P> {
+    /// Create a new BuilderClient
+    pub async fn new(client: GethClient<P>) -> Result<Self, Error> {
+        let constants = ChainConstants {
+            coinbase: client.get_coinbase().await?,
+            chain_id: client.get_chain_id().await?,
+        };
+        Ok(Self {
+            cli: client,
+            constants,
+        })
+    }
+
+    /// Step 1. Query geth for Block, Txs and TxExecTraces
+    pub async fn get_block(
+        &self,
+        block_num: u64,
+    ) -> Result<(EthBlock, Vec<eth_types::GethExecTrace>), Error> {
+        let eth_block = self.cli.get_block_by_number(block_num.into()).await?;
+        let geth_traces =
+            self.cli.trace_block_by_number(block_num.into()).await?;
+        Ok((eth_block, geth_traces))
+    }
+
+    /// Step 2. Get State Accesses from TxExecTraces
+    pub fn get_state_accesses(
+        &self,
+        eth_block: &EthBlock,
+        geth_traces: &[eth_types::GethExecTrace],
+    ) -> Result<AccessSet, Error> {
+        let mut block_access_trace = Vec::new();
+        for (tx_index, tx) in eth_block.transactions.iter().enumerate() {
+            let geth_trace = &geth_traces[tx_index];
+            let tx_access_trace =
+                gen_state_access_trace(eth_block, tx, geth_trace)?;
+            block_access_trace.extend(tx_access_trace);
+        }
+
+        Ok(AccessSet::from(block_access_trace))
+    }
+
+    /// Step 3. Query geth for all accounts, storage keys, and codes from
+    /// Accesses
+    pub async fn get_state(
+        &self,
+        block_num: u64,
+        access_set: AccessSet,
+    ) -> Result<
+        (
+            Vec<eth_types::EIP1186ProofResponse>,
+            HashMap<Address, Vec<u8>>,
+        ),
+        Error,
+    > {
+        let mut proofs = Vec::new();
+        for (address, key_set) in access_set.state {
+            let mut keys: Vec<Word> = key_set.iter().cloned().collect();
+            keys.sort();
+            let proof = self
+                .cli
+                .get_proof(address, keys, (block_num - 1).into())
+                .await
+                .unwrap();
+            proofs.push(proof);
+        }
+        let mut codes: HashMap<Address, Vec<u8>> = HashMap::new();
+        for address in access_set.code {
+            let code = self
+                .cli
+                .get_code(address, (block_num - 1).into())
+                .await
+                .unwrap();
+            codes.insert(address, code);
+        }
+        Ok((proofs, codes))
+    }
+
+    /// Step 4. Build a partial StateDB from step 3
+    pub fn build_state_code_db(
+        &self,
+        proofs: Vec<eth_types::EIP1186ProofResponse>,
+        codes: HashMap<Address, Vec<u8>>,
+    ) -> (StateDB, CodeDB) {
+        let mut sdb = StateDB::new();
+        for proof in proofs {
+            let mut storage = HashMap::new();
+            for storage_proof in proof.storage_proof {
+                storage.insert(storage_proof.key, storage_proof.value);
+            }
+            sdb.set_account(
+                &proof.address,
+                state_db::Account {
+                    nonce: proof.nonce,
+                    balance: proof.balance,
+                    storage,
+                    code_hash: proof.code_hash,
+                },
+            )
+        }
+
+        let mut code_db = CodeDB::new();
+        for (_address, code) in codes {
+            code_db.insert(code.clone());
+        }
+        (sdb, code_db)
+    }
+
+    /// Step 5. For each step in TxExecTraces, gen the associated ops and state
+    /// circuit inputs
+    pub fn gen_inputs_from_state(
+        &self,
+        sdb: StateDB,
+        code_db: CodeDB,
+        eth_block: &EthBlock,
+        geth_traces: &[eth_types::GethExecTrace],
+    ) -> Result<CircuitInputBuilder, Error> {
+        let mut builder = CircuitInputBuilder::new(
+            sdb,
+            code_db,
+            eth_block,
+            self.constants.clone(),
+        );
+        for (tx_index, tx) in eth_block.transactions.iter().enumerate() {
+            let geth_trace = &geth_traces[tx_index];
+            builder.handle_tx(tx, geth_trace)?;
+        }
+        Ok(builder)
+    }
+
+    /// Perform all the steps to generate the circuit inputs
+    pub async fn gen_inputs(
+        &self,
+        block_num: u64,
+    ) -> Result<CircuitInputBuilder, Error> {
+        let (eth_block, geth_traces) = self.get_block(block_num).await?;
+        let access_set = self.get_state_accesses(&eth_block, &geth_traces)?;
+        let (proofs, codes) = self.get_state(block_num, access_set).await?;
+        let (state_db, code_db) = self.build_state_code_db(proofs, codes);
+        let builder = self.gen_inputs_from_state(
+            state_db,
+            code_db,
+            &eth_block,
+            &geth_traces,
+        )?;
+        Ok(builder)
+    }
+}
+
 #[cfg(test)]
 mod tracer_tests {
     use super::*;

integration-tests/Cargo.toml

@@ -11,11 +11,14 @@ ethers = "0.6"
 serde_json = "1.0.66"
 serde = {version = "1.0.130", features = ["derive"] }
 bus-mapping = { path = "../bus-mapping"}
+zkevm-circuits = { path = "../zkevm-circuits"}
 tokio = { version = "1.13", features = ["macros", "rt-multi-thread"] }
 url = "2.2.2"
 pretty_assertions = "1.0.0"
 log = "0.4.14"
 env_logger = "0.9"
+halo2 = { git = "https://github.com/appliedzkp/halo2.git", rev = "b78c39cacc1c79d287032f1b5f94beb661b3fb42" }
+pairing = { git = 'https://github.com/appliedzkp/pairing', package = "pairing_bn256" }
 
 [dev-dependencies]
 pretty_assertions = "1.0.0"
@@ -24,3 +27,4 @@ pretty_assertions = "1.0.0"
 default = []
 rpc = []
 circuit_input_builder = []
+circuits = []

integration-tests/run.sh

@@ -3,7 +3,7 @@ set -e
 
 ARG_DEFAULT_SUDO=
 ARG_DEFAULT_STEPS="setup gendata tests cleanup"
-ARG_DEFAULT_TESTS="rpc circuit_input_builder"
+ARG_DEFAULT_TESTS="rpc circuit_input_builder circuits"
 
 usage() {
     cat >&2 << EOF
@@ -97,7 +97,7 @@ fi
 if [ -n "$STEP_TESTS" ]; then
     for testname in $ARG_TESTS; do
         echo "+ Running test group $testname"
-        cargo test --test $testname --features $testname -- --nocapture
+        cargo test --profile release --test $testname --features $testname -- --nocapture
     done
 fi
 

integration-tests/tests/circuit_input_builder.rs

@@ -1,16 +1,9 @@
 #![cfg(feature = "circuit_input_builder")]
 
-use bus_mapping::circuit_input_builder::{
-    gen_state_access_trace, AccessSet, CircuitInputBuilder,
-};
-use bus_mapping::eth_types::{Address, Word};
-use bus_mapping::state_db::{self, CodeDB, StateDB};
-use integration_tests::{
-    get_chain_constants, get_client, log_init, GenDataOutput,
-};
+use bus_mapping::circuit_input_builder::BuilderClient;
+use integration_tests::{get_client, log_init, GenDataOutput};
 use lazy_static::lazy_static;
 use log::trace;
-use std::collections::HashMap;
 
 lazy_static! {
     pub static ref GEN_DATA: GenDataOutput = GenDataOutput::load();
@@ -24,86 +17,27 @@ async fn test_circuit_input_builder_block_a() {
     let (block_num, _address) = GEN_DATA.deployments.get("Greeter").unwrap();
     let cli = get_client();
 
+    let cli = BuilderClient::new(cli).await.unwrap();
+
     // 1. Query geth for Block, Txs and TxExecTraces
-    let eth_block = cli.get_block_by_number((*block_num).into()).await.unwrap();
-    let geth_trace = cli
-        .trace_block_by_number((*block_num).into())
-        .await
-        .unwrap();
-    let tx_index = 0;
+    let (eth_block, geth_trace) = cli.get_block(*block_num).await.unwrap();
 
     // 2. Get State Accesses from TxExecTraces
-    let access_trace = gen_state_access_trace(
-        &eth_block,
-        &eth_block.transactions[tx_index],
-        &geth_trace[tx_index],
-    )
-    .unwrap();
-    trace!("AccessTrace:");
-    for access in &access_trace {
-        trace!("{:#?}", access);
-    }
-
-    let access_set = AccessSet::from(access_trace);
+    let access_set = cli.get_state_accesses(&eth_block, &geth_trace).unwrap();
     trace!("AccessSet: {:#?}", access_set);
 
     // 3. Query geth for all accounts, storage keys, and codes from Accesses
-    let mut proofs = Vec::new();
-    for (address, key_set) in access_set.state {
-        let mut keys: Vec<Word> = key_set.iter().cloned().collect();
-        keys.sort();
-        let proof = cli
-            .get_proof(address, keys, (*block_num - 1).into())
-            .await
-            .unwrap();
-        proofs.push(proof);
-    }
-    let mut codes: HashMap<Address, Vec<u8>> = HashMap::new();
-    for address in access_set.code {
-        let code = cli
-            .get_code(address, (*block_num - 1).into())
-            .await
-            .unwrap();
-        codes.insert(address, code);
-    }
+    let (proofs, codes) = cli.get_state(*block_num, access_set).await.unwrap();
 
     // 4. Build a partial StateDB from step 3
-    let mut sdb = StateDB::new();
-    for proof in proofs {
-        let mut storage = HashMap::new();
-        for storage_proof in proof.storage_proof {
-            storage.insert(storage_proof.key, storage_proof.value);
-        }
-        sdb.set_account(
-            &proof.address,
-            state_db::Account {
-                nonce: proof.nonce,
-                balance: proof.balance,
-                storage,
-                code_hash: proof.code_hash,
-            },
-        )
-    }
-    trace!("StateDB: {:#?}", sdb);
-
-    let mut code_db = CodeDB::new();
-    for (_address, code) in codes {
-        code_db.insert(code.clone());
-    }
-    let constants = get_chain_constants().await;
-    let mut builder =
-        CircuitInputBuilder::new(sdb, code_db, &eth_block, constants);
+    let (state_db, code_db) = cli.build_state_code_db(proofs, codes);
+    trace!("StateDB: {:#?}", state_db);
 
     // 5. For each step in TxExecTraces, gen the associated ops and state
     // circuit inputs
-    let block_geth_trace = cli
-        .trace_block_by_number((*block_num).into())
-        .await
+    let builder = cli
+        .gen_inputs_from_state(state_db, code_db, &eth_block, &geth_trace)
         .unwrap();
-    for (tx_index, tx) in eth_block.transactions.iter().enumerate() {
-        let geth_trace = &block_geth_trace[tx_index];
-        builder.handle_tx(tx, geth_trace).unwrap();
-    }
 
     trace!("CircuitInputBuilder: {:#?}", builder);
 }