Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Non-breaking changes] Refactor trie interfaces #376

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
8253211
implements RLP encoding/decoding processing for structs
trinhdn2 Jun 16, 2023
fa67be8
Update RLP lib
trinhdn2 Jun 16, 2023
9b5dadb
Implement RLP encoder code generation tool
trinhdn2 Jun 16, 2023
ddf35f8
Update unit tests and benchmark
trinhdn2 Jun 16, 2023
17e4876
Include RLPgen tool to CI
trinhdn2 Jun 20, 2023
aa7c4b4
Add benchmarks for types RLP encoding/decoding
trinhdn2 Jun 21, 2023
ac89cd3
Generate RLP encoder for some structs
trinhdn2 Jun 21, 2023
060f4ce
Convert status of receipts from uint to uint64
trinhdn2 Jun 21, 2023
8fdaf68
Unify multiple keccak interface into one inside crypto package
trinhdn2 Jul 16, 2023
c7be6cc
Merge branch 'refactor/keccak-state' into ft/stacktrie
trinhdn2 Jul 16, 2023
340bf2b
Add encode method to Node interface
trinhdn2 Jul 16, 2023
33149c9
Implement stacktrie
trinhdn2 Jul 16, 2023
2e9abc4
Add encode method for rawShortNode and rawFullNode
trinhdn2 Jul 16, 2023
008274f
Fix import cycle
trinhdn2 Jul 16, 2023
f88dba4
Fix stacktrie unit tests
trinhdn2 Jul 16, 2023
327c90d
Remove rlpLog RLP encoder
trinhdn2 Jul 17, 2023
4d22f8d
Fix unit tests
trinhdn2 Jul 18, 2023
c37affb
Move statedb.Account struct to types.StateAccount
trinhdn2 Jul 20, 2023
e0776fb
Move database_util.go to rawdb package
trinhdn2 Jul 20, 2023
5ce901b
Move db keys to schema.go
trinhdn2 Jul 20, 2023
36373e7
Refine db keys
trinhdn2 Jul 20, 2023
6392afd
Rename
trinhdn2 Jul 20, 2023
cf62b98
Split into sub accessor files
trinhdn2 Jul 20, 2023
024181b
chore: remove intPool
c98tristan Jul 17, 2023
c384646
Feat: Update derive SHA for stacktrie
c98tristan Jul 19, 2023
b8421e5
Chore: Add benchmark for Stacktrie
c98tristan Jul 19, 2023
6099609
Chore: Fix Update function missing return type
c98tristan Jul 19, 2023
ae6ca82
Chore: Change GetRlp to EncodeIndex in LendingTransaction and OrderTr…
c98tristan Jul 24, 2023
515a614
Update DeriveSha with new Hasher and DerivableList interface
trinhdn2 Jul 19, 2023
44d2c54
Chore: Add intPool in interpreter.go
c98tristan Jul 24, 2023
5fae822
Merge branch 'refactor/state-account' into refactor/rawdb
trinhdn2 Jul 24, 2023
72ca227
Chore: Change parameter of NewBlock function from Trie to Stacktrie
c98tristan Jul 25, 2023
530cbb1
Chore: Sorting imported library
c98tristan Jul 25, 2023
260f47e
Chore: Sorting imported library
c98tristan Jul 25, 2023
a6098a1
Merge remote-tracking branch 'c98tristan/feat/update-trie-to-stacktri…
trinhdn2 Jul 25, 2023
b1cb0fc
Resolve conflicts after merged
trinhdn2 Jul 25, 2023
46ca156
[WIP] Trie refactor
trinhdn2 Jul 25, 2023
0a2bbca
[WIP] Implement NodeSet and HashDB of trie
trinhdn2 Jul 25, 2023
0aa6b66
Implement new Trie interface for light trie
trinhdn2 Jul 27, 2023
83b2169
Apply new trie interface to stateDB, etc.
trinhdn2 Jul 27, 2023
ec6ff59
Update unit tests
trinhdn2 Jul 27, 2023
59f0429
[WIP] Trie commit via hashdb
trinhdn2 Jul 31, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 6 additions & 7 deletions accounts/abi/bind/backends/simulated.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
"context"
"errors"
"fmt"
"github.com/tomochain/tomochain/consensus"
"github.com/tomochain/tomochain/core/rawdb"
"math/big"
"sync"
"time"
Expand All @@ -30,9 +28,11 @@ import (
"github.com/tomochain/tomochain/accounts/abi/bind"
"github.com/tomochain/tomochain/common"
"github.com/tomochain/tomochain/common/math"
"github.com/tomochain/tomochain/consensus"
"github.com/tomochain/tomochain/consensus/ethash"
"github.com/tomochain/tomochain/core"
"github.com/tomochain/tomochain/core/bloombits"
"github.com/tomochain/tomochain/core/rawdb"
"github.com/tomochain/tomochain/core/state"
"github.com/tomochain/tomochain/core/types"
"github.com/tomochain/tomochain/core/vm"
Expand Down Expand Up @@ -174,7 +174,7 @@ func (b *SimulatedBackend) ForEachStorageAt(ctx context.Context, contract common

// TransactionReceipt returns the receipt of a transaction.
func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) {
receipt, _, _, _ := core.GetReceipt(b.database, txHash)
receipt, _, _, _ := rawdb.GetReceipt(b.database, txHash)
return receipt, nil
}

Expand Down Expand Up @@ -202,7 +202,7 @@ func (b *SimulatedBackend) CallContract(ctx context.Context, call tomochain.Call
return rval, err
}

//FIXME: please use copyState for this function
// FIXME: please use copyState for this function
// CallContractWithState executes a contract call at the given state.
func (b *SimulatedBackend) CallContractWithState(call tomochain.CallMsg, chain consensus.ChainContext, statedb *state.StateDB) ([]byte, error) {
// Ensure message is initialized properly.
Expand Down Expand Up @@ -285,7 +285,6 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM

snapshot := b.pendingState.Snapshot()
_, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
fmt.Println("EstimateGas",err,failed)
b.pendingState.RevertToSnapshot(snapshot)

if err != nil || failed {
Expand Down Expand Up @@ -485,11 +484,11 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb
}

func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil
return rawdb.GetBlockReceipts(fb.db, hash, rawdb.GetBlockNumber(fb.db, hash)), nil
}

func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash))
receipts := rawdb.GetBlockReceipts(fb.db, hash, rawdb.GetBlockNumber(fb.db, hash))
if receipts == nil {
return nil, nil
}
Expand Down
8 changes: 4 additions & 4 deletions accounts/keystore/keystore_wallet.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte
if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
// StateAccount seems valid, request the keystore to sign
return w.keystore.SignHash(account, hash)
}

Expand All @@ -106,7 +106,7 @@ func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction,
if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
// StateAccount seems valid, request the keystore to sign
return w.keystore.SignTx(account, tx, chainID)
}

Expand All @@ -120,7 +120,7 @@ func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passph
if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
// StateAccount seems valid, request the keystore to sign
return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
}

Expand All @@ -134,6 +134,6 @@ func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphra
if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
// StateAccount seems valid, request the keystore to sign
return w.keystore.SignTxWithPassphrase(account, passphrase, tx, chainID)
}
2 changes: 1 addition & 1 deletion accounts/usbwallet/wallet.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ func (w *wallet) selfDerive() {
// Termination requested
continue
case reqc = <-w.deriveReq:
// Account discovery requested
// StateAccount discovery requested
}
// Derivation needs a chain and device access, skip if either unavailable
w.stateLock.RLock()
Expand Down
13 changes: 7 additions & 6 deletions build/ci.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

//go:build none
// +build none

/*
Expand All @@ -23,14 +24,13 @@ Usage: go run build/ci.go <command> <command flags/arguments>

Available commands are:

install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
test [ -coverage ] [ packages... ] -- runs the tests
lint -- runs certain pre-selected linters
importkeys -- imports signing keys from env
xgo [ -alltools ] [ options ] -- cross builds according to options
install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
test [ -coverage ] [ packages... ] -- runs the tests
lint -- runs certain pre-selected linters
importkeys -- imports signing keys from env
xgo [ -alltools ] [ options ] -- cross builds according to options

For all commands, -n prevents execution of external programs (dry run mode).

*/
package main

Expand Down Expand Up @@ -62,6 +62,7 @@ var (
executablePath("rlpdump"),
executablePath("swarm"),
executablePath("wnode"),
executablePath("rlp/rlpgen"),
}
)

Expand Down
2 changes: 1 addition & 1 deletion cmd/faucet/faucet.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ type faucet struct {
index []byte // Index page to serve up on the web

keystore *keystore.KeyStore // Keystore containing the single signer
account accounts.Account // Account funding user faucet requests
account accounts.Account // StateAccount funding user faucet requests
nonce uint64 // Current pending nonce of the faucet
price *big.Int // Current gas price to issue funds with

Expand Down
23 changes: 11 additions & 12 deletions cmd/gc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,21 @@ package main
import (
"flag"
"fmt"
"github.com/tomochain/tomochain/core/rawdb"
"github.com/tomochain/tomochain/ethdb"
"github.com/tomochain/tomochain/ethdb/leveldb"
"os"
"os/signal"
"runtime"
"sync"
"sync/atomic"
"time"

"github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru"
"github.com/tomochain/tomochain/cmd/utils"
"github.com/tomochain/tomochain/common"
"github.com/tomochain/tomochain/core"
"github.com/tomochain/tomochain/core/state"
"github.com/tomochain/tomochain/core/rawdb"
"github.com/tomochain/tomochain/core/types"
"github.com/tomochain/tomochain/eth"
"github.com/tomochain/tomochain/ethdb"
"github.com/tomochain/tomochain/ethdb/leveldb"
"github.com/tomochain/tomochain/rlp"
"github.com/tomochain/tomochain/trie"
)
Expand Down Expand Up @@ -54,15 +53,15 @@ func main() {
flag.Parse()
db, _ := leveldb.New(*dir, eth.DefaultConfig.DatabaseCache, utils.MakeDatabaseHandles(), "")
lddb := rawdb.NewDatabase(db)
head := core.GetHeadBlockHash(lddb)
currentHeader := core.GetHeader(lddb, head, core.GetBlockNumber(lddb, head))
head := rawdb.GetHeadBlockHash(lddb)
currentHeader := rawdb.GetHeader(lddb, head, rawdb.GetBlockNumber(lddb, head))
tridb := trie.NewDatabase(lddb)
catchEventInterupt(db)
cache, _ = lru.New(*cacheSize)
go func() {
for i := uint64(1); i <= currentHeader.Number.Uint64(); i++ {
hash := core.GetCanonicalHash(lddb, i)
root := core.GetHeader(lddb, hash, i).Root
hash := rawdb.GetCanonicalHash(lddb, i)
root := rawdb.GetHeader(lddb, hash, i).Root
trieRoot, err := trie.NewSecure(root, tridb)
if err != nil {
continue
Expand All @@ -81,8 +80,8 @@ func main() {
atomic.StoreInt32(&finish, 1)
if running {
for _, address := range cleanAddress {
enc := trieRoot.trie.Get(address.Bytes())
var data state.Account
enc := trieRoot.trie.MustGet(address.Bytes())
var data types.StateAccount
rlp.DecodeBytes(enc, &data)
fmt.Println(time.Now().Format(time.RFC3339), "Start clean state address ", address.Hex(), " at block ", trieRoot.number)
signerRoot, err := resolveHash(data.Root[:], db)
Expand Down
5 changes: 2 additions & 3 deletions cmd/tomo/dao_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,14 @@
package main

import (
"github.com/tomochain/tomochain/core/rawdb"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"testing"

"github.com/tomochain/tomochain/common"
"github.com/tomochain/tomochain/core"
"github.com/tomochain/tomochain/core/rawdb"
)

// Genesis block for nodes which don't care about the DAO fork (i.e. not configured)
Expand Down Expand Up @@ -130,7 +129,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
if genesis != "" {
genesisHash = daoGenesisHash
}
config, err := core.GetChainConfig(db, genesisHash)
config, err := rawdb.GetChainConfig(db, genesisHash)
if err != nil {
t.Errorf("test %d: failed to retrieve chain config: %v", test, err)
return // we want to return here, the other checks can't make it past this point (nil panic).
Expand Down
5 changes: 3 additions & 2 deletions cmd/utils/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (

"github.com/tomochain/tomochain/common"
"github.com/tomochain/tomochain/core"
"github.com/tomochain/tomochain/core/rawdb"
"github.com/tomochain/tomochain/core/types"
"github.com/tomochain/tomochain/crypto"
"github.com/tomochain/tomochain/ethdb"
Expand Down Expand Up @@ -271,15 +272,15 @@ func ImportPreimages(db ethdb.Database, fn string) error {
// Accumulate the preimages and flush when enough ws gathered
preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
if len(preimages) > 1024 {
if err := core.WritePreimages(db, 0, preimages); err != nil {
if err := rawdb.WritePreimages(db, 0, preimages); err != nil {
return err
}
preimages = make(map[common.Hash][]byte)
}
}
// Flush the last batch preimage data
if len(preimages) > 0 {
return core.WritePreimages(db, 0, preimages)
return rawdb.WritePreimages(db, 0, preimages)
}
return nil
}
Expand Down
3 changes: 2 additions & 1 deletion consensus/clique/clique.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"github.com/tomochain/tomochain/params"
"github.com/tomochain/tomochain/rlp"
"github.com/tomochain/tomochain/rpc"
"github.com/tomochain/tomochain/trie"
)

const (
Expand Down Expand Up @@ -575,7 +576,7 @@ func (c *Clique) Finalize(chain consensus.ChainReader, header *types.Header, sta
header.UncleHash = types.CalcUncleHash(nil)

// Assemble and return the final block for sealing
return types.NewBlock(header, txs, nil, receipts), nil
return types.NewBlock(header, txs, nil, receipts, new(trie.StackTrie)), nil
}

// Authorize injects a private key into the consensus engine to mint new blocks
Expand Down
2 changes: 1 addition & 1 deletion consensus/clique/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
type Vote struct {
Signer common.Address `json:"signer"` // Authorized signer that cast this vote
Block uint64 `json:"block"` // Block number the vote was cast in (expire old votes)
Address common.Address `json:"address"` // Account being voted on to change its authorization
Address common.Address `json:"address"` // StateAccount being voted on to change its authorization
Authorize bool `json:"authorize"` // Whether to authorize or deauthorize the voted account
}

Expand Down
3 changes: 2 additions & 1 deletion consensus/ethash/consensus.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/tomochain/tomochain/core/state"
"github.com/tomochain/tomochain/core/types"
"github.com/tomochain/tomochain/params"
"github.com/tomochain/tomochain/trie"
)

// Ethash proof-of-work protocol constants.
Expand Down Expand Up @@ -519,7 +520,7 @@ func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header
header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))

// Header seems complete, assemble into a block and return
return types.NewBlock(header, txs, uncles, receipts), nil
return types.NewBlock(header, txs, uncles, receipts, new(trie.StackTrie)), nil
}

// Some weird constants to avoid constant memory allocs for them.
Expand Down
14 changes: 8 additions & 6 deletions consensus/posv/posv.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,6 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/tomochain/tomochain/tomox/tradingstate"
"github.com/tomochain/tomochain/tomoxlending/lendingstate"
"gopkg.in/karalabe/cookiejar.v2/collections/prque"
"io/ioutil"
"math/big"
"math/rand"
Expand All @@ -35,6 +32,8 @@ import (
"time"

lru "github.com/hashicorp/golang-lru"
"gopkg.in/karalabe/cookiejar.v2/collections/prque"

"github.com/tomochain/tomochain/accounts"
"github.com/tomochain/tomochain/common"
"github.com/tomochain/tomochain/common/hexutil"
Expand All @@ -50,6 +49,9 @@ import (
"github.com/tomochain/tomochain/params"
"github.com/tomochain/tomochain/rlp"
"github.com/tomochain/tomochain/rpc"
"github.com/tomochain/tomochain/tomox/tradingstate"
"github.com/tomochain/tomochain/tomoxlending/lendingstate"
"github.com/tomochain/tomochain/trie"
)

const (
Expand Down Expand Up @@ -181,7 +183,7 @@ var (

// SignerFn is a signer callback function to request a hash to be signed by a
// backing account.
//type SignerFn func(accounts.Account, []byte) ([]byte, error)
//type SignerFn func(accounts.StateAccount, []byte) ([]byte, error)

// sigHash returns the hash which is used as input for the proof-of-stake-voting
// signing. It is the hash of the entire header apart from the 65 byte signature
Expand Down Expand Up @@ -985,7 +987,7 @@ func (c *Posv) Finalize(chain consensus.ChainReader, header *types.Header, state
header.UncleHash = types.CalcUncleHash(nil)

// Assemble and return the final block for sealing
return types.NewBlock(header, txs, nil, receipts), nil
return types.NewBlock(header, txs, nil, receipts, new(trie.StackTrie)), nil
}

// Authorize injects a private key into the consensus engine to mint new blocks
Expand Down Expand Up @@ -1146,7 +1148,7 @@ func (c *Posv) CacheData(header *types.Header, txs []*types.Transaction, receipt
signTxs := []*types.Transaction{}
for _, tx := range txs {
if tx.IsSigningTransaction() {
var b uint
var b uint64
for _, r := range receipts {
if r.TxHash == tx.Hash() {
if len(r.PostState) > 0 {
Expand Down
2 changes: 1 addition & 1 deletion consensus/posv/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
//type Vote struct {
// Signer common.Address `json:"signer"` // Authorized signer that cast this vote
// Block uint64 `json:"block"` // Block number the vote was cast in (expire old votes)
// Address common.Address `json:"address"` // Account being voted on to change its authorization
// Address common.Address `json:"address"` // StateAccount being voted on to change its authorization
// Authorize bool `json:"authorize"` // Whether to authorize or deauthorize the voted account
//}

Expand Down
3 changes: 2 additions & 1 deletion contracts/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/tomochain/tomochain/contracts/blocksigner/contract"
randomizeContract "github.com/tomochain/tomochain/contracts/randomize/contract"
"github.com/tomochain/tomochain/core"
"github.com/tomochain/tomochain/core/rawdb"
"github.com/tomochain/tomochain/core/state"
stateDatabase "github.com/tomochain/tomochain/core/state"
"github.com/tomochain/tomochain/core/types"
Expand Down Expand Up @@ -336,7 +337,7 @@ func GetRewardForCheckpoint(c *posv.Posv, chain consensus.ChainReader, header *t
block := chain.GetBlock(header.Hash(), i)
txs := block.Transactions()
if !chain.Config().IsTIPSigning(header.Number) {
receipts := core.GetBlockReceipts(c.GetDb(), header.Hash(), i)
receipts := rawdb.GetBlockReceipts(c.GetDb(), header.Hash(), i)
signData = c.CacheData(header, txs, receipts)
} else {
signData = c.CacheSigner(header.Hash(), txs)
Expand Down