From e228b5f4b2d0bbae5103418693652a0582b61b76 Mon Sep 17 00:00:00 2001 From: qianbin Date: Tue, 1 Mar 2022 14:39:02 +0800 Subject: [PATCH 01/25] trie: implement & apply fastNodeEncoder --- trie/database.go | 2 +- trie/fast_node_encoder.go | 109 ++++++++++++++++++++++++++++++++++++++ trie/hasher.go | 5 +- trie/iterator.go | 3 +- trie/node_test.go | 26 +++++++++ trie/proof.go | 3 +- trie/stacktrie.go | 11 ++-- 7 files changed, 144 insertions(+), 15 deletions(-) create mode 100644 trie/fast_node_encoder.go diff --git a/trie/database.go b/trie/database.go index 58ca4e6f3caab..dd0e82cc2aacc 100644 --- a/trie/database.go +++ b/trie/database.go @@ -164,7 +164,7 @@ func (n *cachedNode) rlp() []byte { if node, ok := n.node.(rawNode); ok { return node } - blob, err := rlp.EncodeToBytes(n.node) + blob, err := frlp.EncodeToBytes(n.node) if err != nil { panic(err) } diff --git a/trie/fast_node_encoder.go b/trie/fast_node_encoder.go new file mode 100644 index 0000000000000..4181b12013062 --- /dev/null +++ b/trie/fast_node_encoder.go @@ -0,0 +1,109 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "io" + + "github.com/ethereum/go-ethereum/rlp" +) + +// fastNodeEncoder is the fast node encoder using rlp.EncoderBuffer. +type fastNodeEncoder struct{} + +var frlp fastNodeEncoder + +// Encode writes the RLP encoding of node to w. +func (fastNodeEncoder) Encode(w io.Writer, node node) error { + enc := rlp.NewEncoderBuffer(w) + if err := fastEncodeNode(&enc, node); err != nil { + return err + } + return enc.Flush() +} + +// EncodeToBytes returns the RLP encoding of node. +func (fastNodeEncoder) EncodeToBytes(node node) ([]byte, error) { + enc := rlp.NewEncoderBuffer(nil) + defer enc.Flush() + + if err := fastEncodeNode(&enc, node); err != nil { + return nil, err + } + return enc.ToBytes(), nil +} + +func fastEncodeNode(w *rlp.EncoderBuffer, n node) error { + switch n := n.(type) { + case *fullNode: + offset := w.List() + for _, c := range n.Children { + if c != nil { + if err := fastEncodeNode(w, c); err != nil { + return err + } + } else { + w.Write(rlp.EmptyString) + } + } + w.ListEnd(offset) + case *shortNode: + offset := w.List() + w.WriteBytes(n.Key) + if n.Val != nil { + if err := fastEncodeNode(w, n.Val); err != nil { + return err + } + } else { + w.Write(rlp.EmptyString) + } + w.ListEnd(offset) + case hashNode: + w.WriteBytes(n) + case valueNode: + w.WriteBytes(n) + case rawFullNode: + offset := w.List() + for _, c := range n { + if c != nil { + if err := fastEncodeNode(w, c); err != nil { + return err + } + } else { + w.Write(rlp.EmptyString) + } + } + w.ListEnd(offset) + case *rawShortNode: + offset := w.List() + w.WriteBytes(n.Key) + if n.Val != nil { + if err := fastEncodeNode(w, n.Val); err != nil { + return err + } + } else { + w.Write(rlp.EmptyString) + } + w.ListEnd(offset) + case rawNode: + w.Write(n) + default: + return fmt.Errorf("unexpected node type: %T", n) + } + return nil +} diff --git a/trie/hasher.go b/trie/hasher.go index 3a62a2f1199c2..83d3b43982c16 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -20,7 +20,6 @@ import ( "sync" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) @@ -154,7 +153,7 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // If the rlp data is smaller than 32 bytes, `nil` is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { h.tmp.Reset() - if err := rlp.Encode(&h.tmp, n); err != nil { + if err := frlp.Encode(&h.tmp, n); err != nil { panic("encode error: " + err.Error()) } @@ -169,7 +168,7 @@ func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { h.tmp.Reset() // Generate the RLP encoding of the node - if err := n.EncodeRLP(&h.tmp); err != nil { + if err := frlp.Encode(&h.tmp, n); err != nil { panic("encode error: " + err.Error()) } diff --git a/trie/iterator.go b/trie/iterator.go index 9b7d97a5f58b0..568edaefcef66 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" ) // Iterator is a key-value trie iterator that traverses a Trie. @@ -214,7 +213,7 @@ func (it *nodeIterator) LeafProof() [][]byte { // Gather nodes that end up as hash nodes (or the root) node, hashed := hasher.proofHash(item.node) if _, ok := hashed.(hashNode); ok || i == 0 { - enc, _ := rlp.EncodeToBytes(node) + enc, _ := frlp.EncodeToBytes(node) proofs = append(proofs, enc) } } diff --git a/trie/node_test.go b/trie/node_test.go index 52720f1c776ee..1533e9241c3e9 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -92,3 +92,29 @@ func TestDecodeFullNode(t *testing.T) { t.Fatalf("decode full node err: %v", err) } } + +func BenchmarkEncodeFullNode(b *testing.B) { + var buf sliceBuffer + var fn fullNode + for i := 0; i < 16; i++ { + fn.Children[i] = hashNode(randBytes(32)) + } + + for i := 0; i < b.N; i++ { + buf.Reset() + rlp.Encode(&buf, &fn) + } +} + +func BenchmarkFastEncodeFullNode(b *testing.B) { + var buf sliceBuffer + var fn fullNode + for i := 0; i < 16; i++ { + fn.Children[i] = hashNode(randBytes(32)) + } + + for i := 0; i < b.N; i++ { + buf.Reset() + frlp.Encode(&buf, &fn) + } +} diff --git a/trie/proof.go b/trie/proof.go index 9be3b62216a80..dd97d101b42c3 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) // Prove constructs a merkle proof for key. The result contains all encoded nodes @@ -79,7 +78,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e if hash, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. - enc, _ := rlp.EncodeToBytes(n) + enc, _ := frlp.EncodeToBytes(n) if !ok { hash = hasher.hashData(enc) } diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 76258c31123c2..33d989ec4ce48 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -376,7 +376,7 @@ func (st *StackTrie) hash() { switch st.nodeType { case branchNode: - var nodes [17]node + var nodes rawFullNode for i, child := range st.children { if child == nil { nodes[i] = nilValueNode @@ -395,7 +395,7 @@ func (st *StackTrie) hash() { h = newHasher(false) defer returnHasherToPool(h) h.tmp.Reset() - if err := rlp.Encode(&h.tmp, nodes); err != nil { + if err := frlp.Encode(&h.tmp, nodes); err != nil { panic(err) } case extNode: @@ -409,14 +409,11 @@ func (st *StackTrie) hash() { } else { valuenode = hashNode(st.children[0].val) } - n := struct { - Key []byte - Val node - }{ + n := &rawShortNode{ Key: hexToCompact(st.key), Val: valuenode, } - if err := rlp.Encode(&h.tmp, n); err != nil { + if err := frlp.Encode(&h.tmp, n); err != nil { panic(err) } returnToPool(st.children[0]) From 8dc0bc28b9e77ace0ffd5c18fa51365ae70bab0a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 12:57:43 +0100 Subject: [PATCH 02/25] trie: remove unused 'tmp' buffer in committer --- trie/committer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/trie/committer.go b/trie/committer.go index 0721990a21799..db753e2fa0c4c 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -44,7 +44,6 @@ type leaf struct { // By 'some level' of parallelism, it's still the case that all leaves will be // processed sequentially - onleaf will never be called in parallel or out of order. type committer struct { - tmp sliceBuffer sha crypto.KeccakState onleaf LeafCallback @@ -55,7 +54,6 @@ type committer struct { var committerPool = sync.Pool{ New: func() interface{} { return &committer{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), } }, From 433515da3790a56b085919059b8220315322f9c4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 13:08:04 +0100 Subject: [PATCH 03/25] rlp: add AppendToBytes --- rlp/encbuffer.go | 32 ++++++++++++++++++++++---------- rlp/encode.go | 4 ++-- rlp/encode_test.go | 15 +++++++++++++++ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go index 50b83099c3496..b4e7e6160bafb 100644 --- a/rlp/encbuffer.go +++ b/rlp/encbuffer.go @@ -36,27 +36,31 @@ func (buf *encBuffer) size() int { return len(buf.str) + buf.lhsize } -// toBytes creates the encoder output. -func (w *encBuffer) toBytes() []byte { +// makeBytes creates the encoder output. +func (w *encBuffer) makeBytes() []byte { out := make([]byte, w.size()) + w.copyTo(out) + return out +} + +func (w *encBuffer) copyTo(dst []byte) { strpos := 0 pos := 0 for _, head := range w.lheads { // write string data before header - n := copy(out[pos:], w.str[strpos:head.offset]) + n := copy(dst[pos:], w.str[strpos:head.offset]) pos += n strpos += n // write the header - enc := head.encode(out[pos:]) + enc := head.encode(dst[pos:]) pos += len(enc) } // copy string data after the last list header - copy(out[pos:], w.str[strpos:]) - return out + copy(dst[pos:], w.str[strpos:]) } -// toWriter writes the encoder output to w. -func (buf *encBuffer) toWriter(w io.Writer) (err error) { +// writeTo writes the encoder output to w. +func (buf *encBuffer) writeTo(w io.Writer) (err error) { strpos := 0 for _, head := range buf.lheads { // write string data before header @@ -303,7 +307,7 @@ func (w *EncoderBuffer) Reset(dst io.Writer) { func (w *EncoderBuffer) Flush() error { var err error if w.dst != nil { - err = w.buf.toWriter(w.dst) + err = w.buf.writeTo(w.dst) } // Release the internal buffer. if w.ownBuffer { @@ -315,7 +319,15 @@ func (w *EncoderBuffer) Flush() error { // ToBytes returns the encoded bytes. func (w *EncoderBuffer) ToBytes() []byte { - return w.buf.toBytes() + return w.buf.makeBytes() +} + +// AppendToBytes appends the encoded value to dst. +func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { + size := w.buf.size() + out := append(dst, make([]byte, size)...) + w.buf.copyTo(out[len(dst):]) + return out } // Write appends b directly to the encoder output. diff --git a/rlp/encode.go b/rlp/encode.go index c3523369f616a..7d83cf6e2bf6a 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -69,7 +69,7 @@ func Encode(w io.Writer, val interface{}) error { if err := buf.encode(val); err != nil { return err } - return buf.toWriter(w) + return buf.writeTo(w) } // EncodeToBytes returns the RLP encoding of val. @@ -81,7 +81,7 @@ func EncodeToBytes(val interface{}) ([]byte, error) { if err := buf.encode(val); err != nil { return nil, err } - return buf.toBytes(), nil + return buf.makeBytes(), nil } // EncodeToReader returns a reader from which the RLP encoding of val diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 5dcfd78fd45bf..1d715e3776c37 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -399,6 +399,21 @@ func TestEncodeToBytes(t *testing.T) { runEncTests(t, EncodeToBytes) } +func TestEncodeAppendToBytes(t *testing.T) { + buffer := make([]byte, 20) + runEncTests(t, func(val interface{}) ([]byte, error) { + w := NewEncoderBuffer(nil) + defer w.Flush() + + err := Encode(w, val) + if err != nil { + return nil, err + } + output := w.AppendToBytes(buffer[:0]) + return output, nil + }) +} + func TestEncodeToReader(t *testing.T) { runEncTests(t, func(val interface{}) ([]byte, error) { _, r, err := EncodeToReader(val) From 59ae72b87465a0758a00ac3fdfb2fb51e7061e07 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 13:09:09 +0100 Subject: [PATCH 04/25] trie: add encode() method to node --- trie/database.go | 6 +--- trie/iterator.go | 3 +- trie/node.go | 1 + trie/node_enc.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ trie/proof.go | 4 +-- 5 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 trie/node_enc.go diff --git a/trie/database.go b/trie/database.go index dd0e82cc2aacc..80d1d9231a067 100644 --- a/trie/database.go +++ b/trie/database.go @@ -164,11 +164,7 @@ func (n *cachedNode) rlp() []byte { if node, ok := n.node.(rawNode); ok { return node } - blob, err := frlp.EncodeToBytes(n.node) - if err != nil { - panic(err) - } - return blob + return nodeToBytes(n.node) } // obj returns the decoded and expanded trie node, either directly from the cache, diff --git a/trie/iterator.go b/trie/iterator.go index 568edaefcef66..1fbf991af09e1 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -213,8 +213,7 @@ func (it *nodeIterator) LeafProof() [][]byte { // Gather nodes that end up as hash nodes (or the root) node, hashed := hasher.proofHash(item.node) if _, ok := hashed.(hashNode); ok || i == 0 { - enc, _ := frlp.EncodeToBytes(node) - proofs = append(proofs, enc) + proofs = append(proofs, nodeToBytes(node)) } } return proofs diff --git a/trie/node.go b/trie/node.go index f4055e779a1b7..aea16a457e83a 100644 --- a/trie/node.go +++ b/trie/node.go @@ -29,6 +29,7 @@ var indices = []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b type node interface { fstring(string) string + encode(w *rlp.EncoderBuffer) cache() (hashNode, bool) } diff --git a/trie/node_enc.go b/trie/node_enc.go new file mode 100644 index 0000000000000..70053a5af2362 --- /dev/null +++ b/trie/node_enc.go @@ -0,0 +1,79 @@ +package trie + +import ( + "github.com/ethereum/go-ethereum/rlp" +) + +func nodeToBytes(n node) []byte { + w := rlp.NewEncoderBuffer(nil) + n.encode(&w) + result := w.ToBytes() + w.Flush() + return result +} + +func appendEncodedNode(n node, dst []byte) []byte { + w := rlp.NewEncoderBuffer(nil) + n.encode(&w) + result := w.AppendToBytes(dst) + w.Flush() + return result +} + +func (n *fullNode) encode(w *rlp.EncoderBuffer) { + offset := w.List() + for _, c := range n.Children { + if c != nil { + c.encode(w) + } else { + w.Write(rlp.EmptyString) + } + } + w.ListEnd(offset) +} + +func (n *shortNode) encode(w *rlp.EncoderBuffer) { + offset := w.List() + w.WriteBytes(n.Key) + if n.Val != nil { + n.Val.encode(w) + } else { + w.Write(rlp.EmptyString) + } + w.ListEnd(offset) +} + +func (n hashNode) encode(w *rlp.EncoderBuffer) { + w.WriteBytes(n) +} + +func (n valueNode) encode(w *rlp.EncoderBuffer) { + w.WriteBytes(n) +} + +func (n rawFullNode) encode(w *rlp.EncoderBuffer) { + offset := w.List() + for _, c := range n { + if c != nil { + c.encode(w) + } else { + w.Write(rlp.EmptyString) + } + } + w.ListEnd(offset) +} + +func (n *rawShortNode) encode(w *rlp.EncoderBuffer) { + offset := w.List() + w.WriteBytes(n.Key) + if n.Val != nil { + n.Val.encode(w) + } else { + w.Write(rlp.EmptyString) + } + w.ListEnd(offset) +} + +func (n rawNode) encode(w *rlp.EncoderBuffer) { + w.Write(n) +} diff --git a/trie/proof.go b/trie/proof.go index dd97d101b42c3..a617ab4a4e17a 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -76,9 +76,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e var hn node n, hn = hasher.proofHash(n) if hash, ok := hn.(hashNode); ok || i == 0 { - // If the node's database encoding is a hash (or is the - // root node), it becomes a proof element. - enc, _ := frlp.EncodeToBytes(n) + enc := nodeToBytes(n) if !ok { hash = hasher.hashData(enc) } From 470249b7d2ec884b63a5cb7431254aef113652e7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 13:11:44 +0100 Subject: [PATCH 05/25] trie: don't print to stdout in tests --- trie/trie_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/trie/trie_test.go b/trie/trie_test.go index 3097c67f0d915..63aed333dbf14 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -414,8 +414,9 @@ func runRandTest(rt randTest) bool { values := make(map[string]string) // tracks content of the trie for i, step := range rt { - fmt.Printf("{op: %d, key: common.Hex2Bytes(\"%x\"), value: common.Hex2Bytes(\"%x\")}, // step %d\n", - step.op, step.key, step.value, i) + // fmt.Printf("{op: %d, key: common.Hex2Bytes(\"%x\"), value: common.Hex2Bytes(\"%x\")}, // step %d\n", + // step.op, step.key, step.value, i) + switch step.op { case opUpdate: tr.Update(step.key, step.value) @@ -885,7 +886,8 @@ func TestCommitSequenceSmallRoot(t *testing.T) { if stRoot != root { t.Fatalf("root wrong, got %x exp %x", stRoot, root) } - fmt.Printf("root: %x\n", stRoot) + + t.Logf("root: %x\n", stRoot) if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) { t.Fatalf("test, disk write sequence wrong:\ngot %x exp %x\n", got, exp) } From ef32db69b45975326c52ef1a187d5bf700af855f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 13:13:53 +0100 Subject: [PATCH 06/25] trie: use new encoder in hasher --- trie/hasher.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 83d3b43982c16..0b87613fa80c1 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -152,30 +152,23 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // into compact form for RLP encoding. // If the rlp data is smaller than 32 bytes, `nil` is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { - h.tmp.Reset() - if err := frlp.Encode(&h.tmp, n); err != nil { - panic("encode error: " + err.Error()) - } - - if len(h.tmp) < 32 && !force { + enc := appendEncodedNode(n, h.tmp[:0]) + h.tmp = enc + if len(enc) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(h.tmp) + return h.hashData(enc) } // shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which // may contain nil values) func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { - h.tmp.Reset() - // Generate the RLP encoding of the node - if err := frlp.Encode(&h.tmp, n); err != nil { - panic("encode error: " + err.Error()) - } - + enc := appendEncodedNode(n, h.tmp[:0]) + h.tmp = enc if len(h.tmp) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(h.tmp) + return h.hashData(enc) } // hashData hashes the provided data From 76c74674bbf3a74476311d14f401db5346934473 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 13:40:34 +0100 Subject: [PATCH 07/25] trie: use new encoder in StackTrie --- trie/fast_node_encoder.go | 109 ----------------------- trie/node_test.go | 13 --- trie/stacktrie.go | 179 +++++++++++++++++++++----------------- 3 files changed, 101 insertions(+), 200 deletions(-) delete mode 100644 trie/fast_node_encoder.go diff --git a/trie/fast_node_encoder.go b/trie/fast_node_encoder.go deleted file mode 100644 index 4181b12013062..0000000000000 --- a/trie/fast_node_encoder.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "fmt" - "io" - - "github.com/ethereum/go-ethereum/rlp" -) - -// fastNodeEncoder is the fast node encoder using rlp.EncoderBuffer. -type fastNodeEncoder struct{} - -var frlp fastNodeEncoder - -// Encode writes the RLP encoding of node to w. -func (fastNodeEncoder) Encode(w io.Writer, node node) error { - enc := rlp.NewEncoderBuffer(w) - if err := fastEncodeNode(&enc, node); err != nil { - return err - } - return enc.Flush() -} - -// EncodeToBytes returns the RLP encoding of node. -func (fastNodeEncoder) EncodeToBytes(node node) ([]byte, error) { - enc := rlp.NewEncoderBuffer(nil) - defer enc.Flush() - - if err := fastEncodeNode(&enc, node); err != nil { - return nil, err - } - return enc.ToBytes(), nil -} - -func fastEncodeNode(w *rlp.EncoderBuffer, n node) error { - switch n := n.(type) { - case *fullNode: - offset := w.List() - for _, c := range n.Children { - if c != nil { - if err := fastEncodeNode(w, c); err != nil { - return err - } - } else { - w.Write(rlp.EmptyString) - } - } - w.ListEnd(offset) - case *shortNode: - offset := w.List() - w.WriteBytes(n.Key) - if n.Val != nil { - if err := fastEncodeNode(w, n.Val); err != nil { - return err - } - } else { - w.Write(rlp.EmptyString) - } - w.ListEnd(offset) - case hashNode: - w.WriteBytes(n) - case valueNode: - w.WriteBytes(n) - case rawFullNode: - offset := w.List() - for _, c := range n { - if c != nil { - if err := fastEncodeNode(w, c); err != nil { - return err - } - } else { - w.Write(rlp.EmptyString) - } - } - w.ListEnd(offset) - case *rawShortNode: - offset := w.List() - w.WriteBytes(n.Key) - if n.Val != nil { - if err := fastEncodeNode(w, n.Val); err != nil { - return err - } - } else { - w.Write(rlp.EmptyString) - } - w.ListEnd(offset) - case rawNode: - w.Write(n) - default: - return fmt.Errorf("unexpected node type: %T", n) - } - return nil -} diff --git a/trie/node_test.go b/trie/node_test.go index 1533e9241c3e9..8c70680c2c866 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -105,16 +105,3 @@ func BenchmarkEncodeFullNode(b *testing.B) { rlp.Encode(&buf, &fn) } } - -func BenchmarkFastEncodeFullNode(b *testing.B) { - var buf sliceBuffer - var fn fullNode - for i := 0; i < 16; i++ { - fn.Children[i] = hashNode(randBytes(32)) - } - - for i := 0; i < b.N; i++ { - buf.Reset() - frlp.Encode(&buf, &fn) - } -} diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 33d989ec4ce48..e1e6a3970ca6c 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -352,8 +352,34 @@ func (st *StackTrie) insert(key, value []byte) { } } -// hash() hashes the node 'st' and converts it into 'hashedNode', if possible. -// Possible outcomes: +type stackTrieHashContext struct { + h *hasher + encbuf rlp.EncoderBuffer +} + +func newStackTrieHashContext() stackTrieHashContext { + return stackTrieHashContext{ + h: newHasher(false), + encbuf: rlp.NewEncoderBuffer(nil), + } +} + +func (ctx *stackTrieHashContext) release() { + returnHasherToPool(ctx.h) + ctx.encbuf.Flush() + *ctx = stackTrieHashContext{} +} + +func (ctx *stackTrieHashContext) encode(n node) []byte { + ctx.encbuf.Reset(nil) + n.encode(&ctx.encbuf) + enc := ctx.encbuf.AppendToBytes(ctx.h.tmp[:0]) + ctx.h.tmp = enc + return enc +} + +// hash converts st into a 'hashedNode', if possible. Possible outcomes: +// // 1. The rlp-encoded value was >= 32 bytes: // - Then the 32-byte `hash` will be accessible in `st.val`. // - And the 'st.type' will be 'hashedNode' @@ -361,18 +387,21 @@ func (st *StackTrie) insert(key, value []byte) { // - Then the <32 byte rlp-encoded value will be accessible in 'st.val'. // - And the 'st.type' will be 'hashedNode' AGAIN // -// This method will also: -// set 'st.type' to hashedNode -// clear 'st.key' +// This method also sets 'st.type' to hashedNode, and clears 'st.key'. func (st *StackTrie) hash() { - /* Shortcut if node is already hashed */ + ctx := newStackTrieHashContext() + defer ctx.release() + + st.hashRec(ctx) +} + +func (st *StackTrie) hashRec(ctx stackTrieHashContext) { + // Shortcut if node is already hashed. if st.nodeType == hashedNode { return } - // The 'hasher' is taken from a pool, but we don't actually - // claim an instance until all children are done with their hashing, - // and we actually need one - var h *hasher + + var encodedNode []byte switch st.nodeType { case branchNode: @@ -382,95 +411,86 @@ func (st *StackTrie) hash() { nodes[i] = nilValueNode continue } - child.hash() + child.hashRec(ctx) if len(child.val) < 32 { nodes[i] = rawNode(child.val) } else { nodes[i] = hashNode(child.val) } - st.children[i] = nil // Reclaim mem from subtree + + // Release children back to pool. + st.children[i] = nil returnToPool(child) } - nodes[16] = nilValueNode - h = newHasher(false) - defer returnHasherToPool(h) - h.tmp.Reset() - if err := frlp.Encode(&h.tmp, nodes); err != nil { - panic(err) - } + encodedNode = ctx.encode(nodes) + case extNode: - st.children[0].hash() - h = newHasher(false) - defer returnHasherToPool(h) - h.tmp.Reset() - var valuenode node + st.children[0].hashRec(ctx) + + // TODO(fjl): can this use hexToCompactInPlace? + n := &rawShortNode{Key: hexToCompact(st.key)} if len(st.children[0].val) < 32 { - valuenode = rawNode(st.children[0].val) + n.Val = rawNode(st.children[0].val) } else { - valuenode = hashNode(st.children[0].val) - } - n := &rawShortNode{ - Key: hexToCompact(st.key), - Val: valuenode, - } - if err := frlp.Encode(&h.tmp, n); err != nil { - panic(err) + n.Val = hashNode(st.children[0].val) } + encodedNode = ctx.encode(n) + + // Release child back to pool. returnToPool(st.children[0]) - st.children[0] = nil // Reclaim mem from subtree + st.children[0] = nil + case leafNode: - h = newHasher(false) - defer returnHasherToPool(h) - h.tmp.Reset() st.key = append(st.key, byte(16)) sz := hexToCompactInPlace(st.key) - n := [][]byte{st.key[:sz], st.val} - if err := rlp.Encode(&h.tmp, n); err != nil { - panic(err) - } + n := &rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} + encodedNode = ctx.encode(n) + case emptyNode: st.val = emptyRoot.Bytes() st.key = st.key[:0] st.nodeType = hashedNode return + default: - panic("Invalid node type") + panic("invalid node type") } + st.key = st.key[:0] st.nodeType = hashedNode - if len(h.tmp) < 32 { - st.val = common.CopyBytes(h.tmp) + if len(encodedNode) < 32 { + st.val = common.CopyBytes(encodedNode) return } + // Write the hash to the 'val'. We allocate a new val here to not mutate // input values - st.val = make([]byte, 32) - h.sha.Reset() - h.sha.Write(h.tmp) - h.sha.Read(st.val) + st.val = ctx.h.hashData(encodedNode) if st.db != nil { // TODO! Is it safe to Put the slice here? // Do all db implementations copy the value provided? - st.db.Put(st.val, h.tmp) + st.db.Put(st.val, encodedNode) } } -// Hash returns the hash of the current node +// Hash returns the hash of the current node. func (st *StackTrie) Hash() (h common.Hash) { - st.hash() - if len(st.val) != 32 { - // If the node's RLP isn't 32 bytes long, the node will not - // be hashed, and instead contain the rlp-encoding of the - // node. For the top level node, we need to force the hashing. - ret := make([]byte, 32) - h := newHasher(false) - defer returnHasherToPool(h) - h.sha.Reset() - h.sha.Write(st.val) - h.sha.Read(ret) - return common.BytesToHash(ret) + ctx := newStackTrieHashContext() + defer ctx.release() + + st.hashRec(ctx) + if len(st.val) == 32 { + copy(h[:], st.val) + return h } - return common.BytesToHash(st.val) + + // If the node's RLP isn't 32 bytes long, the node will not + // be hashed, and instead contain the rlp-encoding of the + // node. For the top level node, we need to force the hashing. + ctx.h.sha.Reset() + ctx.h.sha.Write(st.val) + ctx.h.sha.Read(h[:]) + return h } // Commit will firstly hash the entrie trie if it's still not hashed @@ -480,23 +500,26 @@ func (st *StackTrie) Hash() (h common.Hash) { // // The associated database is expected, otherwise the whole commit // functionality should be disabled. -func (st *StackTrie) Commit() (common.Hash, error) { +func (st *StackTrie) Commit() (h common.Hash, err error) { if st.db == nil { return common.Hash{}, ErrCommitDisabled } - st.hash() - if len(st.val) != 32 { - // If the node's RLP isn't 32 bytes long, the node will not - // be hashed (and committed), and instead contain the rlp-encoding of the - // node. For the top level node, we need to force the hashing+commit. - ret := make([]byte, 32) - h := newHasher(false) - defer returnHasherToPool(h) - h.sha.Reset() - h.sha.Write(st.val) - h.sha.Read(ret) - st.db.Put(ret, st.val) - return common.BytesToHash(ret), nil + + ctx := newStackTrieHashContext() + defer ctx.release() + + st.hashRec(ctx) + if len(st.val) == 32 { + copy(h[:], st.val) + return h, nil } - return common.BytesToHash(st.val), nil + + // If the node's RLP isn't 32 bytes long, the node will not + // be hashed (and committed), and instead contain the rlp-encoding of the + // node. For the top level node, we need to force the hashing+commit. + ctx.h.sha.Reset() + ctx.h.sha.Write(st.val) + ctx.h.sha.Read(h[:]) + st.db.Put(h[:], st.val) + return h, nil } From 2060ce34a2dcae5e9f736a393fe4668311fc0c72 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 14:37:24 +0100 Subject: [PATCH 08/25] rlp: apply buffer-reusing optimization for *EncoderBuffer, too --- rlp/encbuffer.go | 19 ++++++++++++++----- rlp/encode.go | 6 +----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go index b4e7e6160bafb..0d8b5a76df20d 100644 --- a/rlp/encbuffer.go +++ b/rlp/encbuffer.go @@ -256,6 +256,19 @@ func (r *encReader) next() []byte { } } +func encBufferFromWriter(w io.Writer) *encBuffer { + switch w := w.(type) { + case EncoderBuffer: + return w.buf + case *EncoderBuffer: + return w.buf + case *encBuffer: + return w + default: + return nil + } +} + // EncoderBuffer is a buffer for incremental encoding. // // The zero value is NOT ready for use. To get a usable buffer, @@ -283,14 +296,10 @@ func (w *EncoderBuffer) Reset(dst io.Writer) { // If the destination writer has an *encBuffer, use it. // Note that w.ownBuffer is left false here. if dst != nil { - if outer, ok := dst.(*encBuffer); ok { + if outer := encBufferFromWriter(dst); outer != nil { *w = EncoderBuffer{outer, nil, false} return } - if outer, ok := dst.(EncoderBuffer); ok { - *w = EncoderBuffer{outer.buf, nil, false} - return - } } // Get a fresh buffer. diff --git a/rlp/encode.go b/rlp/encode.go index 7d83cf6e2bf6a..b96505f56dfeb 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -56,16 +56,12 @@ type Encoder interface { // Please see package-level documentation of encoding rules. func Encode(w io.Writer, val interface{}) error { // Optimization: reuse *encBuffer when called by EncodeRLP. - if buf, ok := w.(*encBuffer); ok { + if buf := encBufferFromWriter(w); buf != nil { return buf.encode(val) } - if ebuf, ok := w.(EncoderBuffer); ok { - return ebuf.buf.encode(val) - } buf := getEncBuffer() defer encBufferPool.Put(buf) - if err := buf.encode(val); err != nil { return err } From 994a67a5dc30d0630db06857aa116632ab7040b1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 14:37:48 +0100 Subject: [PATCH 09/25] trie: move emptyNode case --- trie/stacktrie.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index e1e6a3970ca6c..f55383f571717 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -404,6 +404,12 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { var encodedNode []byte switch st.nodeType { + case emptyNode: + st.val = emptyRoot.Bytes() + st.key = st.key[:0] + st.nodeType = hashedNode + return + case branchNode: var nodes rawFullNode for i, child := range st.children { @@ -446,12 +452,6 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { n := &rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} encodedNode = ctx.encode(n) - case emptyNode: - st.val = emptyRoot.Bytes() - st.key = st.key[:0] - st.nodeType = hashedNode - return - default: panic("invalid node type") } From a622a8e8aaa3851a6329637c2e20ba227f6ba5f2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 14:39:07 +0100 Subject: [PATCH 10/25] trie: add comment in StackTrie.hash --- trie/stacktrie.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index f55383f571717..9a08e0be07cdf 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -396,14 +396,13 @@ func (st *StackTrie) hash() { } func (st *StackTrie) hashRec(ctx stackTrieHashContext) { - // Shortcut if node is already hashed. - if st.nodeType == hashedNode { - return - } - + // The switch below sets this to the RLP-encoding of st. var encodedNode []byte switch st.nodeType { + case hashedNode: + return + case emptyNode: st.val = emptyRoot.Bytes() st.key = st.key[:0] From 8a478b986224714d354161e43a1a5ef2ed72a9f3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 18:20:35 +0100 Subject: [PATCH 11/25] trie: implement EncodeRLP using encode --- trie/database.go | 13 +++---------- trie/node.go | 13 +++---------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/trie/database.go b/trie/database.go index 80d1d9231a067..aa381e1c19769 100644 --- a/trie/database.go +++ b/trie/database.go @@ -113,16 +113,9 @@ func (n rawFullNode) cache() (hashNode, bool) { panic("this should never end u func (n rawFullNode) fstring(ind string) string { panic("this should never end up in a live trie") } func (n rawFullNode) EncodeRLP(w io.Writer) error { - var nodes [17]node - - for i, child := range n { - if child != nil { - nodes[i] = child - } else { - nodes[i] = nilValueNode - } - } - return rlp.Encode(w, nodes) + eb := rlp.NewEncoderBuffer(w) + n.encode(&eb) + return eb.Flush() } // rawShortNode represents only the useful data content of a short node, with the diff --git a/trie/node.go b/trie/node.go index aea16a457e83a..805f3776a3f7d 100644 --- a/trie/node.go +++ b/trie/node.go @@ -53,16 +53,9 @@ var nilValueNode = valueNode(nil) // EncodeRLP encodes a full node into the consensus RLP format. func (n *fullNode) EncodeRLP(w io.Writer) error { - var nodes [17]node - - for i, child := range &n.Children { - if child != nil { - nodes[i] = child - } else { - nodes[i] = nilValueNode - } - } - return rlp.Encode(w, nodes) + eb := rlp.NewEncoderBuffer(w) + n.encode(&eb) + return eb.Flush() } func (n *fullNode) copy() *fullNode { copy := *n; return © } From a08eb13d754981389ec72e58bedd92abc18cc3a7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 19:08:18 +0100 Subject: [PATCH 12/25] trie: avoid node encoding allocs in StackTrie --- trie/stacktrie.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 9a08e0be07cdf..c84009e74419d 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -370,11 +370,16 @@ func (ctx *stackTrieHashContext) release() { *ctx = stackTrieHashContext{} } -func (ctx *stackTrieHashContext) encode(n node) []byte { - ctx.encbuf.Reset(nil) - n.encode(&ctx.encbuf) +// encodedBytes returns the result of the last encoding operation +// on ctx.encbuf. This exists because node.encode can only be inlined +// when using a concrete receiver type. +func (ctx *stackTrieHashContext) encodedBytes() []byte { enc := ctx.encbuf.AppendToBytes(ctx.h.tmp[:0]) ctx.h.tmp = enc + + // Reset the buffer here so the next encoding operation doesn't + // need to care about resetting. + ctx.encbuf.Reset(nil) return enc } @@ -427,19 +432,23 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { st.children[i] = nil returnToPool(child) } - encodedNode = ctx.encode(nodes) + + nodes.encode(&ctx.encbuf) + encodedNode = ctx.encodedBytes() case extNode: st.children[0].hashRec(ctx) // TODO(fjl): can this use hexToCompactInPlace? - n := &rawShortNode{Key: hexToCompact(st.key)} + n := rawShortNode{Key: hexToCompact(st.key)} if len(st.children[0].val) < 32 { n.Val = rawNode(st.children[0].val) } else { n.Val = hashNode(st.children[0].val) } - encodedNode = ctx.encode(n) + + n.encode(&ctx.encbuf) + encodedNode = ctx.encodedBytes() // Release child back to pool. returnToPool(st.children[0]) @@ -448,15 +457,17 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { case leafNode: st.key = append(st.key, byte(16)) sz := hexToCompactInPlace(st.key) - n := &rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} - encodedNode = ctx.encode(n) + n := rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} + + n.encode(&ctx.encbuf) + encodedNode = ctx.encodedBytes() default: panic("invalid node type") } - st.key = st.key[:0] st.nodeType = hashedNode + st.key = st.key[:0] if len(encodedNode) < 32 { st.val = common.CopyBytes(encodedNode) return From 0077809472aa44de0f765797a8e2409d09334b68 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 19:09:25 +0100 Subject: [PATCH 13/25] trie: avoid node encoding allocs in hasher --- trie/hasher.go | 27 +++++++++++++++++---------- trie/node_enc.go | 8 -------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 0b87613fa80c1..2a7d96b54387a 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -20,6 +20,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) @@ -38,7 +39,8 @@ func (b *sliceBuffer) Reset() { // internal preallocated temp space type hasher struct { sha crypto.KeccakState - tmp sliceBuffer + tmp []byte + encbuf rlp.EncoderBuffer parallel bool // Whether to use paralallel threads when hashing } @@ -46,8 +48,9 @@ type hasher struct { var hasherPool = sync.Pool{ New: func() interface{} { return &hasher{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. - sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), + tmp: make([]byte, 0, 550), // cap is as large as a full fullNode. + sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), + encbuf: rlp.NewEncoderBuffer(nil), } }, } @@ -152,23 +155,27 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // into compact form for RLP encoding. // If the rlp data is smaller than 32 bytes, `nil` is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { - enc := appendEncodedNode(n, h.tmp[:0]) - h.tmp = enc - if len(enc) < 32 && !force { + h.encbuf.Reset(nil) + n.encode(&h.encbuf) + h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) + + if len(h.tmp) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(enc) + return h.hashData(h.tmp) } // shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which // may contain nil values) func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { - enc := appendEncodedNode(n, h.tmp[:0]) - h.tmp = enc + h.encbuf.Reset(nil) + n.encode(&h.encbuf) + h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) + if len(h.tmp) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(enc) + return h.hashData(h.tmp) } // hashData hashes the provided data diff --git a/trie/node_enc.go b/trie/node_enc.go index 70053a5af2362..0acda51b6fe93 100644 --- a/trie/node_enc.go +++ b/trie/node_enc.go @@ -12,14 +12,6 @@ func nodeToBytes(n node) []byte { return result } -func appendEncodedNode(n node, dst []byte) []byte { - w := rlp.NewEncoderBuffer(nil) - n.encode(&w) - result := w.AppendToBytes(dst) - w.Flush() - return result -} - func (n *fullNode) encode(w *rlp.EncoderBuffer) { offset := w.List() for _, c := range n.Children { From a905cc862fb95d2234c29e29da6858150666aff3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 19:21:03 +0100 Subject: [PATCH 14/25] trie: use hasher instead of stackTrieHashContext --- trie/hasher.go | 29 ++++++++++++----- trie/stacktrie.go | 83 +++++++++++++++-------------------------------- 2 files changed, 47 insertions(+), 65 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 2a7d96b54387a..247882a87b418 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -155,27 +155,40 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // into compact form for RLP encoding. // If the rlp data is smaller than 32 bytes, `nil` is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { - h.encbuf.Reset(nil) n.encode(&h.encbuf) - h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) + enc := h.encodedBytes() - if len(h.tmp) < 32 && !force { + if len(enc) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(h.tmp) + return h.hashData(enc) } // shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which // may contain nil values) func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { - h.encbuf.Reset(nil) n.encode(&h.encbuf) - h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) + enc := h.encodedBytes() - if len(h.tmp) < 32 && !force { + if len(enc) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(h.tmp) + return h.hashData(enc) +} + +// encodedBytes returns the result of the last encoding operation on h.encbuf. This exists +// because node.encode can only be inlined when using a concrete receiver type. Basically, +// all node encoding must be done like this: +// +// node.encode(&h.encbuf) +// enc := h.encodedBytes() +// +func (h *hasher) encodedBytes() []byte { + h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) + // Reset the buffer here so the next encoding operation doesn't + // need to care about it. + h.encbuf.Reset(nil) + return h.tmp } // hashData hashes the provided data diff --git a/trie/stacktrie.go b/trie/stacktrie.go index c84009e74419d..e65a5c9ef84f8 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ErrCommitDisabled = errors.New("no database for committing") @@ -352,37 +351,6 @@ func (st *StackTrie) insert(key, value []byte) { } } -type stackTrieHashContext struct { - h *hasher - encbuf rlp.EncoderBuffer -} - -func newStackTrieHashContext() stackTrieHashContext { - return stackTrieHashContext{ - h: newHasher(false), - encbuf: rlp.NewEncoderBuffer(nil), - } -} - -func (ctx *stackTrieHashContext) release() { - returnHasherToPool(ctx.h) - ctx.encbuf.Flush() - *ctx = stackTrieHashContext{} -} - -// encodedBytes returns the result of the last encoding operation -// on ctx.encbuf. This exists because node.encode can only be inlined -// when using a concrete receiver type. -func (ctx *stackTrieHashContext) encodedBytes() []byte { - enc := ctx.encbuf.AppendToBytes(ctx.h.tmp[:0]) - ctx.h.tmp = enc - - // Reset the buffer here so the next encoding operation doesn't - // need to care about resetting. - ctx.encbuf.Reset(nil) - return enc -} - // hash converts st into a 'hashedNode', if possible. Possible outcomes: // // 1. The rlp-encoded value was >= 32 bytes: @@ -394,13 +362,13 @@ func (ctx *stackTrieHashContext) encodedBytes() []byte { // // This method also sets 'st.type' to hashedNode, and clears 'st.key'. func (st *StackTrie) hash() { - ctx := newStackTrieHashContext() - defer ctx.release() + h := newHasher(false) + defer returnHasherToPool(h) - st.hashRec(ctx) + st.hashRec(h) } -func (st *StackTrie) hashRec(ctx stackTrieHashContext) { +func (st *StackTrie) hashRec(hasher *hasher) { // The switch below sets this to the RLP-encoding of st. var encodedNode []byte @@ -421,7 +389,8 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { nodes[i] = nilValueNode continue } - child.hashRec(ctx) + + child.hashRec(hasher) if len(child.val) < 32 { nodes[i] = rawNode(child.val) } else { @@ -433,11 +402,11 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { returnToPool(child) } - nodes.encode(&ctx.encbuf) - encodedNode = ctx.encodedBytes() + nodes.encode(&hasher.encbuf) + encodedNode = hasher.encodedBytes() case extNode: - st.children[0].hashRec(ctx) + st.children[0].hashRec(hasher) // TODO(fjl): can this use hexToCompactInPlace? n := rawShortNode{Key: hexToCompact(st.key)} @@ -447,8 +416,8 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { n.Val = hashNode(st.children[0].val) } - n.encode(&ctx.encbuf) - encodedNode = ctx.encodedBytes() + n.encode(&hasher.encbuf) + encodedNode = hasher.encodedBytes() // Release child back to pool. returnToPool(st.children[0]) @@ -459,8 +428,8 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { sz := hexToCompactInPlace(st.key) n := rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} - n.encode(&ctx.encbuf) - encodedNode = ctx.encodedBytes() + n.encode(&hasher.encbuf) + encodedNode = hasher.encodedBytes() default: panic("invalid node type") @@ -475,7 +444,7 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { // Write the hash to the 'val'. We allocate a new val here to not mutate // input values - st.val = ctx.h.hashData(encodedNode) + st.val = hasher.hashData(encodedNode) if st.db != nil { // TODO! Is it safe to Put the slice here? // Do all db implementations copy the value provided? @@ -485,10 +454,10 @@ func (st *StackTrie) hashRec(ctx stackTrieHashContext) { // Hash returns the hash of the current node. func (st *StackTrie) Hash() (h common.Hash) { - ctx := newStackTrieHashContext() - defer ctx.release() + hasher := newHasher(false) + defer returnHasherToPool(hasher) - st.hashRec(ctx) + st.hashRec(hasher) if len(st.val) == 32 { copy(h[:], st.val) return h @@ -497,9 +466,9 @@ func (st *StackTrie) Hash() (h common.Hash) { // If the node's RLP isn't 32 bytes long, the node will not // be hashed, and instead contain the rlp-encoding of the // node. For the top level node, we need to force the hashing. - ctx.h.sha.Reset() - ctx.h.sha.Write(st.val) - ctx.h.sha.Read(h[:]) + hasher.sha.Reset() + hasher.sha.Write(st.val) + hasher.sha.Read(h[:]) return h } @@ -515,10 +484,10 @@ func (st *StackTrie) Commit() (h common.Hash, err error) { return common.Hash{}, ErrCommitDisabled } - ctx := newStackTrieHashContext() - defer ctx.release() + hasher := newHasher(false) + defer returnHasherToPool(hasher) - st.hashRec(ctx) + st.hashRec(hasher) if len(st.val) == 32 { copy(h[:], st.val) return h, nil @@ -527,9 +496,9 @@ func (st *StackTrie) Commit() (h common.Hash, err error) { // If the node's RLP isn't 32 bytes long, the node will not // be hashed (and committed), and instead contain the rlp-encoding of the // node. For the top level node, we need to force the hashing+commit. - ctx.h.sha.Reset() - ctx.h.sha.Write(st.val) - ctx.h.sha.Read(h[:]) + hasher.sha.Reset() + hasher.sha.Write(st.val) + hasher.sha.Read(h[:]) st.db.Put(h[:], st.val) return h, nil } From 727dab9882cf8713075c6e3d75ff4b1d3a1b0952 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 19:22:04 +0100 Subject: [PATCH 15/25] trie: improve comment --- trie/stacktrie.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index e65a5c9ef84f8..9a3a971222c48 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -369,7 +369,7 @@ func (st *StackTrie) hash() { } func (st *StackTrie) hashRec(hasher *hasher) { - // The switch below sets this to the RLP-encoding of st. + // The switch below sets this to the RLP-encoding of this node. var encodedNode []byte switch st.nodeType { From f5e390107ec36706ad2858dbb15e31e644aea359 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 19:46:55 +0100 Subject: [PATCH 16/25] trie: use hexToCompactInPlace for StackTrie shortnode --- trie/stacktrie.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 9a3a971222c48..9b7aa2a992c68 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -397,7 +397,7 @@ func (st *StackTrie) hashRec(hasher *hasher) { nodes[i] = hashNode(child.val) } - // Release children back to pool. + // Release child back to pool. st.children[i] = nil returnToPool(child) } @@ -408,8 +408,8 @@ func (st *StackTrie) hashRec(hasher *hasher) { case extNode: st.children[0].hashRec(hasher) - // TODO(fjl): can this use hexToCompactInPlace? - n := rawShortNode{Key: hexToCompact(st.key)} + sz := hexToCompactInPlace(st.key) + n := rawShortNode{Key: st.key[:sz]} if len(st.children[0].val) < 32 { n.Val = rawNode(st.children[0].val) } else { From faee3b07d1a7ff9b6be8baeb507b682c651df35a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:09:32 +0100 Subject: [PATCH 17/25] trie: remove sliceBuffer --- trie/hasher.go | 11 ----------- trie/node_test.go | 13 ------------- 2 files changed, 24 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 247882a87b418..f93f2a99e32bf 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -24,17 +24,6 @@ import ( "golang.org/x/crypto/sha3" ) -type sliceBuffer []byte - -func (b *sliceBuffer) Write(data []byte) (n int, err error) { - *b = append(*b, data...) - return len(data), nil -} - -func (b *sliceBuffer) Reset() { - *b = (*b)[:0] -} - // hasher is a type used for the trie Hash operation. A hasher has some // internal preallocated temp space type hasher struct { diff --git a/trie/node_test.go b/trie/node_test.go index 8c70680c2c866..52720f1c776ee 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -92,16 +92,3 @@ func TestDecodeFullNode(t *testing.T) { t.Fatalf("decode full node err: %v", err) } } - -func BenchmarkEncodeFullNode(b *testing.B) { - var buf sliceBuffer - var fn fullNode - for i := 0; i < 16; i++ { - fn.Children[i] = hashNode(randBytes(32)) - } - - for i := 0; i < b.N; i++ { - buf.Reset() - rlp.Encode(&buf, &fn) - } -} From 781ca961f9ab12908dcc965f5d8b38b3373ef263 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:31:06 +0100 Subject: [PATCH 18/25] trie: avoid *EncoderBuffer --- trie/database.go | 2 +- trie/hasher.go | 13 +++++++------ trie/node.go | 6 +++--- trie/node_enc.go | 16 ++++++++-------- trie/stacktrie.go | 6 +++--- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/trie/database.go b/trie/database.go index aa381e1c19769..d71abeee476a5 100644 --- a/trie/database.go +++ b/trie/database.go @@ -114,7 +114,7 @@ func (n rawFullNode) fstring(ind string) string { panic("this should never end u func (n rawFullNode) EncodeRLP(w io.Writer) error { eb := rlp.NewEncoderBuffer(w) - n.encode(&eb) + n.encode(eb) return eb.Flush() } diff --git a/trie/hasher.go b/trie/hasher.go index f93f2a99e32bf..60fce438a642e 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -144,7 +144,7 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // into compact form for RLP encoding. // If the rlp data is smaller than 32 bytes, `nil` is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { - n.encode(&h.encbuf) + n.encode(h.encbuf) enc := h.encodedBytes() if len(enc) < 32 && !force { @@ -156,7 +156,7 @@ func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { // shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which // may contain nil values) func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { - n.encode(&h.encbuf) + n.encode(h.encbuf) enc := h.encodedBytes() if len(enc) < 32 && !force { @@ -165,13 +165,14 @@ func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { return h.hashData(enc) } -// encodedBytes returns the result of the last encoding operation on h.encbuf. This exists -// because node.encode can only be inlined when using a concrete receiver type. Basically, -// all node encoding must be done like this: +// encodedBytes returns the result of the last encoding operation on h.encbuf. All node +// encoding must be done like this: // -// node.encode(&h.encbuf) +// node.encode(h.encbuf) // enc := h.encodedBytes() // +// This encoding convention exists because node.encode can only be inlined/escape-analyzed +// when called on a concrete receiver type. func (h *hasher) encodedBytes() []byte { h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) // Reset the buffer here so the next encoding operation doesn't diff --git a/trie/node.go b/trie/node.go index 805f3776a3f7d..bf3f024bb8a78 100644 --- a/trie/node.go +++ b/trie/node.go @@ -28,9 +28,9 @@ import ( var indices = []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "[17]"} type node interface { - fstring(string) string - encode(w *rlp.EncoderBuffer) cache() (hashNode, bool) + encode(w rlp.EncoderBuffer) + fstring(string) string } type ( @@ -54,7 +54,7 @@ var nilValueNode = valueNode(nil) // EncodeRLP encodes a full node into the consensus RLP format. func (n *fullNode) EncodeRLP(w io.Writer) error { eb := rlp.NewEncoderBuffer(w) - n.encode(&eb) + n.encode(eb) return eb.Flush() } diff --git a/trie/node_enc.go b/trie/node_enc.go index 0acda51b6fe93..2e2cd5a4fe820 100644 --- a/trie/node_enc.go +++ b/trie/node_enc.go @@ -6,13 +6,13 @@ import ( func nodeToBytes(n node) []byte { w := rlp.NewEncoderBuffer(nil) - n.encode(&w) + n.encode(w) result := w.ToBytes() w.Flush() return result } -func (n *fullNode) encode(w *rlp.EncoderBuffer) { +func (n *fullNode) encode(w rlp.EncoderBuffer) { offset := w.List() for _, c := range n.Children { if c != nil { @@ -24,7 +24,7 @@ func (n *fullNode) encode(w *rlp.EncoderBuffer) { w.ListEnd(offset) } -func (n *shortNode) encode(w *rlp.EncoderBuffer) { +func (n *shortNode) encode(w rlp.EncoderBuffer) { offset := w.List() w.WriteBytes(n.Key) if n.Val != nil { @@ -35,15 +35,15 @@ func (n *shortNode) encode(w *rlp.EncoderBuffer) { w.ListEnd(offset) } -func (n hashNode) encode(w *rlp.EncoderBuffer) { +func (n hashNode) encode(w rlp.EncoderBuffer) { w.WriteBytes(n) } -func (n valueNode) encode(w *rlp.EncoderBuffer) { +func (n valueNode) encode(w rlp.EncoderBuffer) { w.WriteBytes(n) } -func (n rawFullNode) encode(w *rlp.EncoderBuffer) { +func (n rawFullNode) encode(w rlp.EncoderBuffer) { offset := w.List() for _, c := range n { if c != nil { @@ -55,7 +55,7 @@ func (n rawFullNode) encode(w *rlp.EncoderBuffer) { w.ListEnd(offset) } -func (n *rawShortNode) encode(w *rlp.EncoderBuffer) { +func (n *rawShortNode) encode(w rlp.EncoderBuffer) { offset := w.List() w.WriteBytes(n.Key) if n.Val != nil { @@ -66,6 +66,6 @@ func (n *rawShortNode) encode(w *rlp.EncoderBuffer) { w.ListEnd(offset) } -func (n rawNode) encode(w *rlp.EncoderBuffer) { +func (n rawNode) encode(w rlp.EncoderBuffer) { w.Write(n) } diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 9b7aa2a992c68..284ae9647f174 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -402,7 +402,7 @@ func (st *StackTrie) hashRec(hasher *hasher) { returnToPool(child) } - nodes.encode(&hasher.encbuf) + nodes.encode(hasher.encbuf) encodedNode = hasher.encodedBytes() case extNode: @@ -416,7 +416,7 @@ func (st *StackTrie) hashRec(hasher *hasher) { n.Val = hashNode(st.children[0].val) } - n.encode(&hasher.encbuf) + n.encode(hasher.encbuf) encodedNode = hasher.encodedBytes() // Release child back to pool. @@ -428,7 +428,7 @@ func (st *StackTrie) hashRec(hasher *hasher) { sz := hexToCompactInPlace(st.key) n := rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} - n.encode(&hasher.encbuf) + n.encode(hasher.encbuf) encodedNode = hasher.encodedBytes() default: From 9ff55d1f9f8b2bcf9ceec34bc09195bdf3af8578 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:37:26 +0100 Subject: [PATCH 19/25] trie: add copyright header --- trie/node_enc.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/trie/node_enc.go b/trie/node_enc.go index 2e2cd5a4fe820..cade35b707c25 100644 --- a/trie/node_enc.go +++ b/trie/node_enc.go @@ -1,3 +1,19 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package trie import ( From d083f5dcd0c0dde02bf1b3185047ade8170b4cd7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:40:12 +0100 Subject: [PATCH 20/25] trie: comment update --- trie/hasher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 60fce438a642e..1709a0a6e7afc 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -171,8 +171,8 @@ func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { // node.encode(h.encbuf) // enc := h.encodedBytes() // -// This encoding convention exists because node.encode can only be inlined/escape-analyzed -// when called on a concrete receiver type. +// This convention exists because node.encode can only be inlined/escape-analyzed when +// called on a concrete receiver type. func (h *hasher) encodedBytes() []byte { h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) // Reset the buffer here so the next encoding operation doesn't From 7ccf22e683797a1b19ba56e8fe93dd3e514e315a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:40:28 +0100 Subject: [PATCH 21/25] rlp: comment update --- rlp/encbuffer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go index 0d8b5a76df20d..289e7448c6ca1 100644 --- a/rlp/encbuffer.go +++ b/rlp/encbuffer.go @@ -331,7 +331,7 @@ func (w *EncoderBuffer) ToBytes() []byte { return w.buf.makeBytes() } -// AppendToBytes appends the encoded value to dst. +// AppendToBytes appends the encoded bytes to dst. func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { size := w.buf.size() out := append(dst, make([]byte, size)...) From ea6d398d0aba183f4d14eb1960eb023cb9e642e6 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:41:29 +0100 Subject: [PATCH 22/25] trie: comment update --- trie/hasher.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 1709a0a6e7afc..552a774288be5 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -165,8 +165,10 @@ func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { return h.hashData(enc) } -// encodedBytes returns the result of the last encoding operation on h.encbuf. All node -// encoding must be done like this: +// encodedBytes returns the result of the last encoding operation on h.encbuf. +// This also resets the encoder buffer. +// +// All node encoding must be done like this: // // node.encode(h.encbuf) // enc := h.encodedBytes() From d8192d4fb98e749b9ea000e24021320d5a126e4b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:42:17 +0100 Subject: [PATCH 23/25] trie: comment update --- trie/hasher.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 552a774288be5..7f0748c13df30 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -177,8 +177,6 @@ func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { // called on a concrete receiver type. func (h *hasher) encodedBytes() []byte { h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) - // Reset the buffer here so the next encoding operation doesn't - // need to care about it. h.encbuf.Reset(nil) return h.tmp } From 930fc95984ccfa60517816d9a1ff8ac1c8545a39 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 21:43:35 +0100 Subject: [PATCH 24/25] trie: restore accidentally removed comment in proof.go --- trie/proof.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trie/proof.go b/trie/proof.go index a617ab4a4e17a..88ca80b0e7066 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -76,6 +76,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e var hn node n, hn = hasher.proofHash(n) if hash, ok := hn.(hashNode); ok || i == 0 { + // If the node's database encoding is a hash (or is the + // root node), it becomes a proof element. enc := nodeToBytes(n) if !ok { hash = hasher.hashData(enc) From 32b7f3776c62079e6395472c37256048a281ee5a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 8 Mar 2022 23:41:18 +0100 Subject: [PATCH 25/25] trie: comment update --- trie/stacktrie.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 284ae9647f174..b38bb01b0fb32 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -223,6 +223,7 @@ func (st *StackTrie) insert(key, value []byte) { switch st.nodeType { case branchNode: /* Branch */ idx := int(key[0]) + // Unresolve elder siblings for i := idx - 1; i >= 0; i-- { if st.children[i] != nil { @@ -232,12 +233,14 @@ func (st *StackTrie) insert(key, value []byte) { break } } + // Add new child if st.children[idx] == nil { st.children[idx] = newLeaf(key[1:], value, st.db) } else { st.children[idx].insert(key[1:], value) } + case extNode: /* Ext */ // Compare both key chunks and see where they differ diffidx := st.getDiffIndex(key) @@ -325,10 +328,9 @@ func (st *StackTrie) insert(key, value []byte) { p = st.children[0] } - // Create the two child leaves: the one containing the - // original value and the one containing the new value - // The child leave will be hashed directly in order to - // free up some memory. + // Create the two child leaves: one containing the original + // value and another containing the new value. The child leaf + // is hashed directly in order to free up some memory. origIdx := st.key[diffidx] p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db) p.children[origIdx].hash() @@ -340,12 +342,15 @@ func (st *StackTrie) insert(key, value []byte) { // over to the children. st.key = st.key[:diffidx] st.val = nil + case emptyNode: /* Empty */ st.nodeType = leafNode st.key = key st.val = value + case hashedNode: panic("trying to insert into hash") + default: panic("invalid type") }