Skip to content

Commit

Permalink
Upgrade leveldb (#539)
Browse files Browse the repository at this point in the history 
* kv: remove GetTo method of Getter interface

* deps: upgrade goleveldb

* muxdb: tweak leveldb options

* deps: upgrade leveldb
  • Loading branch information
@qianbin
qianbin committed Jul 27, 2022
1 parent 9177228 commit b5e16aa
Show file tree
Hide file tree
Showing 10 changed files with 108 additions and 87 deletions.
3 changes: 1 addition & 2 deletions cmd/thor/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"github.com/inconshreveable/log15"
"github.com/pkg/errors"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/vechain/thor/block"
"github.com/vechain/thor/cache"
"github.com/vechain/thor/chain"
Expand Down Expand Up @@ -225,7 +224,7 @@ func (n *Node) txStashLoop(ctx context.Context) {
log.Debug("enter tx stash loop")
defer log.Debug("leave tx stash loop")

db, err := leveldb.OpenFile(n.txStashPath, &opt.Options{NoFullFSync: true})
db, err := leveldb.OpenFile(n.txStashPath, nil)
if err != nil {
log.Error("create tx stash", "err", err)
return
Expand Down
12 changes: 6 additions & 6 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/beevik/ntp v0.2.0
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 // indirect
github.com/cespare/cp v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.0
github.com/davecgh/go-spew v1.1.1
github.com/deckarep/golang-set v1.7.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1
github.com/elastic/gosigar v0.10.5
Expand All @@ -34,17 +34,17 @@ require (
github.com/pmezard/go-difflib v1.0.0
github.com/qianbin/directcache v0.9.6
github.com/rjeczalik/notify v0.9.1 // indirect
github.com/stretchr/testify v1.4.0
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
github.com/stretchr/testify v1.7.2
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a
github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
gopkg.in/cheggaaa/pb.v1 v1.0.28
gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 // indirect
gopkg.in/urfave/cli.v1 v1.20.0
gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v2 v2.4.0
)

replace github.com/syndtr/goleveldb => github.com/vechain/goleveldb v1.0.1-0.20220107084823-eb292ce39601
replace github.com/syndtr/goleveldb => github.com/vechain/goleveldb v1.0.1-0.20220727025255-5a7c90a37a27

replace github.com/ethereum/go-ethereum => github.com/vechain/go-ethereum v1.8.15-0.20220606031836-4784dac628d7
81 changes: 63 additions & 18 deletions go.sum
View file

Large diffs are not rendered by default.

8 changes: 0 additions & 8 deletions kv/bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ type Bucket string
func (b Bucket) NewGetter(src Getter) Getter {
return &struct {
GetFunc
GetToFunc
HasFunc
IsNotFoundFunc
}{
Expand All @@ -29,13 +28,6 @@ func (b Bucket) NewGetter(src Getter) Getter {

return src.Get(buf.k)
},
func(key, dst []byte) ([]byte, error) {
buf := bufPool.Get().(*buf)
defer bufPool.Put(buf)
buf.k = append(append(buf.k[:0], b...), key...)

return src.GetTo(buf.k, dst)
},
func(key []byte) (bool, error) {
buf := bufPool.Get().(*buf)
defer bufPool.Put(buf)
Expand Down
6 changes: 0 additions & 6 deletions kv/bucket_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,6 @@ func (m mem) Get(k []byte) ([]byte, error) {
}
return nil, errors.New("not found")
}
func (m mem) GetTo(k, dst []byte) ([]byte, error) {
if v, ok := m[string(k)]; ok {
return append(dst, v...), nil
}
return nil, errors.New("not found")
}

func (m mem) Has(k []byte) (bool, error) {
_, ok := m[string(k)]
Expand Down
38 changes: 18 additions & 20 deletions kv/f.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package kv

type (
GetFunc func(key []byte) ([]byte, error)
GetToFunc func(key, dst []byte) ([]byte, error)
HasFunc func(key []byte) (bool, error)
IsNotFoundFunc func(err error) bool
PutFunc func(key, val []byte) error
Expand All @@ -29,22 +28,21 @@ type (
ErrorFunc func() error
)

func (f GetFunc) Get(key []byte) ([]byte, error) { return f(key) }
func (f GetToFunc) GetTo(key, dst []byte) ([]byte, error) { return f(key, dst) }
func (f HasFunc) Has(key []byte) (bool, error) { return f(key) }
func (f IsNotFoundFunc) IsNotFound(err error) bool { return f(err) }
func (f PutFunc) Put(key, val []byte) error { return f(key, val) }
func (f DeleteFunc) Delete(key []byte) error { return f(key) }
func (f SnapshotFunc) Snapshot() Snapshot { return f() }
func (f BulkFunc) Bulk() Bulk { return f() }
func (f IterateFunc) Iterate(r Range) Iterator { return f(r) }
func (f EnableAutoFlushFunc) EnableAutoFlush() { f() }
func (f WriteFunc) Write() error { return f() }
func (f FirstFunc) First() bool { return f() }
func (f LastFunc) Last() bool { return f() }
func (f NextFunc) Next() bool { return f() }
func (f PrevFunc) Prev() bool { return f() }
func (f KeyFunc) Key() []byte { return f() }
func (f ValueFunc) Value() []byte { return f() }
func (f ReleaseFunc) Release() { f() }
func (f ErrorFunc) Error() error { return f() }
func (f GetFunc) Get(key []byte) ([]byte, error) { return f(key) }
func (f HasFunc) Has(key []byte) (bool, error) { return f(key) }
func (f IsNotFoundFunc) IsNotFound(err error) bool { return f(err) }
func (f PutFunc) Put(key, val []byte) error { return f(key, val) }
func (f DeleteFunc) Delete(key []byte) error { return f(key) }
func (f SnapshotFunc) Snapshot() Snapshot { return f() }
func (f BulkFunc) Bulk() Bulk { return f() }
func (f IterateFunc) Iterate(r Range) Iterator { return f(r) }
func (f EnableAutoFlushFunc) EnableAutoFlush() { f() }
func (f WriteFunc) Write() error { return f() }
func (f FirstFunc) First() bool { return f() }
func (f LastFunc) Last() bool { return f() }
func (f NextFunc) Next() bool { return f() }
func (f PrevFunc) Prev() bool { return f() }
func (f KeyFunc) Key() []byte { return f() }
func (f ValueFunc) Value() []byte { return f() }
func (f ReleaseFunc) Release() { f() }
func (f ErrorFunc) Error() error { return f() }
1 change: 0 additions & 1 deletion kv/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ package kv
// Getter defines methods to read kv.
type Getter interface {
Get(key []byte) ([]byte, error)
GetTo(key, dst []byte) ([]byte, error)
Has(key []byte) (bool, error)
IsNotFound(err error) bool
}
Expand Down
20 changes: 0 additions & 20 deletions muxdb/internal/engine/leveldb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,6 @@ func (ldb *levelEngine) Get(key []byte) ([]byte, error) {
return val, nil
}

func (ldb *levelEngine) GetTo(key, dst []byte) ([]byte, error) {
val, err := ldb.db.GetTo(key, dst, &readOpt)
// val will be []byte{} if error occurs, which is not expected
if err != nil {
return nil, err
}
return val, nil
}

func (ldb *levelEngine) Has(key []byte) (bool, error) {
return ldb.db.Has(key, &readOpt)
}
Expand All @@ -79,7 +70,6 @@ func (ldb *levelEngine) Snapshot() kv.Snapshot {
s, err := ldb.db.GetSnapshot()
return &struct {
kv.GetFunc
kv.GetToFunc
kv.HasFunc
kv.IsNotFoundFunc
kv.ReleaseFunc
Expand All @@ -94,16 +84,6 @@ func (ldb *levelEngine) Snapshot() kv.Snapshot {
}
return val, nil
},
func(key, dst []byte) ([]byte, error) {
if err != nil {
return nil, err
}
val, err := s.GetTo(key, dst, &readOpt)
if err != nil {
return nil, err
}
return val, nil
},
func(key []byte) (bool, error) {
if err != nil {
return false, err
Expand Down
24 changes: 19 additions & 5 deletions muxdb/internal/trie/trie.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func (t *Trie) newDatabase() trie.Database {
trie.DatabaseReader
trie.DatabaseWriter
}{
kv.GetToFunc(func(_ []byte, dst []byte) (blob []byte, err error) {
databaseGetToFunc(func(_ []byte, dst []byte) (blob []byte, err error) {
// get from cache
if blob = t.back.Cache.GetNodeBlob(t.name, thisSeq, thisPath, t.noFillCache, dst); len(blob) > 0 {
return
Expand Down Expand Up @@ -155,14 +155,23 @@ func (t *Trie) newDatabase() trie.Database {

// get from hist space first
keyBuf = t.makeHistNodeKey(keyBuf[:0], thisSeq, thisPath)
if blob, err = snapshot.GetTo(keyBuf, dst); err == nil || !snapshot.IsNotFound(err) {
// found or error
return
if val, err := snapshot.Get(keyBuf); err == nil {
// found
return append(dst, val...), nil
} else if !snapshot.IsNotFound(err) {
// error
if !snapshot.IsNotFound(err) {
return nil, err
}
}

// then from deduped space
keyBuf = t.makeDedupedNodeKey(keyBuf[:0], thisSeq, thisPath)
return snapshot.GetTo(keyBuf, dst)
if val, err := snapshot.Get(keyBuf); err == nil {
return append(dst, val...), nil
} else {
return nil, err
}
}),
databaseKeyEncodeFunc(func(hash []byte, seq uint64, path []byte) []byte {
thisHash = hash
Expand Down Expand Up @@ -450,12 +459,17 @@ func CleanHistory(ctx context.Context, back *Backend, startCommitNum, limitCommi
// individual functions of trie database interface.
type (
databaseKeyEncodeFunc func(hash []byte, seq uint64, path []byte) []byte
databaseGetToFunc func(key, dst []byte) ([]byte, error)
)

func (f databaseKeyEncodeFunc) Encode(hash []byte, seq uint64, path []byte) []byte {
return f(hash, seq, path)
}

func (f databaseGetToFunc) GetTo(key, dst []byte) ([]byte, error) {
return f(key, dst)
}

// leafAvailable is a special error type to short circuit trie get method.
type leafAvailable struct {
*trie.Leaf
Expand Down
2 changes: 1 addition & 1 deletion muxdb/muxdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func Open(path string, options *Options) (*MuxDB, error) {
WriteBuffer: options.WriteBufferMB * opt.MiB,
Filter: filter.NewBloomFilter(10),
BlockSize: 1024 * 32, // balance performance of point reads and compression ratio.
NoFullFSync: true,
CompactionTableSize: 4 * opt.MiB,
}

if options.TrieWillCleanHistory {
Expand Down

0 comments on commit b5e16aa

Please sign in to comment.