From 9f4ff8f7edbd8a83741b1530c45896e87ed9e39b Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 8 Nov 2021 15:31:16 -0800 Subject: [PATCH 1/6] Revert "metadb: split backends into packages, add metadb (#151)" This reverts commit 71bc5d82caa5e88f683df6853a46a5c138475694. --- .github/workflows/lint.yml | 1 - metadb/backend_test.go => backend_test.go | 203 +++++++----------- badgerdb/db.go => badger_db.go | 49 +++-- boltdb/db.go => boltdb.go | 53 +++-- boltdb/db_test.go | 34 --- boltdb/batch.go => boltdb_batch.go | 34 +-- boltdb/iterator.go => boltdb_iterator.go | 7 +- boltdb_test.go | 35 +++ cleveldb/db.go => cleveldb.go | 43 ++-- cleveldb/db_test.go | 78 ------- cleveldb/batch.go => cleveldb_batch.go | 23 +- cleveldb/iterator.go => cleveldb_iterator.go | 11 +- cleveldb_test.go | 103 +++++++++ internal/dbtest/dbtest.go => common_test.go | 126 +++++------ metadb/db.go => db.go | 14 +- metadb/db_test.go => db_test.go | 45 ++-- goleveldb/db.go => goleveldb.go | 42 ++-- goleveldb/db_test.go | 44 ---- goleveldb/batch.go => goleveldb_batch.go | 17 +- .../iterator.go => goleveldb_iterator.go | 11 +- goleveldb_test.go | 43 ++++ makefile | 15 +- memdb/db.go => memdb.go | 35 +-- memdb/db_test.go | 28 --- memdb/batch.go => memdb_batch.go | 22 +- memdb/iterator.go => memdb_iterator.go | 5 +- memdb_test.go | 26 +++ metadb/db_badgerdb.go | 14 -- metadb/db_boltdb.go | 14 -- metadb/db_cleveldb.go | 14 -- metadb/db_goleveldb.go | 14 -- metadb/db_memdb.go | 14 -- metadb/db_rocksdb.go | 14 -- prefixdb.go | 20 +- prefixdb_batch.go | 6 +- prefixdb_test.go | 133 ++++++------ remotedb/batch.go | 15 +- remotedb/doc.go | 4 +- remotedb/grpcdb/server.go | 3 +- remotedb/iterator.go | 12 +- remotedb/{db.go => remotedb.go} | 18 +- remotedb/{db_test.go => remotedb_test.go} | 2 +- rocksdb/db.go => rocksdb.go | 44 ++-- rocksdb/db_test.go | 3 - rocksdb/batch.go => rocksdb_batch.go | 25 ++- rocksdb/iterator.go => rocksdb_iterator.go | 7 +- rocksdb_test.go | 35 +++ test_helpers.go | 36 ++++ types.go | 12 +- metadb/util_test.go => util_test.go | 70 +++--- 50 files changed, 789 insertions(+), 887 deletions(-) rename metadb/backend_test.go => backend_test.go (63%) rename badgerdb/db.go => badger_db.go (85%) rename boltdb/db.go => boltdb.go (78%) delete mode 100644 boltdb/db_test.go rename boltdb/batch.go => boltdb_batch.go (77%) rename boltdb/iterator.go => boltdb_iterator.go (96%) create mode 100644 boltdb_test.go rename cleveldb/db.go => cleveldb.go (82%) delete mode 100644 cleveldb/db_test.go rename cleveldb/batch.go => cleveldb_batch.go (81%) rename cleveldb/iterator.go => cleveldb_iterator.go (94%) create mode 100644 cleveldb_test.go rename internal/dbtest/dbtest.go => common_test.go (56%) rename metadb/db.go => db.go (80%) rename metadb/db_test.go => db_test.go (74%) rename goleveldb/db.go => goleveldb.go (80%) delete mode 100644 goleveldb/db_test.go rename goleveldb/batch.go => goleveldb_batch.go (82%) rename goleveldb/iterator.go => goleveldb_iterator.go (93%) create mode 100644 goleveldb_test.go rename memdb/db.go => memdb.go (87%) delete mode 100644 memdb/db_test.go rename memdb/batch.go => memdb_batch.go (84%) rename memdb/iterator.go => memdb_iterator.go (97%) create mode 100644 memdb_test.go delete mode 100644 metadb/db_badgerdb.go delete mode 100644 metadb/db_boltdb.go delete mode 100644 metadb/db_cleveldb.go delete mode 100644 metadb/db_goleveldb.go delete mode 100644 metadb/db_memdb.go delete mode 100644 metadb/db_rocksdb.go rename remotedb/{db.go => remotedb.go} (83%) rename remotedb/{db_test.go => remotedb_test.go} (98%) rename rocksdb/db.go => rocksdb.go (81%) delete mode 100644 rocksdb/db_test.go rename rocksdb/batch.go => rocksdb_batch.go (79%) rename rocksdb/iterator.go => rocksdb_iterator.go (96%) create mode 100644 rocksdb_test.go create mode 100644 test_helpers.go rename metadb/util_test.go => util_test.go (56%) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c291823ed..fb3f7bb71 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,6 @@ jobs: # We need to run the linter on the same image we use for building, since it # needs the C libraries installed for the dependencies to typecheck. runs-on: ubuntu-latest - container: tendermintdev/docker-tm-db-testing steps: - uses: actions/checkout@v2 - uses: golangci/golangci-lint-action@v2.5.2 diff --git a/metadb/backend_test.go b/backend_test.go similarity index 63% rename from metadb/backend_test.go rename to backend_test.go index dff5f71e9..63630beca 100644 --- a/metadb/backend_test.go +++ b/backend_test.go @@ -1,43 +1,45 @@ -package metadb +package db import ( "fmt" "io/ioutil" "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/cleveldb" - "github.com/tendermint/tm-db/goleveldb" - "github.com/tendermint/tm-db/internal/dbtest" - "github.com/tendermint/tm-db/memdb" - "github.com/tendermint/tm-db/rocksdb" ) // Register a test backend for PrefixDB as well, with some unrelated junk data func init() { // nolint: errcheck - registerDBCreator("prefixdb", func(name, dir string) (tmdb.DB, error) { - mdb := memdb.NewDB() + registerDBCreator("prefixdb", func(name, dir string) (DB, error) { + mdb := NewMemDB() mdb.Set([]byte("a"), []byte{1}) mdb.Set([]byte("b"), []byte{2}) mdb.Set([]byte("t"), []byte{20}) mdb.Set([]byte("test"), []byte{0}) mdb.Set([]byte("u"), []byte{21}) mdb.Set([]byte("z"), []byte{26}) - return tmdb.NewPrefixDB(mdb, []byte("test/")), nil + return NewPrefixDB(mdb, []byte("test/")), nil }, false) } +func cleanupDBDir(dir, name string) { + err := os.RemoveAll(filepath.Join(dir, name) + ".db") + if err != nil { + panic(err) + } +} + func testBackendGetSetDelete(t *testing.T, backend BackendType) { // Default dirname, err := ioutil.TempDir("", fmt.Sprintf("test_backend_%s_", backend)) require.Nil(t, err) db, err := NewDB("testdb", backend, dirname) require.NoError(t, err) - defer dbtest.CleanupDBDir(dirname, "testdb") + defer cleanupDBDir(dirname, "testdb") // A nonexistent key should return nil. value, err := db.Get([]byte("a")) @@ -91,38 +93,38 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { // Setting, getting, and deleting an empty key should error. _, err = db.Get([]byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) _, err = db.Get(nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) _, err = db.Has([]byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) _, err = db.Has(nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.Set([]byte{}, []byte{0x01}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.Set(nil, []byte{0x01}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.SetSync([]byte{}, []byte{0x01}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.SetSync(nil, []byte{0x01}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.Delete([]byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.Delete(nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.DeleteSync([]byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = db.DeleteSync(nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) // Setting a nil value should error, but an empty value is fine. err = db.Set([]byte("x"), nil) - require.Equal(t, tmdb.ErrValueNil, err) + require.Equal(t, errValueNil, err) err = db.SetSync([]byte("x"), nil) - require.Equal(t, tmdb.ErrValueNil, err) + require.Equal(t, errValueNil, err) err = db.Set([]byte("x"), []byte{}) require.NoError(t, err) @@ -142,74 +144,15 @@ func TestBackendsGetSetDelete(t *testing.T) { } func TestGoLevelDBBackend(t *testing.T) { - if _, ok := backends[GoLevelDBBackend]; !ok { - t.Skip("skipping since -tags=goleveldb was not used") - } - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) + name := fmt.Sprintf("test_%x", randStr(12)) db, err := NewDB(name, GoLevelDBBackend, "") require.NoError(t, err) - defer dbtest.CleanupDBDir("", name) - - _, ok := db.(*goleveldb.GoLevelDB) - assert.True(t, ok) -} - -func TestCLevelDBBackend(t *testing.T) { - if _, ok := backends[CLevelDBBackend]; !ok { - t.Skip("skipping since -tags=cleveldb was not used") - } - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - // Can't use "" (current directory) or "./" here because levigo.Open returns: - // "Error initializing DB: IO error: test_XXX.db: Invalid argument" - dir := os.TempDir() - db, err := NewDB(name, CLevelDBBackend, dir) - require.NoError(t, err) - defer dbtest.CleanupDBDir(dir, name) - - _, ok := db.(*cleveldb.CLevelDB) - assert.True(t, ok) -} - -func TestCLevelDBStats(t *testing.T) { - if _, ok := backends[CLevelDBBackend]; !ok { - t.Skip("skipping since -tags=cleveldb was not used") - } - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - dir := os.TempDir() - db, err := NewDB(name, CLevelDBBackend, dir) - require.NoError(t, err) - defer dbtest.CleanupDBDir(dir, name) - - assert.NotEmpty(t, db.Stats()) -} + defer cleanupDBDir("", name) -func TestRocksDBBackend(t *testing.T) { - if _, ok := backends[RocksDBBackend]; !ok { - t.Skip("skipping since -tags=rocksdb was not used") - } - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - dir := os.TempDir() - db, err := NewDB(name, RocksDBBackend, dir) - require.NoError(t, err) - defer dbtest.CleanupDBDir(dir, name) - - _, ok := db.(*rocksdb.RocksDB) + _, ok := db.(*GoLevelDB) assert.True(t, ok) } -func TestRocksDBStats(t *testing.T) { - if _, ok := backends[RocksDBBackend]; !ok { - t.Skip("skipping since -tags=rocksdb was not used") - } - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - dir := os.TempDir() - db, err := NewDB(name, RocksDBBackend, dir) - require.NoError(t, err) - defer dbtest.CleanupDBDir(dir, name) - - assert.NotEmpty(t, db.Stats()) -} - func TestDBIterator(t *testing.T) { for dbType := range backends { t.Run(string(dbType), func(t *testing.T) { @@ -219,28 +162,28 @@ func TestDBIterator(t *testing.T) { } func testDBIterator(t *testing.T, backend BackendType) { - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) + name := fmt.Sprintf("test_%x", randStr(12)) dir := os.TempDir() db, err := NewDB(name, backend, dir) require.NoError(t, err) - defer dbtest.CleanupDBDir(dir, name) + defer cleanupDBDir(dir, name) for i := 0; i < 10; i++ { if i != 6 { // but skip 6. - err := db.Set(dbtest.Int642Bytes(int64(i)), []byte{}) + err := db.Set(int642Bytes(int64(i)), []byte{}) require.NoError(t, err) } } // Blank iterator keys should error _, err = db.Iterator([]byte{}, nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) _, err = db.Iterator(nil, []byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) _, err = db.ReverseIterator([]byte{}, nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) _, err = db.ReverseIterator(nil, []byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) itr, err := db.Iterator(nil, nil) require.NoError(t, err) @@ -250,110 +193,110 @@ func testDBIterator(t *testing.T, backend BackendType) { require.NoError(t, err) verifyIterator(t, ritr, []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator") - itr, err = db.Iterator(nil, dbtest.Int642Bytes(0)) + itr, err = db.Iterator(nil, int642Bytes(0)) require.NoError(t, err) verifyIterator(t, itr, []int64(nil), "forward iterator to 0") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(10), nil) + ritr, err = db.ReverseIterator(int642Bytes(10), nil) require.NoError(t, err) verifyIterator(t, ritr, []int64(nil), "reverse iterator from 10 (ex)") - itr, err = db.Iterator(dbtest.Int642Bytes(0), nil) + itr, err = db.Iterator(int642Bytes(0), nil) require.NoError(t, err) verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 0") - itr, err = db.Iterator(dbtest.Int642Bytes(1), nil) + itr, err = db.Iterator(int642Bytes(1), nil) require.NoError(t, err) verifyIterator(t, itr, []int64{1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 1") - ritr, err = db.ReverseIterator(nil, dbtest.Int642Bytes(10)) + ritr, err = db.ReverseIterator(nil, int642Bytes(10)) require.NoError(t, err) verifyIterator(t, ritr, []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10 (ex)") - ritr, err = db.ReverseIterator(nil, dbtest.Int642Bytes(9)) + ritr, err = db.ReverseIterator(nil, int642Bytes(9)) require.NoError(t, err) verifyIterator(t, ritr, []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9 (ex)") - ritr, err = db.ReverseIterator(nil, dbtest.Int642Bytes(8)) + ritr, err = db.ReverseIterator(nil, int642Bytes(8)) require.NoError(t, err) verifyIterator(t, ritr, []int64{7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8 (ex)") - itr, err = db.Iterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(6)) + itr, err = db.Iterator(int642Bytes(5), int642Bytes(6)) require.NoError(t, err) verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 6") - itr, err = db.Iterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(7)) + itr, err = db.Iterator(int642Bytes(5), int642Bytes(7)) require.NoError(t, err) verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 7") - itr, err = db.Iterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(8)) + itr, err = db.Iterator(int642Bytes(5), int642Bytes(8)) require.NoError(t, err) verifyIterator(t, itr, []int64{5, 7}, "forward iterator from 5 to 8") - itr, err = db.Iterator(dbtest.Int642Bytes(6), dbtest.Int642Bytes(7)) + itr, err = db.Iterator(int642Bytes(6), int642Bytes(7)) require.NoError(t, err) verifyIterator(t, itr, []int64(nil), "forward iterator from 6 to 7") - itr, err = db.Iterator(dbtest.Int642Bytes(6), dbtest.Int642Bytes(8)) + itr, err = db.Iterator(int642Bytes(6), int642Bytes(8)) require.NoError(t, err) verifyIterator(t, itr, []int64{7}, "forward iterator from 6 to 8") - itr, err = db.Iterator(dbtest.Int642Bytes(7), dbtest.Int642Bytes(8)) + itr, err = db.Iterator(int642Bytes(7), int642Bytes(8)) require.NoError(t, err) verifyIterator(t, itr, []int64{7}, "forward iterator from 7 to 8") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(5)) + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(5)) require.NoError(t, err) verifyIterator(t, ritr, []int64{4}, "reverse iterator from 5 (ex) to 4") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(6)) + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(6)) require.NoError(t, err) verifyIterator(t, ritr, []int64{5, 4}, "reverse iterator from 6 (ex) to 4") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(7)) + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(7)) require.NoError(t, err) verifyIterator(t, ritr, []int64{5, 4}, "reverse iterator from 7 (ex) to 4") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(6)) + ritr, err = db.ReverseIterator(int642Bytes(5), int642Bytes(6)) require.NoError(t, err) verifyIterator(t, ritr, []int64{5}, "reverse iterator from 6 (ex) to 5") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(7)) + ritr, err = db.ReverseIterator(int642Bytes(5), int642Bytes(7)) require.NoError(t, err) verifyIterator(t, ritr, []int64{5}, "reverse iterator from 7 (ex) to 5") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(6), dbtest.Int642Bytes(7)) + ritr, err = db.ReverseIterator(int642Bytes(6), int642Bytes(7)) require.NoError(t, err) verifyIterator(t, ritr, []int64(nil), "reverse iterator from 7 (ex) to 6") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(10), nil) + ritr, err = db.ReverseIterator(int642Bytes(10), nil) require.NoError(t, err) verifyIterator(t, ritr, []int64(nil), "reverse iterator to 10") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(6), nil) + ritr, err = db.ReverseIterator(int642Bytes(6), nil) require.NoError(t, err) verifyIterator(t, ritr, []int64{9, 8, 7}, "reverse iterator to 6") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(5), nil) + ritr, err = db.ReverseIterator(int642Bytes(5), nil) require.NoError(t, err) verifyIterator(t, ritr, []int64{9, 8, 7, 5}, "reverse iterator to 5") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(8), dbtest.Int642Bytes(9)) + ritr, err = db.ReverseIterator(int642Bytes(8), int642Bytes(9)) require.NoError(t, err) verifyIterator(t, ritr, []int64{8}, "reverse iterator from 9 (ex) to 8") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(2), dbtest.Int642Bytes(4)) + ritr, err = db.ReverseIterator(int642Bytes(2), int642Bytes(4)) require.NoError(t, err) verifyIterator(t, ritr, []int64{3, 2}, "reverse iterator from 4 (ex) to 2") - ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(2)) + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(2)) require.NoError(t, err) verifyIterator(t, ritr, []int64(nil), "reverse iterator from 2 (ex) to 4") @@ -363,7 +306,7 @@ func testDBIterator(t *testing.T, backend BackendType) { require.NoError(t, err) db2, err := NewDB(name, backend, dir2) require.NoError(t, err) - defer dbtest.CleanupDBDir(dir2, name) + defer cleanupDBDir(dir2, name) itr, err = db2.Iterator(nil, nil) require.NoError(t, err) @@ -375,11 +318,11 @@ func testDBIterator(t *testing.T, backend BackendType) { } -func verifyIterator(t *testing.T, itr tmdb.Iterator, expected []int64, msg string) { +func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) { var list []int64 for itr.Valid() { key := itr.Key() - list = append(list, dbtest.Bytes2Int64(key)) + list = append(list, bytes2Int64(key)) itr.Next() } assert.Equal(t, expected, list, msg) @@ -394,11 +337,11 @@ func TestDBBatch(t *testing.T) { } func testDBBatch(t *testing.T, backend BackendType) { - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) + name := fmt.Sprintf("test_%x", randStr(12)) dir := os.TempDir() db, err := NewDB(name, backend, dir) require.NoError(t, err) - defer dbtest.CleanupDBDir(dir, name) + defer cleanupDBDir(dir, name) // create a new batch, and some items - they should not be visible until we write batch := db.NewBatch() @@ -433,16 +376,16 @@ func testDBBatch(t *testing.T, backend BackendType) { // empty and nil keys, as well as nil values, should be disallowed batch = db.NewBatch() err = batch.Set([]byte{}, []byte{0x01}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = batch.Set(nil, []byte{0x01}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = batch.Set([]byte("a"), nil) - require.Equal(t, tmdb.ErrValueNil, err) + require.Equal(t, errValueNil, err) err = batch.Delete([]byte{}) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = batch.Delete(nil) - require.Equal(t, tmdb.ErrKeyEmpty, err) + require.Equal(t, errKeyEmpty, err) err = batch.Close() require.NoError(t, err) @@ -465,7 +408,7 @@ func testDBBatch(t *testing.T, backend BackendType) { require.Error(t, batch.WriteSync()) } -func assertKeyValues(t *testing.T, db tmdb.DB, expect map[string][]byte) { +func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) { iter, err := db.Iterator(nil, nil) require.NoError(t, err) defer iter.Close() diff --git a/badgerdb/db.go b/badger_db.go similarity index 85% rename from badgerdb/db.go rename to badger_db.go index 33e5bebee..22b852c39 100644 --- a/badgerdb/db.go +++ b/badger_db.go @@ -1,4 +1,6 @@ -package badgerdb +// +build badgerdb + +package db import ( "bytes" @@ -7,12 +9,17 @@ import ( "path/filepath" "github.com/dgraph-io/badger/v2" - tmdb "github.com/tendermint/tm-db" ) -// NewDB creates a Badger key-value store backed to the +func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } + +func badgerDBCreator(dbName, dir string) (DB, error) { + return NewBadgerDB(dbName, dir) +} + +// NewBadgerDB creates a Badger key-value store backed to the // directory dir supplied. If dir does not exist, it will be created. -func NewDB(dbName, dir string) (*BadgerDB, error) { +func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { // Since Badger doesn't support database names, we join both to obtain // the final directory to use for the database. path := filepath.Join(dir, dbName) @@ -23,13 +30,13 @@ func NewDB(dbName, dir string) (*BadgerDB, error) { opts := badger.DefaultOptions(path) opts.SyncWrites = false // note that we have Sync methods opts.Logger = nil // badger is too chatty by default - return NewDBWithOptions(opts) + return NewBadgerDBWithOptions(opts) } -// NewDBWithOptions creates a BadgerDB key value store +// NewBadgerDBWithOptions creates a BadgerDB key value store // gives the flexibility of initializing a database with the // respective options. -func NewDBWithOptions(opts badger.Options) (*BadgerDB, error) { +func NewBadgerDBWithOptions(opts badger.Options) (*BadgerDB, error) { db, err := badger.Open(opts) if err != nil { return nil, err @@ -41,11 +48,11 @@ type BadgerDB struct { db *badger.DB } -var _ tmdb.DB = (*BadgerDB)(nil) +var _ DB = (*BadgerDB)(nil) func (b *BadgerDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } var val []byte err := b.db.View(func(txn *badger.Txn) error { @@ -66,7 +73,7 @@ func (b *BadgerDB) Get(key []byte) ([]byte, error) { func (b *BadgerDB) Has(key []byte) (bool, error) { if len(key) == 0 { - return false, tmdb.ErrKeyEmpty + return false, errKeyEmpty } var found bool err := b.db.View(func(txn *badger.Txn) error { @@ -82,10 +89,10 @@ func (b *BadgerDB) Has(key []byte) (bool, error) { func (b *BadgerDB) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } return b.db.Update(func(txn *badger.Txn) error { return txn.Set(key, value) @@ -105,7 +112,7 @@ func (b *BadgerDB) SetSync(key, value []byte) error { func (b *BadgerDB) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } return b.db.Update(func(txn *badger.Txn) error { return txn.Delete(key) @@ -126,7 +133,7 @@ func (b *BadgerDB) Print() error { func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions) (*badgerDBIterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } txn := b.db.NewTransaction(false) iter := txn.NewIterator(opts) @@ -147,12 +154,12 @@ func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions) }, nil } -func (b *BadgerDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (b *BadgerDB) Iterator(start, end []byte) (Iterator, error) { opts := badger.DefaultIteratorOptions return b.iteratorOpts(start, end, opts) } -func (b *BadgerDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) { opts := badger.DefaultIteratorOptions opts.Reverse = true return b.iteratorOpts(end, start, opts) @@ -162,7 +169,7 @@ func (b *BadgerDB) Stats() map[string]string { return nil } -func (b *BadgerDB) NewBatch() tmdb.Batch { +func (b *BadgerDB) NewBatch() Batch { wb := &badgerDBBatch{ db: b.db, wb: b.db.NewWriteBatch(), @@ -172,7 +179,7 @@ func (b *BadgerDB) NewBatch() tmdb.Batch { return wb } -var _ tmdb.Batch = (*badgerDBBatch)(nil) +var _ Batch = (*badgerDBBatch)(nil) type badgerDBBatch struct { db *badger.DB @@ -189,17 +196,17 @@ type badgerDBBatch struct { func (b *badgerDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } return b.wb.Set(key, value) } func (b *badgerDBBatch) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } return b.wb.Delete(key) } diff --git a/boltdb/db.go b/boltdb.go similarity index 78% rename from boltdb/db.go rename to boltdb.go index 8dfbde500..398c401a4 100644 --- a/boltdb/db.go +++ b/boltdb.go @@ -1,4 +1,6 @@ -package boltdb +// +build boltdb + +package db import ( "errors" @@ -6,11 +8,18 @@ import ( "os" "path/filepath" - tmdb "github.com/tendermint/tm-db" "go.etcd.io/bbolt" ) -var bucket = []byte("tm") +var ( + bucket = []byte("tm") +) + +func init() { + registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) { + return NewBoltDB(name, dir) + }, false) +} // BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt). // @@ -23,16 +32,16 @@ type BoltDB struct { db *bbolt.DB } -var _ tmdb.DB = (*BoltDB)(nil) +var _ DB = (*BoltDB)(nil) -// NewDB returns a BoltDB with default options. -func NewDB(name, dir string) (tmdb.DB, error) { - return NewDBWithOpts(name, dir, bbolt.DefaultOptions) +// NewBoltDB returns a BoltDB with default options. +func NewBoltDB(name, dir string) (DB, error) { + return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions) } -// NewDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not -// supported because NewDBWithOpts creates a global bucket. -func NewDBWithOpts(name string, dir string, opts *bbolt.Options) (tmdb.DB, error) { +// NewBoltDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not +// supported because NewBoltDBWithOpts creates a global bucket. +func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error) { if opts.ReadOnly { return nil, errors.New("ReadOnly: true is not supported") } @@ -58,7 +67,7 @@ func NewDBWithOpts(name string, dir string, opts *bbolt.Options) (tmdb.DB, error // Get implements DB. func (bdb *BoltDB) Get(key []byte) (value []byte, err error) { if len(key) == 0 { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } err = bdb.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(bucket) @@ -85,10 +94,10 @@ func (bdb *BoltDB) Has(key []byte) (bool, error) { // Set implements DB. func (bdb *BoltDB) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } err := bdb.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(bucket) @@ -108,7 +117,7 @@ func (bdb *BoltDB) SetSync(key, value []byte) error { // Delete implements DB. func (bdb *BoltDB) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } err := bdb.db.Update(func(tx *bbolt.Tx) error { return tx.Bucket(bucket).Delete(key) @@ -135,12 +144,10 @@ func (bdb *BoltDB) Print() error { fmt.Printf("%v\n", stats) err := bdb.db.View(func(tx *bbolt.Tx) error { - if err := tx.Bucket(bucket).ForEach(func(k, v []byte) error { + tx.Bucket(bucket).ForEach(func(k, v []byte) error { fmt.Printf("[%X]:\t[%X]\n", k, v) return nil - }); err != nil { - return err - } + }) return nil }) if err != nil { @@ -168,15 +175,15 @@ func (bdb *BoltDB) Stats() map[string]string { } // NewBatch implements DB. -func (bdb *BoltDB) NewBatch() tmdb.Batch { +func (bdb *BoltDB) NewBatch() Batch { return newBoltDBBatch(bdb) } // WARNING: Any concurrent writes or reads will block until the iterator is // closed. -func (bdb *BoltDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } tx, err := bdb.db.Begin(false) if err != nil { @@ -187,9 +194,9 @@ func (bdb *BoltDB) Iterator(start, end []byte) (tmdb.Iterator, error) { // WARNING: Any concurrent writes or reads will block until the iterator is // closed. -func (bdb *BoltDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } tx, err := bdb.db.Begin(false) if err != nil { diff --git a/boltdb/db_test.go b/boltdb/db_test.go deleted file mode 100644 index 503d663aa..000000000 --- a/boltdb/db_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package boltdb - -import ( - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tendermint/tm-db/internal/dbtest" -) - -func TestBoltDBNewBoltDB(t *testing.T) { - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - dir := os.TempDir() - defer dbtest.CleanupDBDir(dir, name) - - db, err := NewDB(name, dir) - require.NoError(t, err) - db.Close() -} - -func BenchmarkBoltDBRandomReadsWrites(b *testing.B) { - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - db, err := NewDB(name, "") - if err != nil { - b.Fatal(err) - } - defer func() { - db.Close() - dbtest.CleanupDBDir("", name) - }() - - dbtest.BenchmarkRandomReadsWrites(b, db) -} diff --git a/boltdb/batch.go b/boltdb_batch.go similarity index 77% rename from boltdb/batch.go rename to boltdb_batch.go index 294291a3f..390bc309b 100644 --- a/boltdb/batch.go +++ b/boltdb_batch.go @@ -1,22 +1,8 @@ -package boltdb +// +build boltdb -import ( - tmdb "github.com/tendermint/tm-db" - "go.etcd.io/bbolt" -) +package db -type opType int - -const ( - opTypeSet opType = iota + 1 - opTypeDelete -) - -type operation struct { - opType - key []byte - value []byte -} +import "go.etcd.io/bbolt" // boltDBBatch stores operations internally and dumps them to BoltDB on Write(). type boltDBBatch struct { @@ -24,7 +10,7 @@ type boltDBBatch struct { ops []operation } -var _ tmdb.Batch = (*boltDBBatch)(nil) +var _ Batch = (*boltDBBatch)(nil) func newBoltDBBatch(db *BoltDB) *boltDBBatch { return &boltDBBatch{ @@ -36,13 +22,13 @@ func newBoltDBBatch(db *BoltDB) *boltDBBatch { // Set implements Batch. func (b *boltDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.ops = append(b.ops, operation{opTypeSet, key, value}) return nil @@ -51,10 +37,10 @@ func (b *boltDBBatch) Set(key, value []byte) error { // Delete implements Batch. func (b *boltDBBatch) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.ops = append(b.ops, operation{opTypeDelete, key, nil}) return nil @@ -63,7 +49,7 @@ func (b *boltDBBatch) Delete(key []byte) error { // Write implements Batch. func (b *boltDBBatch) Write() error { if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } err := b.db.db.Batch(func(tx *bbolt.Tx) error { bkt := tx.Bucket(bucket) diff --git a/boltdb/iterator.go b/boltdb_iterator.go similarity index 96% rename from boltdb/iterator.go rename to boltdb_iterator.go index fa855ac74..74212c5ca 100644 --- a/boltdb/iterator.go +++ b/boltdb_iterator.go @@ -1,9 +1,10 @@ -package boltdb +// +build boltdb + +package db import ( "bytes" - tmdb "github.com/tendermint/tm-db" "go.etcd.io/bbolt" ) @@ -23,7 +24,7 @@ type boltDBIterator struct { isReverse bool } -var _ tmdb.Iterator = (*boltDBIterator)(nil) +var _ Iterator = (*boltDBIterator)(nil) // newBoltDBIterator creates a new boltDBIterator. func newBoltDBIterator(tx *bbolt.Tx, start, end []byte, isReverse bool) *boltDBIterator { diff --git a/boltdb_test.go b/boltdb_test.go new file mode 100644 index 000000000..0fd12bbe2 --- /dev/null +++ b/boltdb_test.go @@ -0,0 +1,35 @@ +// +build boltdb + +package db + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBoltDBNewBoltDB(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + defer cleanupDBDir(dir, name) + + db, err := NewBoltDB(name, dir) + require.NoError(t, err) + db.Close() +} + +func BenchmarkBoltDBRandomReadsWrites(b *testing.B) { + name := fmt.Sprintf("test_%x", randStr(12)) + db, err := NewBoltDB(name, "") + if err != nil { + b.Fatal(err) + } + defer func() { + db.Close() + cleanupDBDir("", name) + }() + + benchmarkRandomReadsWrites(b, db) +} diff --git a/cleveldb/db.go b/cleveldb.go similarity index 82% rename from cleveldb/db.go rename to cleveldb.go index e796ea83f..377956deb 100644 --- a/cleveldb/db.go +++ b/cleveldb.go @@ -1,14 +1,21 @@ -package cleveldb +// +build cleveldb + +package db import ( "fmt" "path/filepath" "github.com/jmhodges/levigo" - - tmdb "github.com/tendermint/tm-db" ) +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewCLevelDB(name, dir) + } + registerDBCreator(CLevelDBBackend, dbCreator, false) +} + // CLevelDB uses the C LevelDB database via a Go wrapper. type CLevelDB struct { db *levigo.DB @@ -17,10 +24,10 @@ type CLevelDB struct { woSync *levigo.WriteOptions } -var _ tmdb.DB = (*CLevelDB)(nil) +var _ DB = (*CLevelDB)(nil) -// New creates a new CLevelDB. -func NewDB(name string, dir string) (*CLevelDB, error) { +// NewCLevelDB creates a new CLevelDB. +func NewCLevelDB(name string, dir string) (*CLevelDB, error) { dbPath := filepath.Join(dir, name+".db") opts := levigo.NewOptions() @@ -46,7 +53,7 @@ func NewDB(name string, dir string) (*CLevelDB, error) { // Get implements DB. func (db *CLevelDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } res, err := db.db.Get(db.ro, key) if err != nil { @@ -67,10 +74,10 @@ func (db *CLevelDB) Has(key []byte) (bool, error) { // Set implements DB. func (db *CLevelDB) Set(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if err := db.db.Put(db.wo, key, value); err != nil { return err @@ -81,10 +88,10 @@ func (db *CLevelDB) Set(key []byte, value []byte) error { // SetSync implements DB. func (db *CLevelDB) SetSync(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if err := db.db.Put(db.woSync, key, value); err != nil { return err @@ -95,7 +102,7 @@ func (db *CLevelDB) SetSync(key []byte, value []byte) error { // Delete implements DB. func (db *CLevelDB) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if err := db.db.Delete(db.wo, key); err != nil { return err @@ -106,7 +113,7 @@ func (db *CLevelDB) Delete(key []byte) error { // DeleteSync implements DB. func (db *CLevelDB) DeleteSync(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if err := db.db.Delete(db.woSync, key); err != nil { return err @@ -165,23 +172,23 @@ func (db *CLevelDB) Stats() map[string]string { } // NewBatch implements DB. -func (db *CLevelDB) NewBatch() tmdb.Batch { +func (db *CLevelDB) NewBatch() Batch { return newCLevelDBBatch(db) } // Iterator implements DB. -func (db *CLevelDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (db *CLevelDB) Iterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } itr := db.db.NewIterator(db.ro) return newCLevelDBIterator(itr, start, end, false), nil } // ReverseIterator implements DB. -func (db *CLevelDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (db *CLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } itr := db.db.NewIterator(db.ro) return newCLevelDBIterator(itr, start, end, true), nil diff --git a/cleveldb/db_test.go b/cleveldb/db_test.go deleted file mode 100644 index a451f01fc..000000000 --- a/cleveldb/db_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package cleveldb - -import ( - "bytes" - "fmt" - "math/rand" - "testing" - - "github.com/tendermint/tm-db/internal/dbtest" -) - -func BenchmarkRandomReadsWrites2(b *testing.B) { - b.StopTimer() - - numItems := int64(1000000) - internal := map[int64]int64{} - for i := 0; i < int(numItems); i++ { - internal[int64(i)] = int64(0) - } - db, err := NewDB(fmt.Sprintf("test_%x", dbtest.RandStr(12)), "") - if err != nil { - b.Fatal(err.Error()) - return - } - - fmt.Println("ok, starting") - b.StartTimer() - - for i := 0; i < b.N; i++ { - // Write something - { - idx := (int64(rand.Int()) % numItems) - internal[idx]++ - val := internal[idx] - idxBytes := dbtest.Int642Bytes(idx) - valBytes := dbtest.Int642Bytes(val) - // fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) - if err = db.Set( - idxBytes, - valBytes, - ); err != nil { - b.Error(err) - } - } - // Read something - { - idx := (int64(rand.Int()) % numItems) - val := internal[idx] - idxBytes := dbtest.Int642Bytes(idx) - valBytes, err := db.Get(idxBytes) - if err != nil { - b.Error(err) - } - // fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) - if val == 0 { - if !bytes.Equal(valBytes, nil) { - b.Errorf("Expected %v for %v, got %X", - nil, idx, valBytes) - break - } - } else { - if len(valBytes) != 8 { - b.Errorf("Expected length 8 for %v, got %X", - idx, valBytes) - break - } - valGot := dbtest.Bytes2Int64(valBytes) - if val != valGot { - b.Errorf("Expected %v for %v, got %v", - val, idx, valGot) - break - } - } - } - } - - db.Close() -} diff --git a/cleveldb/batch.go b/cleveldb_batch.go similarity index 81% rename from cleveldb/batch.go rename to cleveldb_batch.go index c1a2aa210..132a521cb 100644 --- a/cleveldb/batch.go +++ b/cleveldb_batch.go @@ -1,9 +1,8 @@ -package cleveldb +// +build cleveldb -import ( - "github.com/jmhodges/levigo" - tmdb "github.com/tendermint/tm-db" -) +package db + +import "github.com/jmhodges/levigo" // cLevelDBBatch is a LevelDB batch. type cLevelDBBatch struct { @@ -21,13 +20,13 @@ func newCLevelDBBatch(db *CLevelDB) *cLevelDBBatch { // Set implements Batch. func (b *cLevelDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.batch.Put(key, value) return nil @@ -36,10 +35,10 @@ func (b *cLevelDBBatch) Set(key, value []byte) error { // Delete implements Batch. func (b *cLevelDBBatch) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.batch.Delete(key) return nil @@ -48,7 +47,7 @@ func (b *cLevelDBBatch) Delete(key []byte) error { // Write implements Batch. func (b *cLevelDBBatch) Write() error { if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } err := b.db.db.Write(b.db.wo, b.batch) if err != nil { @@ -61,7 +60,7 @@ func (b *cLevelDBBatch) Write() error { // WriteSync implements Batch. func (b *cLevelDBBatch) WriteSync() error { if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } err := b.db.db.Write(b.db.woSync, b.batch) if err != nil { diff --git a/cleveldb/iterator.go b/cleveldb_iterator.go similarity index 94% rename from cleveldb/iterator.go rename to cleveldb_iterator.go index ce019880f..04375d565 100644 --- a/cleveldb/iterator.go +++ b/cleveldb_iterator.go @@ -1,10 +1,11 @@ -package cleveldb +// +build cleveldb + +package db import ( "bytes" "github.com/jmhodges/levigo" - tmdb "github.com/tendermint/tm-db" ) // cLevelDBIterator is a cLevelDB iterator. @@ -15,11 +16,11 @@ type cLevelDBIterator struct { isInvalid bool } -var _ tmdb.Iterator = (*cLevelDBIterator)(nil) +var _ Iterator = (*cLevelDBIterator)(nil) func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator { if isReverse { - if len(end) == 0 { + if end == nil || len(end) == 0 { source.SeekToLast() } else { source.Seek(end) @@ -33,7 +34,7 @@ func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse b } } } else { - if len(start) == 0 { + if start == nil || len(start) == 0 { source.SeekToFirst() } else { source.Seek(start) diff --git a/cleveldb_test.go b/cleveldb_test.go new file mode 100644 index 000000000..61e2fb6ef --- /dev/null +++ b/cleveldb_test.go @@ -0,0 +1,103 @@ +// +build cleveldb + +package db + +import ( + "bytes" + "fmt" + "math/rand" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func BenchmarkRandomReadsWrites2(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(rand.Int()) % numItems) + internal[idx]++ + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := int642Bytes(int64(val)) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(rand.Int()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes, err := db.Get(idxBytes) + if err != nil { + b.Error(err) + } + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %v for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +func TestCLevelDBBackend(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + // Can't use "" (current directory) or "./" here because levigo.Open returns: + // "Error initializing DB: IO error: test_XXX.db: Invalid argument" + dir := os.TempDir() + db, err := NewDB(name, CLevelDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + _, ok := db.(*CLevelDB) + assert.True(t, ok) +} + +func TestCLevelDBStats(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, CLevelDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + assert.NotEmpty(t, db.Stats()) +} diff --git a/internal/dbtest/dbtest.go b/common_test.go similarity index 56% rename from internal/dbtest/dbtest.go rename to common_test.go index 4315b886f..f48efd7aa 100644 --- a/internal/dbtest/dbtest.go +++ b/common_test.go @@ -1,44 +1,48 @@ -package dbtest +package db import ( "bytes" "encoding/binary" + "io/ioutil" "math/rand" - "os" - "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmdb "github.com/tendermint/tm-db" ) //---------------------------------------- // Helper functions. -func Valid(t *testing.T, itr tmdb.Iterator, expected bool) { +func checkValue(t *testing.T, db DB, key []byte, valueWanted []byte) { + valueGot, err := db.Get(key) + assert.NoError(t, err) + assert.Equal(t, valueWanted, valueGot) +} + +func checkValid(t *testing.T, itr Iterator, expected bool) { valid := itr.Valid() require.Equal(t, expected, valid) } -func Next(t *testing.T, itr tmdb.Iterator, expected bool) { +func checkNext(t *testing.T, itr Iterator, expected bool) { itr.Next() // assert.NoError(t, err) TODO: look at fixing this valid := itr.Valid() require.Equal(t, expected, valid) } -func NextPanics(t *testing.T, itr tmdb.Iterator) { +func checkNextPanics(t *testing.T, itr Iterator) { assert.Panics(t, func() { itr.Next() }, "checkNextPanics expected an error but didn't") } -func Domain(t *testing.T, itr tmdb.Iterator, start, end []byte) { +func checkDomain(t *testing.T, itr Iterator, start, end []byte) { ds, de := itr.Domain() assert.Equal(t, start, ds, "checkDomain domain start incorrect") assert.Equal(t, end, de, "checkDomain domain end incorrect") } -func Item(t *testing.T, itr tmdb.Iterator, key []byte, value []byte) { +func checkItem(t *testing.T, itr Iterator, key []byte, value []byte) { v := itr.Value() k := itr.Key() @@ -47,71 +51,30 @@ func Item(t *testing.T, itr tmdb.Iterator, key []byte, value []byte) { assert.Exactly(t, value, v) } -func Invalid(t *testing.T, itr tmdb.Iterator) { - Valid(t, itr, false) - KeyPanics(t, itr) - ValuePanics(t, itr) - NextPanics(t, itr) +func checkInvalid(t *testing.T, itr Iterator) { + checkValid(t, itr, false) + checkKeyPanics(t, itr) + checkValuePanics(t, itr) + checkNextPanics(t, itr) } -func KeyPanics(t *testing.T, itr tmdb.Iterator) { +func checkKeyPanics(t *testing.T, itr Iterator) { assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") } -func Value(t *testing.T, db tmdb.DB, key []byte, valueWanted []byte) { - valueGot, err := db.Get(key) - assert.NoError(t, err) - assert.Equal(t, valueWanted, valueGot) -} - -func ValuePanics(t *testing.T, itr tmdb.Iterator) { +func checkValuePanics(t *testing.T, itr Iterator) { assert.Panics(t, func() { itr.Value() }) } -func CleanupDBDir(dir, name string) { - err := os.RemoveAll(filepath.Join(dir, name) + ".db") - if err != nil { - panic(err) - } -} - -const strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters - -// RandStr constructs a random alphanumeric string of given length. -func RandStr(length int) string { - chars := []byte{} -MAIN_LOOP: - for { - val := rand.Int63() // nolint:gosec // G404: Use of weak random number generator - for i := 0; i < 10; i++ { - v := int(val & 0x3f) // rightmost 6 bits - if v >= 62 { // only 62 characters in strChars - val >>= 6 - continue - } else { - chars = append(chars, strChars[v]) - if len(chars) == length { - break MAIN_LOOP - } - val >>= 6 - } - } - } - - return string(chars) -} - -func Int642Bytes(i int64) []byte { - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, uint64(i)) - return buf -} - -func Bytes2Int64(buf []byte) int64 { - return int64(binary.BigEndian.Uint64(buf)) +func newTempDB(t *testing.T, backend BackendType) (db DB, dbDir string) { + dirname, err := ioutil.TempDir("", "db_common_test") + require.NoError(t, err) + db, err = NewDB("testdb", backend, dirname) + require.NoError(t, err) + return db, dirname } -func BenchmarkRangeScans(b *testing.B, db tmdb.DB, dbSize int64) { +func benchmarkRangeScans(b *testing.B, db DB, dbSize int64) { b.StopTimer() rangeSize := int64(10000) @@ -120,7 +83,7 @@ func BenchmarkRangeScans(b *testing.B, db tmdb.DB, dbSize int64) { } for i := int64(0); i < dbSize; i++ { - bytes := Int642Bytes(i) + bytes := int642Bytes(i) err := db.Set(bytes, bytes) if err != nil { // require.NoError() is very expensive (according to profiler), so check manually @@ -130,10 +93,9 @@ func BenchmarkRangeScans(b *testing.B, db tmdb.DB, dbSize int64) { b.StartTimer() for i := 0; i < b.N; i++ { - - start := rand.Int63n(dbSize - rangeSize) // nolint: gosec + start := rand.Int63n(dbSize - rangeSize) end := start + rangeSize - iter, err := db.Iterator(Int642Bytes(start), Int642Bytes(end)) + iter, err := db.Iterator(int642Bytes(start), int642Bytes(end)) require.NoError(b, err) count := 0 for ; iter.Valid(); iter.Next() { @@ -144,7 +106,7 @@ func BenchmarkRangeScans(b *testing.B, db tmdb.DB, dbSize int64) { } } -func BenchmarkRandomReadsWrites(b *testing.B, db tmdb.DB) { +func benchmarkRandomReadsWrites(b *testing.B, db DB) { b.StopTimer() // create dummy data @@ -160,12 +122,12 @@ func BenchmarkRandomReadsWrites(b *testing.B, db tmdb.DB) { for i := 0; i < b.N; i++ { // Write something { - idx := rand.Int63n(numItems) // nolint: gosec + idx := rand.Int63n(numItems) internal[idx]++ val := internal[idx] - idxBytes := Int642Bytes(idx) - valBytes := Int642Bytes(val) - // fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + idxBytes := int642Bytes(idx) + valBytes := int642Bytes(val) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) err := db.Set(idxBytes, valBytes) if err != nil { // require.NoError() is very expensive (according to profiler), so check manually @@ -175,15 +137,15 @@ func BenchmarkRandomReadsWrites(b *testing.B, db tmdb.DB) { // Read something { - idx := rand.Int63n(numItems) // nolint: gosec + idx := rand.Int63n(numItems) valExp := internal[idx] - idxBytes := Int642Bytes(idx) + idxBytes := int642Bytes(idx) valBytes, err := db.Get(idxBytes) if err != nil { // require.NoError() is very expensive (according to profiler), so check manually b.Fatal(b, err) } - // fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) if valExp == 0 { if !bytes.Equal(valBytes, nil) { b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes) @@ -194,7 +156,7 @@ func BenchmarkRandomReadsWrites(b *testing.B, db tmdb.DB) { b.Errorf("Expected length 8 for %v, got %X", idx, valBytes) break } - valGot := Bytes2Int64(valBytes) + valGot := bytes2Int64(valBytes) if valExp != valGot { b.Errorf("Expected %v for %v, got %v", valExp, idx, valGot) break @@ -204,3 +166,13 @@ func BenchmarkRandomReadsWrites(b *testing.B, db tmdb.DB) { } } + +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} diff --git a/metadb/db.go b/db.go similarity index 80% rename from metadb/db.go rename to db.go index 58951f4e0..4d518c0b0 100644 --- a/metadb/db.go +++ b/db.go @@ -1,10 +1,8 @@ -package metadb +package db import ( "fmt" "strings" - - tmdb "github.com/tendermint/tm-db" ) type BackendType string @@ -15,7 +13,6 @@ const ( // popular implementation) // - pure go // - stable - // - use goleveldb build tag (go build -tags goleveldb) GoLevelDBBackend BackendType = "goleveldb" // CLevelDBBackend represents cleveldb (uses levigo wrapper) // - fast @@ -24,7 +21,6 @@ const ( CLevelDBBackend BackendType = "cleveldb" // MemDBBackend represents in-memory key value store, which is mostly used // for testing. - // - use memdb build tag (go build -tags memdb) MemDBBackend BackendType = "memdb" // BoltDBBackend represents bolt (uses etcd's fork of bolt - // github.com/etcd-io/bbolt) @@ -37,13 +33,11 @@ const ( // - requires gcc // - use rocksdb build tag (go build -tags rocksdb) RocksDBBackend BackendType = "rocksdb" - // BadgerDBBackend represents badger (uses github.com/dgraph-io/badger/v2) - // - EXPERIMENTAL - // - use badgerdb build tag (go build -tags badgerdb) + BadgerDBBackend BackendType = "badgerdb" ) -type dbCreator func(name string, dir string) (tmdb.DB, error) +type dbCreator func(name string, dir string) (DB, error) var backends = map[BackendType]dbCreator{} @@ -56,7 +50,7 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) { } // NewDB creates a new database of type backend with the given name. -func NewDB(name string, backend BackendType, dir string) (tmdb.DB, error) { +func NewDB(name string, backend BackendType, dir string) (DB, error) { dbCreator, ok := backends[backend] if !ok { keys := make([]string, 0, len(backends)) diff --git a/metadb/db_test.go b/db_test.go similarity index 74% rename from metadb/db_test.go rename to db_test.go index 02fd4f43c..dc81d663b 100644 --- a/metadb/db_test.go +++ b/db_test.go @@ -1,4 +1,4 @@ -package metadb +package db import ( "fmt" @@ -6,7 +6,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/tm-db/internal/dbtest" ) func TestDBIteratorSingleKey(t *testing.T) { @@ -15,18 +14,18 @@ func TestDBIteratorSingleKey(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - err := db.SetSync([]byte("1"), []byte("value_1")) + err := db.SetSync(bz("1"), bz("value_1")) assert.NoError(t, err) itr, err := db.Iterator(nil, nil) assert.NoError(t, err) - dbtest.Valid(t, itr, true) - dbtest.Next(t, itr, false) - dbtest.Valid(t, itr, false) - dbtest.NextPanics(t, itr) + checkValid(t, itr, true) + checkNext(t, itr, false) + checkValid(t, itr, false) + checkNextPanics(t, itr) // Once invalid... - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -37,27 +36,27 @@ func TestDBIteratorTwoKeys(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - err := db.SetSync([]byte("1"), []byte("value_1")) + err := db.SetSync(bz("1"), bz("value_1")) assert.NoError(t, err) - err = db.SetSync([]byte("2"), []byte("value_1")) + err = db.SetSync(bz("2"), bz("value_1")) assert.NoError(t, err) { // Fail by calling Next too much itr, err := db.Iterator(nil, nil) assert.NoError(t, err) - dbtest.Valid(t, itr, true) + checkValid(t, itr, true) - dbtest.Next(t, itr, true) - dbtest.Valid(t, itr, true) + checkNext(t, itr, true) + checkValid(t, itr, true) - dbtest.Next(t, itr, false) - dbtest.Valid(t, itr, false) + checkNext(t, itr, false) + checkValid(t, itr, false) - dbtest.NextPanics(t, itr) + checkNextPanics(t, itr) // Once invalid... - dbtest.Invalid(t, itr) + checkInvalid(t, itr) } }) } @@ -104,7 +103,7 @@ func TestDBIteratorEmpty(t *testing.T) { itr, err := db.Iterator(nil, nil) assert.NoError(t, err) - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -115,10 +114,10 @@ func TestDBIteratorEmptyBeginAfter(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - itr, err := db.Iterator([]byte("1"), nil) + itr, err := db.Iterator(bz("1"), nil) assert.NoError(t, err) - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -129,12 +128,12 @@ func TestDBIteratorNonemptyBeginAfter(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - err := db.SetSync([]byte("1"), []byte("value_1")) + err := db.SetSync(bz("1"), bz("value_1")) assert.NoError(t, err) - itr, err := db.Iterator([]byte("2"), nil) + itr, err := db.Iterator(bz("2"), nil) assert.NoError(t, err) - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } diff --git a/goleveldb/db.go b/goleveldb.go similarity index 80% rename from goleveldb/db.go rename to goleveldb.go index c60fdf543..fd1bffd18 100644 --- a/goleveldb/db.go +++ b/goleveldb.go @@ -1,4 +1,4 @@ -package goleveldb +package db import ( "fmt" @@ -8,20 +8,26 @@ import ( "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" - tmdb "github.com/tendermint/tm-db" ) +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewGoLevelDB(name, dir) + } + registerDBCreator(GoLevelDBBackend, dbCreator, false) +} + type GoLevelDB struct { db *leveldb.DB } -var _ tmdb.DB = (*GoLevelDB)(nil) +var _ DB = (*GoLevelDB)(nil) -func NewDB(name string, dir string) (*GoLevelDB, error) { - return NewDBWithOpts(name, dir, nil) +func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { + return NewGoLevelDBWithOpts(name, dir, nil) } -func NewDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { +func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { dbPath := filepath.Join(dir, name+".db") db, err := leveldb.OpenFile(dbPath, o) if err != nil { @@ -36,7 +42,7 @@ func NewDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) // Get implements DB. func (db *GoLevelDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } res, err := db.db.Get(key, nil) if err != nil { @@ -60,10 +66,10 @@ func (db *GoLevelDB) Has(key []byte) (bool, error) { // Set implements DB. func (db *GoLevelDB) Set(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if err := db.db.Put(key, value, nil); err != nil { return err @@ -74,10 +80,10 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error { // SetSync implements DB. func (db *GoLevelDB) SetSync(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}); err != nil { return err @@ -88,7 +94,7 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error { // Delete implements DB. func (db *GoLevelDB) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if err := db.db.Delete(key, nil); err != nil { return err @@ -99,7 +105,7 @@ func (db *GoLevelDB) Delete(key []byte) error { // DeleteSync implements DB. func (db *GoLevelDB) DeleteSync(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) if err != nil { @@ -161,23 +167,23 @@ func (db *GoLevelDB) Stats() map[string]string { } // NewBatch implements DB. -func (db *GoLevelDB) NewBatch() tmdb.Batch { +func (db *GoLevelDB) NewBatch() Batch { return newGoLevelDBBatch(db) } // Iterator implements DB. -func (db *GoLevelDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (db *GoLevelDB) Iterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) return newGoLevelDBIterator(itr, start, end, false), nil } // ReverseIterator implements DB. -func (db *GoLevelDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) return newGoLevelDBIterator(itr, start, end, true), nil diff --git a/goleveldb/db_test.go b/goleveldb/db_test.go deleted file mode 100644 index a969a2464..000000000 --- a/goleveldb/db_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package goleveldb - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/tendermint/tm-db/internal/dbtest" -) - -func TestGoLevelDBNewDB(t *testing.T) { - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - defer dbtest.CleanupDBDir("", name) - - // Test we can't open the db twice for writing - wr1, err := NewDB(name, "") - require.Nil(t, err) - _, err = NewDB(name, "") - require.NotNil(t, err) - wr1.Close() // Close the db to release the lock - - // Test we can open the db twice for reading only - ro1, err := NewDBWithOpts(name, "", &opt.Options{ReadOnly: true}) - require.Nil(t, err) - defer ro1.Close() - ro2, err := NewDBWithOpts(name, "", &opt.Options{ReadOnly: true}) - require.Nil(t, err) - defer ro2.Close() -} - -func BenchmarkGoLevelDBRandomReadsWrites(b *testing.B) { - name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) - db, err := NewDB(name, "") - if err != nil { - b.Fatal(err) - } - defer func() { - db.Close() - dbtest.CleanupDBDir("", name) - }() - - dbtest.BenchmarkRandomReadsWrites(b, db) -} diff --git a/goleveldb/batch.go b/goleveldb_batch.go similarity index 82% rename from goleveldb/batch.go rename to goleveldb_batch.go index 5f9f4a7d6..4c1c6a2ba 100644 --- a/goleveldb/batch.go +++ b/goleveldb_batch.go @@ -1,9 +1,8 @@ -package goleveldb +package db import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" - tmdb "github.com/tendermint/tm-db" ) type goLevelDBBatch struct { @@ -11,7 +10,7 @@ type goLevelDBBatch struct { batch *leveldb.Batch } -var _ tmdb.Batch = (*goLevelDBBatch)(nil) +var _ Batch = (*goLevelDBBatch)(nil) func newGoLevelDBBatch(db *GoLevelDB) *goLevelDBBatch { return &goLevelDBBatch{ @@ -23,13 +22,13 @@ func newGoLevelDBBatch(db *GoLevelDB) *goLevelDBBatch { // Set implements Batch. func (b *goLevelDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.batch.Put(key, value) return nil @@ -38,10 +37,10 @@ func (b *goLevelDBBatch) Set(key, value []byte) error { // Delete implements Batch. func (b *goLevelDBBatch) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.batch.Delete(key) return nil @@ -59,7 +58,7 @@ func (b *goLevelDBBatch) WriteSync() error { func (b *goLevelDBBatch) write(sync bool) error { if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync}) if err != nil { diff --git a/goleveldb/iterator.go b/goleveldb_iterator.go similarity index 93% rename from goleveldb/iterator.go rename to goleveldb_iterator.go index 4a6add77d..9cf9a5e15 100644 --- a/goleveldb/iterator.go +++ b/goleveldb_iterator.go @@ -1,10 +1,9 @@ -package goleveldb +package db import ( "bytes" "github.com/syndtr/goleveldb/leveldb/iterator" - tmdb "github.com/tendermint/tm-db" ) type goLevelDBIterator struct { @@ -15,7 +14,7 @@ type goLevelDBIterator struct { isInvalid bool } -var _ tmdb.Iterator = (*goLevelDBIterator)(nil) +var _ Iterator = (*goLevelDBIterator)(nil) func newGoLevelDBIterator(source iterator.Iterator, start, end []byte, isReverse bool) *goLevelDBIterator { if isReverse { @@ -110,12 +109,6 @@ func (itr *goLevelDBIterator) Value() []byte { return cp(itr.source.Value()) } -func cp(bz []byte) (ret []byte) { - ret = make([]byte, len(bz)) - copy(ret, bz) - return ret -} - // Next implements Iterator. func (itr *goLevelDBIterator) Next() { itr.assertIsValid() diff --git a/goleveldb_test.go b/goleveldb_test.go new file mode 100644 index 000000000..e1c879f10 --- /dev/null +++ b/goleveldb_test.go @@ -0,0 +1,43 @@ +package db + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +func TestGoLevelDBNewGoLevelDB(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + defer cleanupDBDir("", name) + + // Test we can't open the db twice for writing + wr1, err := NewGoLevelDB(name, "") + require.Nil(t, err) + _, err = NewGoLevelDB(name, "") + require.NotNil(t, err) + wr1.Close() // Close the db to release the lock + + // Test we can open the db twice for reading only + ro1, err := NewGoLevelDBWithOpts(name, "", &opt.Options{ReadOnly: true}) + require.Nil(t, err) + defer ro1.Close() + ro2, err := NewGoLevelDBWithOpts(name, "", &opt.Options{ReadOnly: true}) + require.Nil(t, err) + defer ro2.Close() +} + +func BenchmarkGoLevelDBRandomReadsWrites(b *testing.B) { + name := fmt.Sprintf("test_%x", randStr(12)) + db, err := NewGoLevelDB(name, "") + if err != nil { + b.Fatal(err) + } + defer func() { + db.Close() + cleanupDBDir("", name) + }() + + benchmarkRandomReadsWrites(b, db) +} diff --git a/makefile b/makefile index fa195ea1e..bedebc457 100644 --- a/makefile +++ b/makefile @@ -12,14 +12,6 @@ test: @echo "--> Running go test" @go test $(PACKAGES) -v -test-memdb: - @echo "--> Running go test" - @go test $(PACKAGES) -tags memdb -v - -test-goleveldb: - @echo "--> Running go test" - @go test $(PACKAGES) -tags goleveldb -v - test-cleveldb: @echo "--> Running go test" @go test $(PACKAGES) -tags cleveldb -v @@ -38,12 +30,7 @@ test-badgerdb: test-all: @echo "--> Running go test" - @go test $(PACKAGES) -tags memdb,goleveldb,cleveldb,boltdb,rocksdb,badgerdb -v - -test-all-docker: - @echo "--> Running go test" - @docker run --rm -v $(CURDIR):/workspace --workdir /workspace tendermintdev/docker-tm-db-testing go test $(PACKAGES) -tags memdb,goleveldb,cleveldb,boltdb,rocksdb,badgerdb -v -.PHONY: test-all-docker + @go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,badgerdb -v lint: @echo "--> Running linter" diff --git a/memdb/db.go b/memdb.go similarity index 87% rename from memdb/db.go rename to memdb.go index 1053f637b..604e6a06e 100644 --- a/memdb/db.go +++ b/memdb.go @@ -1,4 +1,4 @@ -package memdb +package db import ( "bytes" @@ -6,7 +6,6 @@ import ( "sync" "github.com/google/btree" - tmdb "github.com/tendermint/tm-db" ) const ( @@ -14,6 +13,12 @@ const ( bTreeDegree = 32 ) +func init() { + registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) { + return NewMemDB(), nil + }, false) +} + // item is a btree.Item with byte slices as keys and values type item struct { key []byte @@ -48,10 +53,10 @@ type MemDB struct { btree *btree.BTree } -var _ tmdb.DB = (*MemDB)(nil) +var _ DB = (*MemDB)(nil) -// NewDB creates a new in-memory database. -func NewDB() *MemDB { +// NewMemDB creates a new in-memory database. +func NewMemDB() *MemDB { database := &MemDB{ btree: btree.New(bTreeDegree), } @@ -61,7 +66,7 @@ func NewDB() *MemDB { // Get implements DB. func (db *MemDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } db.mtx.RLock() defer db.mtx.RUnlock() @@ -76,7 +81,7 @@ func (db *MemDB) Get(key []byte) ([]byte, error) { // Has implements DB. func (db *MemDB) Has(key []byte) (bool, error) { if len(key) == 0 { - return false, tmdb.ErrKeyEmpty + return false, errKeyEmpty } db.mtx.RLock() defer db.mtx.RUnlock() @@ -87,10 +92,10 @@ func (db *MemDB) Has(key []byte) (bool, error) { // Set implements DB. func (db *MemDB) Set(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } db.mtx.Lock() defer db.mtx.Unlock() @@ -112,7 +117,7 @@ func (db *MemDB) SetSync(key []byte, value []byte) error { // Delete implements DB. func (db *MemDB) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } db.mtx.Lock() defer db.mtx.Unlock() @@ -164,24 +169,24 @@ func (db *MemDB) Stats() map[string]string { } // NewBatch implements DB. -func (db *MemDB) NewBatch() tmdb.Batch { +func (db *MemDB) NewBatch() Batch { return newMemDBBatch(db) } // Iterator implements DB. // Takes out a read-lock on the database until the iterator is closed. -func (db *MemDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (db *MemDB) Iterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } return newMemDBIterator(db, start, end, false), nil } // ReverseIterator implements DB. // Takes out a read-lock on the database until the iterator is closed. -func (db *MemDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (db *MemDB) ReverseIterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } return newMemDBIterator(db, start, end, true), nil } diff --git a/memdb/db_test.go b/memdb/db_test.go deleted file mode 100644 index 9d38def9f..000000000 --- a/memdb/db_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package memdb - -import ( - "testing" - - "github.com/tendermint/tm-db/internal/dbtest" -) - -func BenchmarkMemDBRangeScans1M(b *testing.B) { - db := NewDB() - defer db.Close() - - dbtest.BenchmarkRangeScans(b, db, int64(1e6)) -} - -func BenchmarkMemDBRangeScans10M(b *testing.B) { - db := NewDB() - defer db.Close() - - dbtest.BenchmarkRangeScans(b, db, int64(10e6)) -} - -func BenchmarkMemDBRandomReadsWrites(b *testing.B) { - db := NewDB() - defer db.Close() - - dbtest.BenchmarkRandomReadsWrites(b, db) -} diff --git a/memdb/batch.go b/memdb_batch.go similarity index 84% rename from memdb/batch.go rename to memdb_batch.go index cd3b4c1a2..6ff30fea6 100644 --- a/memdb/batch.go +++ b/memdb_batch.go @@ -1,10 +1,6 @@ -package memdb +package db -import ( - "fmt" - - tmdb "github.com/tendermint/tm-db" -) +import "fmt" // memDBBatch operations type opType int @@ -26,7 +22,7 @@ type memDBBatch struct { ops []operation } -var _ tmdb.Batch = (*memDBBatch)(nil) +var _ Batch = (*memDBBatch)(nil) // newMemDBBatch creates a new memDBBatch func newMemDBBatch(db *MemDB) *memDBBatch { @@ -39,13 +35,13 @@ func newMemDBBatch(db *MemDB) *memDBBatch { // Set implements Batch. func (b *memDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.ops = append(b.ops, operation{opTypeSet, key, value}) return nil @@ -54,10 +50,10 @@ func (b *memDBBatch) Set(key, value []byte) error { // Delete implements Batch. func (b *memDBBatch) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.ops = append(b.ops, operation{opTypeDelete, key, nil}) return nil @@ -66,7 +62,7 @@ func (b *memDBBatch) Delete(key []byte) error { // Write implements Batch. func (b *memDBBatch) Write() error { if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.db.mtx.Lock() defer b.db.mtx.Unlock() diff --git a/memdb/iterator.go b/memdb_iterator.go similarity index 97% rename from memdb/iterator.go rename to memdb_iterator.go index 8def00650..2a61e3757 100644 --- a/memdb/iterator.go +++ b/memdb_iterator.go @@ -1,11 +1,10 @@ -package memdb +package db import ( "bytes" "context" "github.com/google/btree" - tmdb "github.com/tendermint/tm-db" ) const ( @@ -24,7 +23,7 @@ type memDBIterator struct { end []byte } -var _ tmdb.Iterator = (*memDBIterator)(nil) +var _ Iterator = (*memDBIterator)(nil) // newMemDBIterator creates a new memDBIterator. func newMemDBIterator(db *MemDB, start []byte, end []byte, reverse bool) *memDBIterator { diff --git a/memdb_test.go b/memdb_test.go new file mode 100644 index 000000000..4e67e813d --- /dev/null +++ b/memdb_test.go @@ -0,0 +1,26 @@ +package db + +import ( + "testing" +) + +func BenchmarkMemDBRangeScans1M(b *testing.B) { + db := NewMemDB() + defer db.Close() + + benchmarkRangeScans(b, db, int64(1e6)) +} + +func BenchmarkMemDBRangeScans10M(b *testing.B) { + db := NewMemDB() + defer db.Close() + + benchmarkRangeScans(b, db, int64(10e6)) +} + +func BenchmarkMemDBRandomReadsWrites(b *testing.B) { + db := NewMemDB() + defer db.Close() + + benchmarkRandomReadsWrites(b, db) +} diff --git a/metadb/db_badgerdb.go b/metadb/db_badgerdb.go deleted file mode 100644 index ee45ae910..000000000 --- a/metadb/db_badgerdb.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build badgerdb - -package metadb - -import ( - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/badgerdb" -) - -func badgerDBCreator(name, dir string) (tmdb.DB, error) { - return badgerdb.NewDB(name, dir) -} - -func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } diff --git a/metadb/db_boltdb.go b/metadb/db_boltdb.go deleted file mode 100644 index afcc92aab..000000000 --- a/metadb/db_boltdb.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build boltdb - -package metadb - -import ( - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/boltdb" -) - -func boltDBCreator(name, dir string) (tmdb.DB, error) { - return boltdb.NewDB(name, dir) -} - -func init() { registerDBCreator(BoltDBBackend, boltDBCreator, true) } diff --git a/metadb/db_cleveldb.go b/metadb/db_cleveldb.go deleted file mode 100644 index bed0bdb46..000000000 --- a/metadb/db_cleveldb.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build cleveldb - -package metadb - -import ( - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/cleveldb" -) - -func clevelDBCreator(name string, dir string) (tmdb.DB, error) { - return cleveldb.NewDB(name, dir) -} - -func init() { registerDBCreator(CLevelDBBackend, clevelDBCreator, false) } diff --git a/metadb/db_goleveldb.go b/metadb/db_goleveldb.go deleted file mode 100644 index d4ee7b5c1..000000000 --- a/metadb/db_goleveldb.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build goleveldb - -package metadb - -import ( - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/goleveldb" -) - -func golevelDBCreator(name, dir string) (tmdb.DB, error) { - return goleveldb.NewDB(name, dir) -} - -func init() { registerDBCreator(GoLevelDBBackend, golevelDBCreator, true) } diff --git a/metadb/db_memdb.go b/metadb/db_memdb.go deleted file mode 100644 index 4273f5402..000000000 --- a/metadb/db_memdb.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build memdb - -package metadb - -import ( - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/memdb" -) - -func memdbDBCreator(name, dir string) (tmdb.DB, error) { - return memdb.NewDB(), nil -} - -func init() { registerDBCreator(MemDBBackend, memdbDBCreator, false) } diff --git a/metadb/db_rocksdb.go b/metadb/db_rocksdb.go deleted file mode 100644 index 98f1b6162..000000000 --- a/metadb/db_rocksdb.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build rocksdb - -package metadb - -import ( - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/rocksdb" -) - -func rocksDBCreator(name, dir string) (tmdb.DB, error) { - return rocksdb.NewDB(name, dir) -} - -func init() { registerDBCreator(RocksDBBackend, rocksDBCreator, true) } diff --git a/prefixdb.go b/prefixdb.go index fd3386e51..0b2d2a1cf 100644 --- a/prefixdb.go +++ b/prefixdb.go @@ -25,7 +25,7 @@ func NewPrefixDB(db DB, prefix []byte) *PrefixDB { // Get implements DB. func (pdb *PrefixDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { - return nil, ErrKeyEmpty + return nil, errKeyEmpty } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -41,7 +41,7 @@ func (pdb *PrefixDB) Get(key []byte) ([]byte, error) { // Has implements DB. func (pdb *PrefixDB) Has(key []byte) (bool, error) { if len(key) == 0 { - return false, ErrKeyEmpty + return false, errKeyEmpty } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -57,10 +57,10 @@ func (pdb *PrefixDB) Has(key []byte) (bool, error) { // Set implements DB. func (pdb *PrefixDB) Set(key []byte, value []byte) error { if len(key) == 0 { - return ErrKeyEmpty + return errKeyEmpty } if value == nil { - return ErrValueNil + return errValueNil } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -75,10 +75,10 @@ func (pdb *PrefixDB) Set(key []byte, value []byte) error { // SetSync implements DB. func (pdb *PrefixDB) SetSync(key []byte, value []byte) error { if len(key) == 0 { - return ErrKeyEmpty + return errKeyEmpty } if value == nil { - return ErrValueNil + return errValueNil } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -89,7 +89,7 @@ func (pdb *PrefixDB) SetSync(key []byte, value []byte) error { // Delete implements DB. func (pdb *PrefixDB) Delete(key []byte) error { if len(key) == 0 { - return ErrKeyEmpty + return errKeyEmpty } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -100,7 +100,7 @@ func (pdb *PrefixDB) Delete(key []byte) error { // DeleteSync implements DB. func (pdb *PrefixDB) DeleteSync(key []byte) error { if len(key) == 0 { - return ErrKeyEmpty + return errKeyEmpty } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -111,7 +111,7 @@ func (pdb *PrefixDB) DeleteSync(key []byte) error { // Iterator implements DB. func (pdb *PrefixDB) Iterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, ErrKeyEmpty + return nil, errKeyEmpty } pdb.mtx.Lock() defer pdb.mtx.Unlock() @@ -134,7 +134,7 @@ func (pdb *PrefixDB) Iterator(start, end []byte) (Iterator, error) { // ReverseIterator implements DB. func (pdb *PrefixDB) ReverseIterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, ErrKeyEmpty + return nil, errKeyEmpty } pdb.mtx.Lock() defer pdb.mtx.Unlock() diff --git a/prefixdb_batch.go b/prefixdb_batch.go index 089dd1adf..1a8005cad 100644 --- a/prefixdb_batch.go +++ b/prefixdb_batch.go @@ -17,10 +17,10 @@ func newPrefixBatch(prefix []byte, source Batch) prefixDBBatch { // Set implements Batch. func (pb prefixDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return ErrKeyEmpty + return errKeyEmpty } if value == nil { - return ErrValueNil + return errValueNil } pkey := append(cp(pb.prefix), key...) return pb.source.Set(pkey, value) @@ -29,7 +29,7 @@ func (pb prefixDBBatch) Set(key, value []byte) error { // Delete implements Batch. func (pb prefixDBBatch) Delete(key []byte) error { if len(key) == 0 { - return ErrKeyEmpty + return errKeyEmpty } pkey := append(cp(pb.prefix), key...) return pb.source.Delete(pkey) diff --git a/prefixdb_test.go b/prefixdb_test.go index b972c9bd3..3fc53ee92 100644 --- a/prefixdb_test.go +++ b/prefixdb_test.go @@ -1,120 +1,117 @@ -package db_test +package db import ( "testing" "github.com/stretchr/testify/require" - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/internal/dbtest" - "github.com/tendermint/tm-db/memdb" ) -func mockDBWithStuff(t *testing.T) tmdb.DB { - db := memdb.NewDB() +func mockDBWithStuff(t *testing.T) DB { + db := NewMemDB() // Under "key" prefix - require.NoError(t, db.Set([]byte("key"), []byte("value"))) - require.NoError(t, db.Set([]byte("key1"), []byte("value1"))) - require.NoError(t, db.Set([]byte("key2"), []byte("value2"))) - require.NoError(t, db.Set([]byte("key3"), []byte("value3"))) - require.NoError(t, db.Set([]byte("something"), []byte("else"))) - require.NoError(t, db.Set([]byte("k"), []byte("val"))) - require.NoError(t, db.Set([]byte("ke"), []byte("valu"))) - require.NoError(t, db.Set([]byte("kee"), []byte("valuu"))) + require.NoError(t, db.Set(bz("key"), bz("value"))) + require.NoError(t, db.Set(bz("key1"), bz("value1"))) + require.NoError(t, db.Set(bz("key2"), bz("value2"))) + require.NoError(t, db.Set(bz("key3"), bz("value3"))) + require.NoError(t, db.Set(bz("something"), bz("else"))) + require.NoError(t, db.Set(bz("k"), bz("val"))) + require.NoError(t, db.Set(bz("ke"), bz("valu"))) + require.NoError(t, db.Set(bz("kee"), bz("valuu"))) return db } func TestPrefixDBSimple(t *testing.T) { db := mockDBWithStuff(t) - pdb := tmdb.NewPrefixDB(db, []byte("key")) + pdb := NewPrefixDB(db, bz("key")) - dbtest.Value(t, pdb, []byte("key"), nil) - dbtest.Value(t, pdb, []byte("key1"), nil) - dbtest.Value(t, pdb, []byte("1"), []byte("value1")) - dbtest.Value(t, pdb, []byte("key2"), nil) - dbtest.Value(t, pdb, []byte("2"), []byte("value2")) - dbtest.Value(t, pdb, []byte("key3"), nil) - dbtest.Value(t, pdb, []byte("3"), []byte("value3")) - dbtest.Value(t, pdb, []byte("something"), nil) - dbtest.Value(t, pdb, []byte("k"), nil) - dbtest.Value(t, pdb, []byte("ke"), nil) - dbtest.Value(t, pdb, []byte("kee"), nil) + checkValue(t, pdb, bz("key"), nil) + checkValue(t, pdb, bz("key1"), nil) + checkValue(t, pdb, bz("1"), bz("value1")) + checkValue(t, pdb, bz("key2"), nil) + checkValue(t, pdb, bz("2"), bz("value2")) + checkValue(t, pdb, bz("key3"), nil) + checkValue(t, pdb, bz("3"), bz("value3")) + checkValue(t, pdb, bz("something"), nil) + checkValue(t, pdb, bz("k"), nil) + checkValue(t, pdb, bz("ke"), nil) + checkValue(t, pdb, bz("kee"), nil) } func TestPrefixDBIterator1(t *testing.T) { db := mockDBWithStuff(t) - pdb := tmdb.NewPrefixDB(db, []byte("key")) + pdb := NewPrefixDB(db, bz("key")) itr, err := pdb.Iterator(nil, nil) require.NoError(t, err) - dbtest.Domain(t, itr, nil, nil) - dbtest.Item(t, itr, []byte("1"), []byte("value1")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("2"), []byte("value2")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("3"), []byte("value3")) - dbtest.Next(t, itr, false) - dbtest.Invalid(t, itr) + checkDomain(t, itr, nil, nil) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, false) + checkInvalid(t, itr) itr.Close() } func TestPrefixDBReverseIterator1(t *testing.T) { db := mockDBWithStuff(t) - pdb := tmdb.NewPrefixDB(db, []byte("key")) + pdb := NewPrefixDB(db, bz("key")) itr, err := pdb.ReverseIterator(nil, nil) require.NoError(t, err) - dbtest.Domain(t, itr, nil, nil) - dbtest.Item(t, itr, []byte("3"), []byte("value3")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("2"), []byte("value2")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("1"), []byte("value1")) - dbtest.Next(t, itr, false) - dbtest.Invalid(t, itr) + checkDomain(t, itr, nil, nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) itr.Close() } func TestPrefixDBReverseIterator5(t *testing.T) { db := mockDBWithStuff(t) - pdb := tmdb.NewPrefixDB(db, []byte("key")) + pdb := NewPrefixDB(db, bz("key")) - itr, err := pdb.ReverseIterator([]byte("1"), nil) + itr, err := pdb.ReverseIterator(bz("1"), nil) require.NoError(t, err) - dbtest.Domain(t, itr, []byte("1"), nil) - dbtest.Item(t, itr, []byte("3"), []byte("value3")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("2"), []byte("value2")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("1"), []byte("value1")) - dbtest.Next(t, itr, false) - dbtest.Invalid(t, itr) + checkDomain(t, itr, bz("1"), nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) itr.Close() } func TestPrefixDBReverseIterator6(t *testing.T) { db := mockDBWithStuff(t) - pdb := tmdb.NewPrefixDB(db, []byte("key")) + pdb := NewPrefixDB(db, bz("key")) - itr, err := pdb.ReverseIterator([]byte("2"), nil) + itr, err := pdb.ReverseIterator(bz("2"), nil) require.NoError(t, err) - dbtest.Domain(t, itr, []byte("2"), nil) - dbtest.Item(t, itr, []byte("3"), []byte("value3")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("2"), []byte("value2")) - dbtest.Next(t, itr, false) - dbtest.Invalid(t, itr) + checkDomain(t, itr, bz("2"), nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, false) + checkInvalid(t, itr) itr.Close() } func TestPrefixDBReverseIterator7(t *testing.T) { db := mockDBWithStuff(t) - pdb := tmdb.NewPrefixDB(db, []byte("key")) + pdb := NewPrefixDB(db, bz("key")) - itr, err := pdb.ReverseIterator(nil, []byte("2")) + itr, err := pdb.ReverseIterator(nil, bz("2")) require.NoError(t, err) - dbtest.Domain(t, itr, nil, []byte("2")) - dbtest.Item(t, itr, []byte("1"), []byte("value1")) - dbtest.Next(t, itr, false) - dbtest.Invalid(t, itr) + checkDomain(t, itr, nil, bz("2")) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) itr.Close() } diff --git a/remotedb/batch.go b/remotedb/batch.go index de3d5e8e3..fe2becb14 100644 --- a/remotedb/batch.go +++ b/remotedb/batch.go @@ -1,18 +1,21 @@ package remotedb import ( + "errors" "fmt" - tmdb "github.com/tendermint/tm-db" + db "github.com/tendermint/tm-db" protodb "github.com/tendermint/tm-db/remotedb/proto" ) +var errBatchClosed = errors.New("batch has been written or closed") + type batch struct { db *RemoteDB ops []*protodb.Operation } -var _ tmdb.Batch = (*batch)(nil) +var _ db.Batch = (*batch)(nil) func newBatch(rdb *RemoteDB) *batch { return &batch{ @@ -24,7 +27,7 @@ func newBatch(rdb *RemoteDB) *batch { // Set implements Batch. func (b *batch) Set(key, value []byte) error { if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } op := &protodb.Operation{ Entity: &protodb.Entity{Key: key, Value: value}, @@ -37,7 +40,7 @@ func (b *batch) Set(key, value []byte) error { // Delete implements Batch. func (b *batch) Delete(key []byte) error { if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } op := &protodb.Operation{ Entity: &protodb.Entity{Key: key}, @@ -50,7 +53,7 @@ func (b *batch) Delete(key []byte) error { // Write implements Batch. func (b *batch) Write() error { if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } _, err := b.db.dc.BatchWrite(b.db.ctx, &protodb.Batch{Ops: b.ops}) if err != nil { @@ -64,7 +67,7 @@ func (b *batch) Write() error { // WriteSync implements Batch. func (b *batch) WriteSync() error { if b.ops == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } _, err := b.db.dc.BatchWriteSync(b.db.ctx, &protodb.Batch{Ops: b.ops}) if err != nil { diff --git a/remotedb/doc.go b/remotedb/doc.go index a1e9e2143..93d9c8a29 100644 --- a/remotedb/doc.go +++ b/remotedb/doc.go @@ -1,5 +1,5 @@ /* -remotedb is a package for connecting to distributed Tendermint tmdb.DB +remotedb is a package for connecting to distributed Tendermint db.DB instances. The purpose is to detach difficult deployments such as CLevelDB that requires gcc or perhaps for databases that require custom configurations such as extra disk space. It also eases @@ -7,7 +7,7 @@ the burden and cost of deployment of dependencies for databases to be used by Tendermint developers. Most importantly it is built over the high performant gRPC transport. -remotedb's RemoteDB implements tmdb.DB so can be used normally +remotedb's RemoteDB implements db.DB so can be used normally like other databases. One just has to explicitly connect to the remote database with a client setup such as: diff --git a/remotedb/grpcdb/server.go b/remotedb/grpcdb/server.go index 54e7d9950..3cedca8c7 100644 --- a/remotedb/grpcdb/server.go +++ b/remotedb/grpcdb/server.go @@ -10,7 +10,6 @@ import ( "google.golang.org/grpc/credentials" db "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/metadb" protodb "github.com/tendermint/tm-db/remotedb/proto" ) @@ -65,7 +64,7 @@ func (s *server) Init(ctx context.Context, in *protodb.Init) (*protodb.Entity, e defer s.mu.Unlock() var err error - s.db, err = metadb.NewDB(in.Name, metadb.BackendType(in.Type), in.Dir) + s.db, err = db.NewDB(in.Name, db.BackendType(in.Type), in.Dir) if err != nil { return nil, err } diff --git a/remotedb/iterator.go b/remotedb/iterator.go index 2fb043b47..325dc5386 100644 --- a/remotedb/iterator.go +++ b/remotedb/iterator.go @@ -1,17 +1,17 @@ package remotedb import ( - tmdb "github.com/tendermint/tm-db" + db "github.com/tendermint/tm-db" protodb "github.com/tendermint/tm-db/remotedb/proto" ) -func makeIterator(dic protodb.DB_IteratorClient) tmdb.Iterator { +func makeIterator(dic protodb.DB_IteratorClient) db.Iterator { itr := &iterator{dic: dic} itr.Next() // We need to call Next to prime the iterator return itr } -func makeReverseIterator(dric protodb.DB_ReverseIteratorClient) tmdb.Iterator { +func makeReverseIterator(dric protodb.DB_ReverseIteratorClient) db.Iterator { rItr := &reverseIterator{dric: dric} rItr.Next() // We need to call Next to prime the iterator return rItr @@ -23,7 +23,7 @@ type reverseIterator struct { err error } -var _ tmdb.Iterator = (*iterator)(nil) +var _ db.Iterator = (*iterator)(nil) // Valid implements Iterator. func (rItr *reverseIterator) Valid() bool { @@ -75,7 +75,7 @@ func (rItr *reverseIterator) assertIsValid() { } } -// iterator implements the tmdb.Iterator by retrieving +// iterator implements the db.Iterator by retrieving // streamed iterators from the remote backend as // needed. It is NOT safe for concurrent usage, // matching the behavior of other iterators. @@ -85,7 +85,7 @@ type iterator struct { err error } -var _ tmdb.Iterator = (*iterator)(nil) +var _ db.Iterator = (*iterator)(nil) // Valid implements Iterator. func (itr *iterator) Valid() bool { diff --git a/remotedb/db.go b/remotedb/remotedb.go similarity index 83% rename from remotedb/db.go rename to remotedb/remotedb.go index e79376c8d..84a57f20f 100644 --- a/remotedb/db.go +++ b/remotedb/remotedb.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - tmdb "github.com/tendermint/tm-db" + db "github.com/tendermint/tm-db" "github.com/tendermint/tm-db/remotedb/grpcdb" protodb "github.com/tendermint/tm-db/remotedb/proto" ) @@ -15,11 +15,11 @@ type RemoteDB struct { dc protodb.DBClient } -func NewDB(serverAddr string, serverKey string) (*RemoteDB, error) { - return newDB(grpcdb.NewClient(serverAddr, serverKey)) +func NewRemoteDB(serverAddr string, serverKey string) (*RemoteDB, error) { + return newRemoteDB(grpcdb.NewClient(serverAddr, serverKey)) } -func newDB(gdc protodb.DBClient, err error) (*RemoteDB, error) { +func newRemoteDB(gdc protodb.DBClient, err error) (*RemoteDB, error) { if err != nil { return nil, err } @@ -37,7 +37,7 @@ func (rd *RemoteDB) InitRemote(in *Init) error { return err } -var _ tmdb.DB = (*RemoteDB)(nil) +var _ db.DB = (*RemoteDB)(nil) // Close is a noop currently func (rd *RemoteDB) Close() error { @@ -88,7 +88,7 @@ func (rd *RemoteDB) Has(key []byte) (bool, error) { return res.Exists, nil } -func (rd *RemoteDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (rd *RemoteDB) ReverseIterator(start, end []byte) (db.Iterator, error) { dic, err := rd.dc.ReverseIterator(rd.ctx, &protodb.Entity{Start: start, End: end}) if err != nil { return nil, fmt.Errorf("RemoteDB.Iterator error: %w", err) @@ -96,11 +96,11 @@ func (rd *RemoteDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { return makeReverseIterator(dic), nil } -func (rd *RemoteDB) NewBatch() tmdb.Batch { +func (rd *RemoteDB) NewBatch() db.Batch { return newBatch(rd) } -// TODO: Implement Print when tmdb.DB implements a method +// TODO: Implement Print when db.DB implements a method // to print to a string and not db.Print to stdout. func (rd *RemoteDB) Print() error { return errors.New("remoteDB.Print: unimplemented") @@ -114,7 +114,7 @@ func (rd *RemoteDB) Stats() map[string]string { return stats.Data } -func (rd *RemoteDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (rd *RemoteDB) Iterator(start, end []byte) (db.Iterator, error) { dic, err := rd.dc.Iterator(rd.ctx, &protodb.Entity{Start: start, End: end}) if err != nil { return nil, fmt.Errorf("RemoteDB.Iterator error: %w", err) diff --git a/remotedb/db_test.go b/remotedb/remotedb_test.go similarity index 98% rename from remotedb/db_test.go rename to remotedb/remotedb_test.go index a4a159f53..8407b3004 100644 --- a/remotedb/db_test.go +++ b/remotedb/remotedb_test.go @@ -26,7 +26,7 @@ func TestRemoteDB(t *testing.T) { } }() - client, err := remotedb.NewDB(ln.Addr().String(), cert) + client, err := remotedb.NewRemoteDB(ln.Addr().String(), cert) require.Nil(t, err, "expecting a successful client creation") dbName := "test-remote-db" require.Nil(t, client.InitRemote(&remotedb.Init{Name: dbName, Type: "goleveldb"})) diff --git a/rocksdb/db.go b/rocksdb.go similarity index 81% rename from rocksdb/db.go rename to rocksdb.go index 68f303606..c3ec29f40 100644 --- a/rocksdb/db.go +++ b/rocksdb.go @@ -1,4 +1,6 @@ -package rocksdb +// +build rocksdb + +package db import ( "fmt" @@ -6,9 +8,15 @@ import ( "runtime" "github.com/tecbot/gorocksdb" - tmdb "github.com/tendermint/tm-db" ) +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewRocksDB(name, dir) + } + registerDBCreator(RocksDBBackend, dbCreator, false) +} + // RocksDB is a RocksDB backend. type RocksDB struct { db *gorocksdb.DB @@ -17,9 +25,9 @@ type RocksDB struct { woSync *gorocksdb.WriteOptions } -var _ tmdb.DB = (*RocksDB)(nil) +var _ DB = (*RocksDB)(nil) -func NewDB(name string, dir string) (*RocksDB, error) { +func NewRocksDB(name string, dir string) (*RocksDB, error) { // default rocksdb option, good enough for most cases, including heavy workloads. // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). // compression: snappy as default, need to -lsnappy to enable. @@ -33,10 +41,10 @@ func NewDB(name string, dir string) (*RocksDB, error) { opts.IncreaseParallelism(runtime.NumCPU()) // 1.5GB maximum memory use for writebuffer. opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) - return NewDBWithOptions(name, dir, opts) + return NewRocksDBWithOptions(name, dir, opts) } -func NewDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { +func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { dbPath := filepath.Join(dir, name+".db") db, err := gorocksdb.OpenDb(opts, dbPath) if err != nil { @@ -58,7 +66,7 @@ func NewDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksD // Get implements DB. func (db *RocksDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } res, err := db.db.Get(db.ro, key) if err != nil { @@ -79,10 +87,10 @@ func (db *RocksDB) Has(key []byte) (bool, error) { // Set implements DB. func (db *RocksDB) Set(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } err := db.db.Put(db.wo, key, value) if err != nil { @@ -94,10 +102,10 @@ func (db *RocksDB) Set(key []byte, value []byte) error { // SetSync implements DB. func (db *RocksDB) SetSync(key []byte, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } err := db.db.Put(db.woSync, key, value) if err != nil { @@ -109,7 +117,7 @@ func (db *RocksDB) SetSync(key []byte, value []byte) error { // Delete implements DB. func (db *RocksDB) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } err := db.db.Delete(db.wo, key) if err != nil { @@ -121,7 +129,7 @@ func (db *RocksDB) Delete(key []byte) error { // DeleteSync implements DB. func (db *RocksDB) DeleteSync(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } err := db.db.Delete(db.woSync, key) if err != nil { @@ -169,23 +177,23 @@ func (db *RocksDB) Stats() map[string]string { } // NewBatch implements DB. -func (db *RocksDB) NewBatch() tmdb.Batch { +func (db *RocksDB) NewBatch() Batch { return newRocksDBBatch(db) } // Iterator implements DB. -func (db *RocksDB) Iterator(start, end []byte) (tmdb.Iterator, error) { +func (db *RocksDB) Iterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } itr := db.db.NewIterator(db.ro) return newRocksDBIterator(itr, start, end, false), nil } // ReverseIterator implements DB. -func (db *RocksDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) { +func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, tmdb.ErrKeyEmpty + return nil, errKeyEmpty } itr := db.db.NewIterator(db.ro) return newRocksDBIterator(itr, start, end, true), nil diff --git a/rocksdb/db_test.go b/rocksdb/db_test.go deleted file mode 100644 index ef4257fb9..000000000 --- a/rocksdb/db_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package rocksdb - -// TODO: Add tests for rocksdb diff --git a/rocksdb/batch.go b/rocksdb_batch.go similarity index 79% rename from rocksdb/batch.go rename to rocksdb_batch.go index d41d72b08..a5d296b40 100644 --- a/rocksdb/batch.go +++ b/rocksdb_batch.go @@ -1,16 +1,15 @@ -package rocksdb +// +build rocksdb -import ( - "github.com/tecbot/gorocksdb" - tmdb "github.com/tendermint/tm-db" -) +package db + +import "github.com/tecbot/gorocksdb" type rocksDBBatch struct { db *RocksDB batch *gorocksdb.WriteBatch } -var _ tmdb.Batch = (*rocksDBBatch)(nil) +var _ Batch = (*rocksDBBatch)(nil) func newRocksDBBatch(db *RocksDB) *rocksDBBatch { return &rocksDBBatch{ @@ -22,13 +21,13 @@ func newRocksDBBatch(db *RocksDB) *rocksDBBatch { // Set implements Batch. func (b *rocksDBBatch) Set(key, value []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if value == nil { - return tmdb.ErrValueNil + return errValueNil } if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.batch.Put(key, value) return nil @@ -37,10 +36,10 @@ func (b *rocksDBBatch) Set(key, value []byte) error { // Delete implements Batch. func (b *rocksDBBatch) Delete(key []byte) error { if len(key) == 0 { - return tmdb.ErrKeyEmpty + return errKeyEmpty } if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } b.batch.Delete(key) return nil @@ -49,7 +48,7 @@ func (b *rocksDBBatch) Delete(key []byte) error { // Write implements Batch. func (b *rocksDBBatch) Write() error { if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } err := b.db.db.Write(b.db.wo, b.batch) if err != nil { @@ -63,7 +62,7 @@ func (b *rocksDBBatch) Write() error { // WriteSync implements Batch. func (b *rocksDBBatch) WriteSync() error { if b.batch == nil { - return tmdb.ErrBatchClosed + return errBatchClosed } err := b.db.db.Write(b.db.woSync, b.batch) if err != nil { diff --git a/rocksdb/iterator.go b/rocksdb_iterator.go similarity index 96% rename from rocksdb/iterator.go rename to rocksdb_iterator.go index 6d18cf9cf..0e7d405fe 100644 --- a/rocksdb/iterator.go +++ b/rocksdb_iterator.go @@ -1,10 +1,11 @@ -package rocksdb +// +build rocksdb + +package db import ( "bytes" "github.com/tecbot/gorocksdb" - tmdb "github.com/tendermint/tm-db" ) type rocksDBIterator struct { @@ -14,7 +15,7 @@ type rocksDBIterator struct { isInvalid bool } -var _ tmdb.Iterator = (*rocksDBIterator)(nil) +var _ Iterator = (*rocksDBIterator)(nil) func newRocksDBIterator(source *gorocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator { if isReverse { diff --git a/rocksdb_test.go b/rocksdb_test.go new file mode 100644 index 000000000..6bbe51133 --- /dev/null +++ b/rocksdb_test.go @@ -0,0 +1,35 @@ +// +build rocksdb + +package db + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRocksDBBackend(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, RocksDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + _, ok := db.(*RocksDB) + assert.True(t, ok) +} + +func TestRocksDBStats(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, RocksDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + assert.NotEmpty(t, db.Stats()) +} + +// TODO: Add tests for rocksdb diff --git a/test_helpers.go b/test_helpers.go new file mode 100644 index 000000000..6a97ad7af --- /dev/null +++ b/test_helpers.go @@ -0,0 +1,36 @@ +package db + +import "math/rand" + +const ( + strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters +) + +// For testing convenience. +func bz(s string) []byte { + return []byte(s) +} + +// Str constructs a random alphanumeric string of given length. +func randStr(length int) string { + chars := []byte{} +MAIN_LOOP: + for { + val := rand.Int63() // nolint:gosec // G404: Use of weak random number generator + for i := 0; i < 10; i++ { + v := int(val & 0x3f) // rightmost 6 bits + if v >= 62 { // only 62 characters in strChars + val >>= 6 + continue + } else { + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } + } + } + + return string(chars) +} diff --git a/types.go b/types.go index 43d0f4e06..3be412fd2 100644 --- a/types.go +++ b/types.go @@ -3,14 +3,14 @@ package db import "errors" var ( - // ErrBatchClosed is returned when a closed or written batch is used. - ErrBatchClosed = errors.New("batch has been written or closed") + // errBatchClosed is returned when a closed or written batch is used. + errBatchClosed = errors.New("batch has been written or closed") - // ErrKeyEmpty is returned when attempting to use an empty or nil key. - ErrKeyEmpty = errors.New("key cannot be empty") + // errKeyEmpty is returned when attempting to use an empty or nil key. + errKeyEmpty = errors.New("key cannot be empty") - // ErrValueNil is returned when attempting to set a nil value. - ErrValueNil = errors.New("value cannot be nil") + // errValueNil is returned when attempting to set a nil value. + errValueNil = errors.New("value cannot be nil") ) // DB is the main interface for all database backends. DBs are concurrency-safe. Callers must call diff --git a/metadb/util_test.go b/util_test.go similarity index 56% rename from metadb/util_test.go rename to util_test.go index 77b69cf22..d58cd2fd7 100644 --- a/metadb/util_test.go +++ b/util_test.go @@ -1,15 +1,11 @@ -package metadb +package db import ( "fmt" - "io/ioutil" "os" "testing" "github.com/stretchr/testify/require" - - tmdb "github.com/tendermint/tm-db" - "github.com/tendermint/tm-db/internal/dbtest" ) // Empty iterator for empty db. @@ -18,10 +14,10 @@ func TestPrefixIteratorNoMatchNil(t *testing.T) { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - itr, err := tmdb.IteratePrefix(db, []byte("2")) + itr, err := IteratePrefix(db, []byte("2")) require.NoError(t, err) - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -37,12 +33,12 @@ func TestPrefixIteratorNoMatch1(t *testing.T) { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - itr, err := tmdb.IteratePrefix(db, []byte("2")) + itr, err := IteratePrefix(db, []byte("2")) require.NoError(t, err) - err = db.SetSync([]byte("1"), []byte("value_1")) + err = db.SetSync(bz("1"), bz("value_1")) require.NoError(t, err) - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -53,12 +49,12 @@ func TestPrefixIteratorNoMatch2(t *testing.T) { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - err := db.SetSync([]byte("3"), []byte("value_3")) + err := db.SetSync(bz("3"), bz("value_3")) require.NoError(t, err) - itr, err := tmdb.IteratePrefix(db, []byte("4")) + itr, err := IteratePrefix(db, []byte("4")) require.NoError(t, err) - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -69,17 +65,17 @@ func TestPrefixIteratorMatch1(t *testing.T) { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db, dir := newTempDB(t, backend) defer os.RemoveAll(dir) - err := db.SetSync([]byte("2"), []byte("value_2")) + err := db.SetSync(bz("2"), bz("value_2")) require.NoError(t, err) - itr, err := tmdb.IteratePrefix(db, []byte("2")) + itr, err := IteratePrefix(db, bz("2")) require.NoError(t, err) - dbtest.Valid(t, itr, true) - dbtest.Item(t, itr, []byte("2"), []byte("value_2")) - dbtest.Next(t, itr, false) + checkValid(t, itr, true) + checkItem(t, itr, bz("2"), bz("value_2")) + checkNext(t, itr, false) // Once invalid... - dbtest.Invalid(t, itr) + checkInvalid(t, itr) }) } } @@ -92,41 +88,33 @@ func TestPrefixIteratorMatches1N(t *testing.T) { defer os.RemoveAll(dir) // prefixed - err := db.SetSync([]byte("a/1"), []byte("value_1")) + err := db.SetSync(bz("a/1"), bz("value_1")) require.NoError(t, err) - err = db.SetSync([]byte("a/3"), []byte("value_3")) + err = db.SetSync(bz("a/3"), bz("value_3")) require.NoError(t, err) // not - err = db.SetSync([]byte("b/3"), []byte("value_3")) + err = db.SetSync(bz("b/3"), bz("value_3")) require.NoError(t, err) - err = db.SetSync([]byte("a-3"), []byte("value_3")) + err = db.SetSync(bz("a-3"), bz("value_3")) require.NoError(t, err) - err = db.SetSync([]byte("a.3"), []byte("value_3")) + err = db.SetSync(bz("a.3"), bz("value_3")) require.NoError(t, err) - err = db.SetSync([]byte("abcdefg"), []byte("value_3")) + err = db.SetSync(bz("abcdefg"), bz("value_3")) require.NoError(t, err) - itr, err := tmdb.IteratePrefix(db, []byte("a/")) + itr, err := IteratePrefix(db, bz("a/")) require.NoError(t, err) - dbtest.Valid(t, itr, true) - dbtest.Item(t, itr, []byte("a/1"), []byte("value_1")) - dbtest.Next(t, itr, true) - dbtest.Item(t, itr, []byte("a/3"), []byte("value_3")) + checkValid(t, itr, true) + checkItem(t, itr, bz("a/1"), bz("value_1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("a/3"), bz("value_3")) // Bad! - dbtest.Next(t, itr, false) + checkNext(t, itr, false) - // Once invalid... - dbtest.Invalid(t, itr) + //Once invalid... + checkInvalid(t, itr) }) } } - -func newTempDB(t *testing.T, backend BackendType) (db tmdb.DB, dbDir string) { - dirname, err := ioutil.TempDir("", "db_common_test") - require.NoError(t, err) - db, err = NewDB("testdb", backend, dirname) - require.NoError(t, err) - return db, dirname -} From 8b2ecdc4e54840c6328a4a5542226e6fea26e176 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 8 Nov 2021 15:34:38 -0800 Subject: [PATCH 2/6] Revert "changelog: add entry for 0.6.5 (#178)" This reverts commit 2ce1ac75cecda0c429f924b1183ebf33f26f552f. --- CHANGELOG.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 165d348dd..7f667baa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,5 @@ # Changelog -## 0.6.5 - -**2021-08-04** - -### Breaking changes (Go API) - -- [metadb] [\#151](https://github.com/tendermint/tm-db/pull/151) split backends into packages and add metadb (@marbar3778, @mvdan) - -### Version bumps - -- Bump dgraph/badger to v2. -- Bump go.etcd.io/bbolt from 1.3.5 to 1.3.6. -- Bump grpc from 1.35.0 to 1.38.0. - ## 0.6.4 **2021-02-09** From eafa5226c282654ddac9bdb215d3f31fb4ede003 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 8 Nov 2021 15:20:09 -0800 Subject: [PATCH 3/6] Prepare CHANGELOG.md for release v0.6.6. --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f667baa6..805bd4d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## 0.6.6 + +**2021-11-08** + +**Important note:** Version v0.6.5 was accidentally tagged and should be +avoided. This version is identical to v0.6.4 in package structure and API, but +has updated the version marker so that normal `go get` upgrades will not +require modifying existing use of v0.6.4. + +### Version bumps (since v0.6.4) + +- Bump grpc from to 1.42.0. +- Bump dgraph/badger to v2 2.2007.3. +- Bump go.etcd.io/bbolt to 1.3.6. + +## 0.6.5 + +**2021-08-04** + +**Important note**: This version was tagged by accident, and should not be +used. The tag now points to the [package-reorg +branch](https://github.com/tendermint/tm-db/tree/package-reorg) so that +any existing dependencies should not break. + +>>>>>>> 0c0f56e (Prepare CHANGELOG.md for release v0.6.6.) ## 0.6.4 **2021-02-09** From 662689570ce2f1747dba280d1b22f14c9ea2f6b2 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 8 Nov 2021 15:46:47 -0800 Subject: [PATCH 4/6] Restore lint container setting. --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fb3f7bb71..c291823ed 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,7 @@ jobs: # We need to run the linter on the same image we use for building, since it # needs the C libraries installed for the dependencies to typecheck. runs-on: ubuntu-latest + container: tendermintdev/docker-tm-db-testing steps: - uses: actions/checkout@v2 - uses: golangci/golangci-lint-action@v2.5.2 From ba884f0dd6463858b7f764b0dedee53ae973751c Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 8 Nov 2021 15:50:24 -0800 Subject: [PATCH 5/6] Fix merge markers. --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 805bd4d26..4b76daa7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,6 @@ used. The tag now points to the [package-reorg branch](https://github.com/tendermint/tm-db/tree/package-reorg) so that any existing dependencies should not break. ->>>>>>> 0c0f56e (Prepare CHANGELOG.md for release v0.6.6.) ## 0.6.4 **2021-02-09** From 36e6d83df12b03c23c561302f4a857ac0e8600c3 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Sun, 10 Oct 2021 06:03:34 -0500 Subject: [PATCH 6/6] Add an option to create MemDB iterators with no Mutex for the CacheKVStore (#187) * Add an option to create MemDB iterators with no Mutex for the CacheKVStore * Update changelog --- .gitignore | 3 ++- memdb.go | 16 ++++++++++++++++ memdb_iterator.go | 14 ++++++++++++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index b3efc3918..ce205f69a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -.idea \ No newline at end of file +.idea +vendor/* diff --git a/memdb.go b/memdb.go index 604e6a06e..d019af230 100644 --- a/memdb.go +++ b/memdb.go @@ -190,3 +190,19 @@ func (db *MemDB) ReverseIterator(start, end []byte) (Iterator, error) { } return newMemDBIterator(db, start, end, true), nil } + +// IteratorNoMtx makes an iterator with no mutex. +func (db *MemDB) IteratorNoMtx(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIteratorMtxChoice(db, start, end, false, false), nil +} + +// ReverseIteratorNoMtx makes an iterator with no mutex. +func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIteratorMtxChoice(db, start, end, true, false), nil +} diff --git a/memdb_iterator.go b/memdb_iterator.go index 2a61e3757..ebd104f67 100644 --- a/memdb_iterator.go +++ b/memdb_iterator.go @@ -21,12 +21,17 @@ type memDBIterator struct { item *item start []byte end []byte + useMtx bool } var _ Iterator = (*memDBIterator)(nil) // newMemDBIterator creates a new memDBIterator. func newMemDBIterator(db *MemDB, start []byte, end []byte, reverse bool) *memDBIterator { + return newMemDBIteratorMtxChoice(db, start, end, reverse, true) +} + +func newMemDBIteratorMtxChoice(db *MemDB, start []byte, end []byte, reverse bool, useMtx bool) *memDBIterator { ctx, cancel := context.WithCancel(context.Background()) ch := make(chan *item, chBufferSize) iter := &memDBIterator{ @@ -34,11 +39,16 @@ func newMemDBIterator(db *MemDB, start []byte, end []byte, reverse bool) *memDBI cancel: cancel, start: start, end: end, + useMtx: useMtx, } - db.mtx.RLock() + if useMtx { + db.mtx.RLock() + } go func() { - defer db.mtx.RUnlock() + if useMtx { + defer db.mtx.RUnlock() + } // Because we use [start, end) for reverse ranges, while btree uses (start, end], we need // the following variables to handle some reverse iteration conditions ourselves. var (