metadb: split backends into packages, add metadb (#151)

authored-by: Daniel Martí <[email protected]>

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: test & coverage report creation
         run: |
-          go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic -tags cleveldb,boltdb,rocksdb,badgerdb -v
+          go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,badgerdb -v
       - uses: codecov/codecov-action@v1
         with:
           file: ./coverage.txt

.github/workflows/lint.yml

@@ -8,6 +8,7 @@ on:
 jobs:
   golangci:
     runs-on: ubuntu-latest
+    container: tendermintdev/docker-tm-db-testing
     steps:
       - uses: actions/checkout@v2
       - uses: golangci/[email protected]

badger_db.go → badgerdb/db.go

@@ -1,6 +1,4 @@
-// +build badgerdb
-
-package db
+package badgerdb
 
 import (
 	"bytes"
@@ -9,17 +7,12 @@ import (
 	"path/filepath"
 
 	"github.com/dgraph-io/badger/v2"
+	tmdb "github.com/tendermint/tm-db"
 )
 
-func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) }
-
-func badgerDBCreator(dbName, dir string) (DB, error) {
-	return NewBadgerDB(dbName, dir)
-}
-
-// NewBadgerDB creates a Badger key-value store backed to the
+// NewDB creates a Badger key-value store backed to the
 // directory dir supplied. If dir does not exist, it will be created.
-func NewBadgerDB(dbName, dir string) (*BadgerDB, error) {
+func NewDB(dbName, dir string) (*BadgerDB, error) {
 	// Since Badger doesn't support database names, we join both to obtain
 	// the final directory to use for the database.
 	path := filepath.Join(dir, dbName)
@@ -30,13 +23,13 @@ func NewBadgerDB(dbName, dir string) (*BadgerDB, error) {
 	opts := badger.DefaultOptions(path)
 	opts.SyncWrites = false // note that we have Sync methods
 	opts.Logger = nil       // badger is too chatty by default
-	return NewBadgerDBWithOptions(opts)
+	return NewDBWithOptions(opts)
 }
 
-// NewBadgerDBWithOptions creates a BadgerDB key value store
+// NewDBWithOptions creates a BadgerDB key value store
 // gives the flexibility of initializing a database with the
 // respective options.
-func NewBadgerDBWithOptions(opts badger.Options) (*BadgerDB, error) {
+func NewDBWithOptions(opts badger.Options) (*BadgerDB, error) {
 	db, err := badger.Open(opts)
 	if err != nil {
 		return nil, err
@@ -48,11 +41,11 @@ type BadgerDB struct {
 	db *badger.DB
 }
 
-var _ DB = (*BadgerDB)(nil)
+var _ tmdb.DB = (*BadgerDB)(nil)
 
 func (b *BadgerDB) Get(key []byte) ([]byte, error) {
 	if len(key) == 0 {
-		return nil, errKeyEmpty
+		return nil, tmdb.ErrKeyEmpty
 	}
 	var val []byte
 	err := b.db.View(func(txn *badger.Txn) error {
@@ -73,7 +66,7 @@ func (b *BadgerDB) Get(key []byte) ([]byte, error) {
 
 func (b *BadgerDB) Has(key []byte) (bool, error) {
 	if len(key) == 0 {
-		return false, errKeyEmpty
+		return false, tmdb.ErrKeyEmpty
 	}
 	var found bool
 	err := b.db.View(func(txn *badger.Txn) error {
@@ -89,10 +82,10 @@ func (b *BadgerDB) Has(key []byte) (bool, error) {
 
 func (b *BadgerDB) Set(key, value []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	if value == nil {
-		return errValueNil
+		return tmdb.ErrValueNil
 	}
 	return b.db.Update(func(txn *badger.Txn) error {
 		return txn.Set(key, value)
@@ -112,7 +105,7 @@ func (b *BadgerDB) SetSync(key, value []byte) error {
 
 func (b *BadgerDB) Delete(key []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	return b.db.Update(func(txn *badger.Txn) error {
 		return txn.Delete(key)
@@ -133,7 +126,7 @@ func (b *BadgerDB) Print() error {
 
 func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions) (*badgerDBIterator, error) {
 	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
-		return nil, errKeyEmpty
+		return nil, tmdb.ErrKeyEmpty
 	}
 	txn := b.db.NewTransaction(false)
 	iter := txn.NewIterator(opts)
@@ -154,12 +147,12 @@ func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions)
 	}, nil
 }
 
-func (b *BadgerDB) Iterator(start, end []byte) (Iterator, error) {
+func (b *BadgerDB) Iterator(start, end []byte) (tmdb.Iterator, error) {
 	opts := badger.DefaultIteratorOptions
 	return b.iteratorOpts(start, end, opts)
 }
 
-func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) {
+func (b *BadgerDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) {
 	opts := badger.DefaultIteratorOptions
 	opts.Reverse = true
 	return b.iteratorOpts(end, start, opts)
@@ -169,7 +162,7 @@ func (b *BadgerDB) Stats() map[string]string {
 	return nil
 }
 
-func (b *BadgerDB) NewBatch() Batch {
+func (b *BadgerDB) NewBatch() tmdb.Batch {
 	wb := &badgerDBBatch{
 		db:         b.db,
 		wb:         b.db.NewWriteBatch(),
@@ -179,7 +172,7 @@ func (b *BadgerDB) NewBatch() Batch {
 	return wb
 }
 
-var _ Batch = (*badgerDBBatch)(nil)
+var _ tmdb.Batch = (*badgerDBBatch)(nil)
 
 type badgerDBBatch struct {
 	db *badger.DB
@@ -196,17 +189,17 @@ type badgerDBBatch struct {
 
 func (b *badgerDBBatch) Set(key, value []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	if value == nil {
-		return errValueNil
+		return tmdb.ErrValueNil
 	}
 	return b.wb.Set(key, value)
 }
 
 func (b *badgerDBBatch) Delete(key []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	return b.wb.Delete(key)
 }

boltdb_batch.go → boltdb/batch.go

@@ -1,16 +1,30 @@
-// +build boltdb
+package boltdb
 
-package db
+import (
+	tmdb "github.com/tendermint/tm-db"
+	"go.etcd.io/bbolt"
+)
 
-import "go.etcd.io/bbolt"
+type opType int
+
+const (
+	opTypeSet opType = iota + 1
+	opTypeDelete
+)
+
+type operation struct {
+	opType
+	key   []byte
+	value []byte
+}
 
 // boltDBBatch stores operations internally and dumps them to BoltDB on Write().
 type boltDBBatch struct {
 	db  *BoltDB
 	ops []operation
 }
 
-var _ Batch = (*boltDBBatch)(nil)
+var _ tmdb.Batch = (*boltDBBatch)(nil)
 
 func newBoltDBBatch(db *BoltDB) *boltDBBatch {
 	return &boltDBBatch{
@@ -22,13 +36,13 @@ func newBoltDBBatch(db *BoltDB) *boltDBBatch {
 // Set implements Batch.
 func (b *boltDBBatch) Set(key, value []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	if value == nil {
-		return errValueNil
+		return tmdb.ErrValueNil
 	}
 	if b.ops == nil {
-		return errBatchClosed
+		return tmdb.ErrBatchClosed
 	}
 	b.ops = append(b.ops, operation{opTypeSet, key, value})
 	return nil
@@ -37,10 +51,10 @@ func (b *boltDBBatch) Set(key, value []byte) error {
 // Delete implements Batch.
 func (b *boltDBBatch) Delete(key []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	if b.ops == nil {
-		return errBatchClosed
+		return tmdb.ErrBatchClosed
 	}
 	b.ops = append(b.ops, operation{opTypeDelete, key, nil})
 	return nil
@@ -49,7 +63,7 @@ func (b *boltDBBatch) Delete(key []byte) error {
 // Write implements Batch.
 func (b *boltDBBatch) Write() error {
 	if b.ops == nil {
-		return errBatchClosed
+		return tmdb.ErrBatchClosed
 	}
 	err := b.db.db.Batch(func(tx *bbolt.Tx) error {
 		bkt := tx.Bucket(bucket)

boltdb.go → boltdb/db.go

@@ -1,25 +1,16 @@
-// +build boltdb
-
-package db
+package boltdb
 
 import (
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	tmdb "github.com/tendermint/tm-db"
 	"go.etcd.io/bbolt"
 )
 
-var (
-	bucket = []byte("tm")
-)
-
-func init() {
-	registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) {
-		return NewBoltDB(name, dir)
-	}, false)
-}
+var bucket = []byte("tm")
 
 // BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt).
 //
@@ -32,16 +23,16 @@ type BoltDB struct {
 	db *bbolt.DB
 }
 
-var _ DB = (*BoltDB)(nil)
+var _ tmdb.DB = (*BoltDB)(nil)
 
-// NewBoltDB returns a BoltDB with default options.
-func NewBoltDB(name, dir string) (DB, error) {
-	return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions)
+// NewDB returns a BoltDB with default options.
+func NewDB(name, dir string) (tmdb.DB, error) {
+	return NewDBWithOpts(name, dir, bbolt.DefaultOptions)
 }
 
-// NewBoltDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not
-// supported because NewBoltDBWithOpts creates a global bucket.
-func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error) {
+// NewDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not
+// supported because NewDBWithOpts creates a global bucket.
+func NewDBWithOpts(name string, dir string, opts *bbolt.Options) (tmdb.DB, error) {
 	if opts.ReadOnly {
 		return nil, errors.New("ReadOnly: true is not supported")
 	}
@@ -67,7 +58,7 @@ func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error)
 // Get implements DB.
 func (bdb *BoltDB) Get(key []byte) (value []byte, err error) {
 	if len(key) == 0 {
-		return nil, errKeyEmpty
+		return nil, tmdb.ErrKeyEmpty
 	}
 	err = bdb.db.View(func(tx *bbolt.Tx) error {
 		b := tx.Bucket(bucket)
@@ -94,10 +85,10 @@ func (bdb *BoltDB) Has(key []byte) (bool, error) {
 // Set implements DB.
 func (bdb *BoltDB) Set(key, value []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	if value == nil {
-		return errValueNil
+		return tmdb.ErrValueNil
 	}
 	err := bdb.db.Update(func(tx *bbolt.Tx) error {
 		b := tx.Bucket(bucket)
@@ -117,7 +108,7 @@ func (bdb *BoltDB) SetSync(key, value []byte) error {
 // Delete implements DB.
 func (bdb *BoltDB) Delete(key []byte) error {
 	if len(key) == 0 {
-		return errKeyEmpty
+		return tmdb.ErrKeyEmpty
 	}
 	err := bdb.db.Update(func(tx *bbolt.Tx) error {
 		return tx.Bucket(bucket).Delete(key)
@@ -144,10 +135,12 @@ func (bdb *BoltDB) Print() error {
 	fmt.Printf("%v\n", stats)
 
 	err := bdb.db.View(func(tx *bbolt.Tx) error {
-		tx.Bucket(bucket).ForEach(func(k, v []byte) error {
+		if err := tx.Bucket(bucket).ForEach(func(k, v []byte) error {
 			fmt.Printf("[%X]:\t[%X]\n", k, v)
 			return nil
-		})
+		}); err != nil {
+			return err
+		}
 		return nil
 	})
 	if err != nil {
@@ -175,15 +168,15 @@ func (bdb *BoltDB) Stats() map[string]string {
 }
 
 // NewBatch implements DB.
-func (bdb *BoltDB) NewBatch() Batch {
+func (bdb *BoltDB) NewBatch() tmdb.Batch {
 	return newBoltDBBatch(bdb)
 }
 
 // WARNING: Any concurrent writes or reads will block until the iterator is
 // closed.
-func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) {
+func (bdb *BoltDB) Iterator(start, end []byte) (tmdb.Iterator, error) {
 	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
-		return nil, errKeyEmpty
+		return nil, tmdb.ErrKeyEmpty
 	}
 	tx, err := bdb.db.Begin(false)
 	if err != nil {
@@ -194,9 +187,9 @@ func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) {
 
 // WARNING: Any concurrent writes or reads will block until the iterator is
 // closed.
-func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) {
+func (bdb *BoltDB) ReverseIterator(start, end []byte) (tmdb.Iterator, error) {
 	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
-		return nil, errKeyEmpty
+		return nil, tmdb.ErrKeyEmpty
 	}
 	tx, err := bdb.db.Begin(false)
 	if err != nil {