Skip to content

Commit

Permalink
Reinstate BoltDB and ClevelDB as backend DBs (#177)
Browse files Browse the repository at this point in the history
* Revert "remove deprecated boltdb and cleveldb (#155)"

This reverts commit badc0b8.

We decided to reinstate boltDB and clevelDB and mark them as deprecated until a
future version of CometBFT in which we'll drop cometbft-db and support only 1 backend
DB.

* updated cleveldb Iterator API docs to conform to the changes made in #168

* updated go.mod

* updated boltDB Iterator APIs to conform to the changes made in #168

* added changelog entry

* Formatting

Co-authored-by: Daniel <[email protected]>

* formatting to please the linter

* added additional context in the docs of backend types constants

---------

Co-authored-by: Daniel <[email protected]>
  • Loading branch information
alesforz and cason authored Aug 9, 2024
1 parent a79d349 commit 9db1a44
Show file tree
Hide file tree
Showing 15 changed files with 1,054 additions and 3 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
- reinstate BoltDB and ClevelDB as backend DBs
([\#177](https://github.com/cometbft/cometbft-db/pull/177))
14 changes: 12 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,21 @@ test:
@go test $(PACKAGES) -v
.PHONY: test

test-cleveldb:
@echo "--> Running go test"
@go test $(PACKAGES) -tags cleveldb -v
.PHONY: test-cleveldb

test-rocksdb:
@echo "--> Running go test"
@go test $(PACKAGES) -tags rocksdb -v
.PHONY: test-rocksdb

test-boltdb:
@echo "--> Running go test"
@go test $(PACKAGES) -tags boltdb -v
.PHONY: test-boltdb

test-badgerdb:
@echo "--> Running go test"
@go test $(PACKAGES) -tags badgerdb -v
Expand All @@ -35,7 +45,7 @@ test-pebble:

test-all:
@echo "--> Running go test"
@go test $(PACKAGES) -tags rocksdb,grocksdb_clean_link,badgerdb,pebbledb -v
@go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb -v
.PHONY: test-all

test-all-with-coverage:
Expand All @@ -46,7 +56,7 @@ test-all-with-coverage:
-race \
-coverprofile=coverage.txt \
-covermode=atomic \
-tags=memdb,goleveldb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb \
-tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb \
-v
.PHONY: test-all-with-coverage

Expand Down
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,19 @@ Go 1.22+
sets, and tests. Used for [IAVL](https://github.com/tendermint/iavl) working
sets when the pruning strategy allows it.

- **[LevelDB](https://github.com/google/leveldb) [DEPRECATED]:** A [Go
wrapper](https://github.com/jmhodges/levigo) around
[LevelDB](https://github.com/google/leveldb). Uses LSM-trees for on-disk
storage, which have good performance for write-heavy workloads, particularly
on spinning disks, but requires periodic compaction to maintain decent read
performance and reclaim disk space. Does not support transactions.

- **[BoltDB](https://github.com/etcd-io/bbolt) [DEPRECATED]:** A
[fork](https://github.com/etcd-io/bbolt) of
[BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage,
which have good performance for read-heavy workloads and range scans. Supports
serializable ACID transactions.

- **[RocksDB](https://github.com/linxGnu/grocksdb) [experimental]:** A [Go
wrapper](https://github.com/linxGnu/grocksdb) around
[RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees
Expand Down
213 changes: 213 additions & 0 deletions boltdb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
//go:build boltdb
// +build boltdb

package db

import (
"errors"
"fmt"
"os"
"path/filepath"

"go.etcd.io/bbolt"
)

var bucket = []byte("tm")

func init() {
registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) {
return NewBoltDB(name, dir)
})
}

// BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt).
//
// NOTE: All operations (including Set, Delete) are synchronous by default. One
// can globally turn it off by using NoSync config option (not recommended).
//
// A single bucket ([]byte("tm")) is used per a database instance. This could
// lead to performance issues when/if there will be lots of keys.
type BoltDB struct {
db *bbolt.DB
}

var _ DB = (*BoltDB)(nil)

// NewBoltDB returns a BoltDB with default options.
//
// Deprecated: boltdb is deprecated and will be removed in the future.
func NewBoltDB(name, dir string) (DB, error) {
return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions)
}

// NewBoltDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not
// supported because NewBoltDBWithOpts creates a global bucket.
func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error) {
if opts.ReadOnly {
return nil, errors.New("ReadOnly: true is not supported")
}

dbPath := filepath.Join(dir, name+".db")
db, err := bbolt.Open(dbPath, os.ModePerm, opts)
if err != nil {
return nil, err
}

// create a global bucket
err = db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucket)
return err
})
if err != nil {
return nil, err
}

return &BoltDB{db: db}, nil
}

// Get implements DB.
func (bdb *BoltDB) Get(key []byte) (value []byte, err error) {
if len(key) == 0 {
return nil, errKeyEmpty
}
err = bdb.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(bucket)
if v := b.Get(key); v != nil {
value = append([]byte{}, v...)
}
return nil
})
if err != nil {
return nil, err
}
return
}

// Has implements DB.
func (bdb *BoltDB) Has(key []byte) (bool, error) {
bytes, err := bdb.Get(key)
if err != nil {
return false, err
}
return bytes != nil, nil
}

// Set implements DB.
func (bdb *BoltDB) Set(key, value []byte) error {
if len(key) == 0 {
return errKeyEmpty
}
if value == nil {
return errValueNil
}
err := bdb.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(bucket)
return b.Put(key, value)
})
if err != nil {
return err
}
return nil
}

// SetSync implements DB.
func (bdb *BoltDB) SetSync(key, value []byte) error {
return bdb.Set(key, value)
}

// Delete implements DB.
func (bdb *BoltDB) Delete(key []byte) error {
if len(key) == 0 {
return errKeyEmpty
}
err := bdb.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket(bucket).Delete(key)
})
if err != nil {
return err
}
return nil
}

// DeleteSync implements DB.
func (bdb *BoltDB) DeleteSync(key []byte) error {
return bdb.Delete(key)
}

// Close implements DB.
func (bdb *BoltDB) Close() error {
return bdb.db.Close()
}

// Print implements DB.
func (bdb *BoltDB) Print() error {
stats := bdb.db.Stats()
fmt.Printf("%v\n", stats)

err := bdb.db.View(func(tx *bbolt.Tx) error {
tx.Bucket(bucket).ForEach(func(k, v []byte) error {
fmt.Printf("[%X]:\t[%X]\n", k, v)
return nil
})
return nil
})
if err != nil {
return err
}
return nil
}

// Stats implements DB.
func (bdb *BoltDB) Stats() map[string]string {
stats := bdb.db.Stats()
m := make(map[string]string)

// Freelist stats
m["FreePageN"] = fmt.Sprintf("%v", stats.FreePageN)
m["PendingPageN"] = fmt.Sprintf("%v", stats.PendingPageN)
m["FreeAlloc"] = fmt.Sprintf("%v", stats.FreeAlloc)
m["FreelistInuse"] = fmt.Sprintf("%v", stats.FreelistInuse)

// Transaction stats
m["TxN"] = fmt.Sprintf("%v", stats.TxN)
m["OpenTxN"] = fmt.Sprintf("%v", stats.OpenTxN)

return m
}

// NewBatch implements DB.
func (bdb *BoltDB) NewBatch() Batch {
return newBoltDBBatch(bdb)
}

// WARNING: Any concurrent writes or reads will block until the iterator is
// closed.
func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) {
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
return nil, errKeyEmpty
}
tx, err := bdb.db.Begin(false)
if err != nil {
return nil, err
}
return newBoltDBIterator(tx, start, end, false), nil
}

// WARNING: Any concurrent writes or reads will block until the iterator is
// closed.
func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) {
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
return nil, errKeyEmpty
}
tx, err := bdb.db.Begin(false)
if err != nil {
return nil, err
}
return newBoltDBIterator(tx, start, end, true), nil
}

func (bdb *BoltDB) Compact(start, end []byte) error {
// There is no explicit CompactRange support in BoltDB, only a function that copies the
// entire DB from one place to another while doing deletions. Hence we do not support it.
return nil
}
87 changes: 87 additions & 0 deletions boltdb_batch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
//go:build boltdb
// +build boltdb

package db

import "go.etcd.io/bbolt"

// boltDBBatch stores operations internally and dumps them to BoltDB on Write().
type boltDBBatch struct {
db *BoltDB
ops []operation
}

var _ Batch = (*boltDBBatch)(nil)

func newBoltDBBatch(db *BoltDB) *boltDBBatch {
return &boltDBBatch{
db: db,
ops: []operation{},
}
}

// Set implements Batch.
func (b *boltDBBatch) Set(key, value []byte) error {
if len(key) == 0 {
return errKeyEmpty
}
if value == nil {
return errValueNil
}
if b.ops == nil {
return errBatchClosed
}
b.ops = append(b.ops, operation{opTypeSet, key, value})
return nil
}

// Delete implements Batch.
func (b *boltDBBatch) Delete(key []byte) error {
if len(key) == 0 {
return errKeyEmpty
}
if b.ops == nil {
return errBatchClosed
}
b.ops = append(b.ops, operation{opTypeDelete, key, nil})
return nil
}

// Write implements Batch.
func (b *boltDBBatch) Write() error {
if b.ops == nil {
return errBatchClosed
}
err := b.db.db.Batch(func(tx *bbolt.Tx) error {
bkt := tx.Bucket(bucket)
for _, op := range b.ops {
switch op.opType {
case opTypeSet:
if err := bkt.Put(op.key, op.value); err != nil {
return err
}
case opTypeDelete:
if err := bkt.Delete(op.key); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
// Make sure batch cannot be used afterwards. Callers should still call Close(), for errors.
return b.Close()
}

// WriteSync implements Batch.
func (b *boltDBBatch) WriteSync() error {
return b.Write()
}

// Close implements Batch.
func (b *boltDBBatch) Close() error {
b.ops = nil
return nil
}
Loading

0 comments on commit 9db1a44

Please sign in to comment.