From 4521959c1a4a3e44354ccd081850c1bd182c908b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 25 Jan 2024 19:58:59 +0800 Subject: [PATCH] chore(pebbledb): remove isForceSync flag (#125) * chore(pebbledb): remove isForceSync flag Downstream dependencies should use `SetSync`/`DeleteSync` to force flushing OS buffers to disk. * gofumpt the code * update readme * add note --- .../breaking-changes/118-remove-remotedb.md | 3 +- .changelog/unreleased/chore/114.md | 1 - .../unreleased/features/112-pebbledb.md | 2 +- .golangci.yml | 2 +- README.md | 14 +++-- badger_db.go | 2 +- boltdb.go | 4 +- cleveldb_iterator.go | 7 +-- codecov.yml | 3 - pebble.go | 56 ------------------- pebble_test.go | 10 ---- rocksdb_iterator.go | 7 +-- 12 files changed, 20 insertions(+), 91 deletions(-) delete mode 100644 .changelog/unreleased/chore/114.md diff --git a/.changelog/unreleased/breaking-changes/118-remove-remotedb.md b/.changelog/unreleased/breaking-changes/118-remove-remotedb.md index 45cfd13..044fb42 100644 --- a/.changelog/unreleased/breaking-changes/118-remove-remotedb.md +++ b/.changelog/unreleased/breaking-changes/118-remove-remotedb.md @@ -1,2 +1 @@ -- Removed supprot for RemoteDB ([\#118](https://github.com/cometbft/cometbft- - db/issues/118)) +- Remove remotedb ([\#121](https://github.com/cometbft/cometbft-db/pull/121)) diff --git a/.changelog/unreleased/chore/114.md b/.changelog/unreleased/chore/114.md deleted file mode 100644 index 738cedc..0000000 --- a/.changelog/unreleased/chore/114.md +++ /dev/null @@ -1 +0,0 @@ -chore: lint code diff --git a/.changelog/unreleased/features/112-pebbledb.md b/.changelog/unreleased/features/112-pebbledb.md index 52d3067..27c5d50 100644 --- a/.changelog/unreleased/features/112-pebbledb.md +++ b/.changelog/unreleased/features/112-pebbledb.md @@ -1 +1 @@ -- Added support for pebbledb [v1.0.0](https://github.com/cockroachdb/pebble/releases/tag/v1.0.0) ([\#112](https://github.com/cometbft/cometbft-db/pull/112)) \ No newline at end of file +- Add [pebbledb](https://github.com/cockroachdb/pebble) ([\#112](https://github.com/cometbft/cometbft-db/pull/112)) diff --git a/.golangci.yml b/.golangci.yml index 7de2bd3..5c8c5b3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -112,4 +112,4 @@ linters-settings: - 'fmt.Printf' - 'fmt.Print' - 'fmt.Println' - - 'myFunction' \ No newline at end of file + - 'myFunction' diff --git a/README.md b/README.md index ab10a7b..5aaed40 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,15 @@ Go 1.21+ performance, and includes advanced features such as serializable ACID transactions, write batches, compression, and more. +- **[PebbleDB](https://github.com/cockroachdb/pebble) [experimental]:** Pebble + is a LevelDB/RocksDB inspired key-value store focused on performance and + internal usage by CockroachDB. Pebble inherits the RocksDB file formats and a + few extensions such as range deletion tombstones, table-level bloom filters, + and updates to the MANIFEST format. + + XXX: there are reports of broken upgrade process when using [Cosmos + SDK](https://github.com/cosmos/cosmos-sdk). + ## Meta-databases - **PrefixDB [stable]:** A database which wraps another database and uses a @@ -66,11 +75,6 @@ Go 1.21+ the Cosmos SDK to give different modules their own namespaced database in a single application database. -- **RemoteDB [experimental]:** A database that connects to distributed - CometBFT db instances via [gRPC](https://grpc.io/). This can help with - detaching difficult deployments such as LevelDB, and can also ease dependency - management for CometBFT developers. - ## Tests To test common databases, run `make test`. If all databases are available on the diff --git a/badger_db.go b/badger_db.go index 08f201f..77a8be2 100644 --- a/badger_db.go +++ b/badger_db.go @@ -25,7 +25,7 @@ func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { // the final directory to use for the database. path := filepath.Join(dir, dbName) - if err := os.MkdirAll(path, 0755); err != nil { + if err := os.MkdirAll(path, 0o755); err != nil { return nil, err } opts := badger.DefaultOptions(path) diff --git a/boltdb.go b/boltdb.go index e452105..1545be9 100644 --- a/boltdb.go +++ b/boltdb.go @@ -12,9 +12,7 @@ import ( "go.etcd.io/bbolt" ) -var ( - bucket = []byte("tm") -) +var bucket = []byte("tm") func init() { registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) { diff --git a/cleveldb_iterator.go b/cleveldb_iterator.go index e56c35e..cf5ac19 100644 --- a/cleveldb_iterator.go +++ b/cleveldb_iterator.go @@ -57,7 +57,6 @@ func (itr cLevelDBIterator) Domain() ([]byte, []byte) { // Valid implements Iterator. func (itr cLevelDBIterator) Valid() bool { - // Once invalid, forever invalid. if itr.isInvalid { return false @@ -76,9 +75,9 @@ func (itr cLevelDBIterator) Valid() bool { } // If key is end or past it, invalid. - var start = itr.start - var end = itr.end - var key = itr.source.Key() + start := itr.start + end := itr.end + key := itr.source.Key() if itr.isReverse { if start != nil && bytes.Compare(key, start) < 0 { itr.isInvalid = true diff --git a/codecov.yml b/codecov.yml index d9813f8..b96739d 100644 --- a/codecov.yml +++ b/codecov.yml @@ -21,6 +21,3 @@ comment: layout: "reach, diff, files, tree" behavior: default # update if exists else create new require_changes: true - -ignore: - - "remotedb/proto" diff --git a/pebble.go b/pebble.go index b19b7df..08ed5aa 100644 --- a/pebble.go +++ b/pebble.go @@ -11,57 +11,11 @@ import ( "github.com/cockroachdb/pebble" ) -// ForceSync is a a compile time flag to force using Sync for NoSync functions (Set, Delete, Write). - -/* -This is set at compile time. Could be 0 or 1, defaults is 1. -It forces using Sync for NoSync functions (Set, Delete, Write) - -Notice if ForceSync=0: performance will be better. However, there is an issue when upgrading. -And the workaround (if using ForceSync=0): -At the upgrade-block, the sdk will panic without flushing data to disk or closing dbs properly. - -Upgrade guide: - 1. After seeing `UPGRADE "xxxx" NEED at height....`, restart current version with `-X github.com/tendermint/tm-db.ForceSync=1` - 2. Restart new version as normal - -Example: Upgrading sifchain from v0.14.0 to v0.15.0 - -# log: -panic: UPGRADE "0.15.0" NEEDED at height: 8170210: {"binaries":{"linux/amd64":"https://github.com/Sifchain/sifnode/releases/download/v0.15.0/sifnoded-v0.15.0-linux-amd64.zip?checksum=0c03b5846c5a13dcc0d9d3127e4f0cee0aeddcf2165177b2f2e0d60dbcf1a5ea"}} - -# step1 -git reset --hard -git checkout v0.14.0 -go mod edit -replace github.com/tendermint/tm-db=github.com/baabeetaa/tm-db@pebble -go mod tidy -go install -tags pebbledb -ldflags "-w -s -X github.com/cosmos/cosmos-sdk/types.DBBackend=pebbledb -X github.com/tendermint/tm-db.ForceSync=1" ./cmd/sifnoded - -$HOME/go/bin/sifnoded start --db_backend=pebbledb - -# step 2 -git reset --hard -git checkout v0.15.0 -go mod edit -replace github.com/tendermint/tm-db=github.com/baabeetaa/tm-db@pebble -go mod tidy -go install -tags pebbledb -ldflags "-w -s -X github.com/cosmos/cosmos-sdk/types.DBBackend=pebbledb" ./cmd/sifnoded - -$HOME/go/bin/sifnoded start --db_backend=pebbledb -*/ -var ( - ForceSync = "1" - isForceSync = false -) - func init() { dbCreator := func(name string, dir string) (DB, error) { return NewPebbleDB(name, dir) } registerDBCreator(PebbleDBBackend, dbCreator) - - if ForceSync == "1" { - isForceSync = true - } } // PebbleDB is a PebbleDB backend. @@ -130,10 +84,6 @@ func (db *PebbleDB) Set(key []byte, value []byte) error { } wopts := pebble.NoSync - if isForceSync { - wopts = pebble.Sync - } - err := db.db.Set(key, value, wopts) if err != nil { return err @@ -163,9 +113,6 @@ func (db *PebbleDB) Delete(key []byte) error { } wopts := pebble.NoSync - if isForceSync { - wopts = pebble.Sync - } err := db.db.Delete(key, wopts) if err != nil { return err @@ -298,9 +245,6 @@ func (b *pebbleDBBatch) Write() error { } wopts := pebble.NoSync - if isForceSync { - wopts = pebble.Sync - } err := b.batch.Commit(wopts) if err != nil { return err diff --git a/pebble_test.go b/pebble_test.go index 58f3a37..17c4e8b 100644 --- a/pebble_test.go +++ b/pebble_test.go @@ -23,16 +23,6 @@ func TestPebbleDBBackend(t *testing.T) { assert.True(t, ok) } -// func TestPebbleDBStats(t *testing.T) { -// name := fmt.Sprintf("test_%x", randStr(12)) -// dir := os.TempDir() -// db, err := NewDB(name, PebbleDBBackend, dir) -// require.NoError(t, err) -// defer cleanupDBDir(dir, name) - -// assert.NotEmpty(t, db.Stats()) -// } - func BenchmarkPebbleDBRandomReadsWrites(b *testing.B) { name := fmt.Sprintf("test_%x", randStr(12)) dir := os.TempDir() diff --git a/rocksdb_iterator.go b/rocksdb_iterator.go index 495517c..53fa9a9 100644 --- a/rocksdb_iterator.go +++ b/rocksdb_iterator.go @@ -56,7 +56,6 @@ func (itr *rocksDBIterator) Domain() ([]byte, []byte) { // Valid implements Iterator. func (itr *rocksDBIterator) Valid() bool { - // Once invalid, forever invalid. if itr.isInvalid { return false @@ -75,9 +74,9 @@ func (itr *rocksDBIterator) Valid() bool { } // If key is end or past it, invalid. - var start = itr.start - var end = itr.end - var key = moveSliceToBytes(itr.source.Key()) + start := itr.start + end := itr.end + key := moveSliceToBytes(itr.source.Key()) if itr.isReverse { if start != nil && bytes.Compare(key, start) < 0 { itr.isInvalid = true