diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..890cdb3 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,21 @@ +# runs benchmarks on the pebble and goleveldb backends +name: Benchmarks + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v3 + with: + go-version: 1.21 + - name: Run benchmarks + run: go test -bench=. diff --git a/.golangci.yml b/.golangci.yml index 1c42543..fd42bb8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,12 +1,13 @@ -linters: +run: tests: true + timeout: 10m + +linters: disable-all: true enable: - - bodyclose - - depguard - - dogsled - - dupl + - exportloopref - errcheck + - gci - goconst - gocritic - gofumpt @@ -14,38 +15,99 @@ linters: - gosimple - govet - ineffassign - - lll - misspell - nakedret - - prealloc - staticcheck + - thelper + - typecheck - stylecheck + - revive - typecheck + - tenv - unconvert + # Prefer unparam over revive's unused param. It is more thorough in its checking. - unparam - unused - - nolintlint + - misspell issues: + exclude-rules: + - text: 'differs only by capitalization to method' + linters: + - revive + - text: 'Use of weak random number generator' + linters: + - gosec + - linters: + - staticcheck + text: "SA1019:" # silence errors on usage of deprecated funcs + + max-issues-per-linter: 10000 + max-same-issues: 10000 + linters-settings: - errcheck: - check-blank: true - depguard: + gci: + sections: + - standard # Standard section: captures all standard packages. + - default # Default section: contains all imports that could not be matched to another section type. + - blank # blank imports + - dot # dot imports + - prefix(github.com/cometbft/cometbft-db) + custom-order: true + revive: + enable-all-rules: true + # Do NOT whine about the following, full explanation found in: + # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#description-of-available-rules rules: - main: - files: - - $all - - "!$test" - allow: - - $gostd - - github.com/cometbft - - github.com/syndtr/goleveldb/leveldb - - github.com/google/btree - test: - files: - - $test - allow: - - $gostd - - github.com/cometbft - - github.com/syndtr/goleveldb/leveldb - - github.com/stretchr/testify + - name: use-any + disabled: true + - name: if-return + disabled: true + - name: max-public-structs + disabled: true + - name: cognitive-complexity + disabled: true + - name: argument-limit + disabled: true + - name: cyclomatic + disabled: true + - name: file-header + disabled: true + - name: function-length + disabled: true + - name: function-result-limit + disabled: true + - name: line-length-limit + disabled: true + - name: flag-parameter + disabled: true + - name: add-constant + disabled: true + - name: empty-lines + disabled: true + - name: banned-characters + disabled: true + - name: deep-exit + disabled: true + - name: confusing-results + disabled: true + - name: unused-parameter + disabled: true + - name: modifies-value-receiver + disabled: true + - name: early-return + disabled: true + - name: confusing-naming + disabled: true + - name: defer + disabled: true + # Disabled in favour of unparam. + - name: unused-parameter + disabled: true + - name: unhandled-error + disabled: false + arguments: + - 'fmt.Printf' + - 'fmt.Print' + - 'fmt.Println' + - 'myFunction' \ No newline at end of file diff --git a/Makefile b/Makefile index 0f74231..a70896c 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,10 @@ test-badgerdb: @go test $(PACKAGES) -tags badgerdb -v .PHONY: test-badgerdb +test-pebble: + @echo "--> Running go test" + @go test $(PACKAGES) -tags pebbledb -v + test-all: @echo "--> Running go test" @go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb -v @@ -52,7 +56,6 @@ test-all-with-coverage: -race \ -coverprofile=coverage.txt \ -covermode=atomic \ - -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb \ -v .PHONY: test-all-with-coverage diff --git a/backend_test.go b/backend_test.go index 0f19241..c523cc1 100644 --- a/backend_test.go +++ b/backend_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ import ( // Register a test backend for PrefixDB as well, with some unrelated junk data func init() { - //nolint: errcheck + //nolint: errcheck, revive // probably should check errors? registerDBCreator("prefixdb", func(name, dir string) (DB, error) { mdb := NewMemDB() mdb.Set([]byte("a"), []byte{1}) @@ -22,17 +23,25 @@ func init() { mdb.Set([]byte("u"), []byte{21}) mdb.Set([]byte("z"), []byte{26}) return NewPrefixDB(mdb, []byte("test/")), nil - }, false) + }) } func cleanupDBDir(dir, name string) { - err := os.RemoveAll(filepath.Join(dir, name) + ".db") - if err != nil { - panic(err) + for i := 0; i < 5; i++ { + err := os.RemoveAll(filepath.Join(dir, name) + ".db") + if err != nil { + if i == 4 { // If this was the last attempt, panic + panic(err) + } + time.Sleep(time.Second) // Wait for a second before the next attempt + } else { + break // If there was no error, break the loop + } } } func testBackendGetSetDelete(t *testing.T, backend BackendType) { + t.Helper() // Default dirname, err := os.MkdirTemp("", fmt.Sprintf("test_backend_%s_", backend)) require.Nil(t, err) @@ -161,6 +170,8 @@ func TestDBIterator(t *testing.T) { } func testDBIterator(t *testing.T, backend BackendType) { + t.Helper() + name := fmt.Sprintf("test_%x", randStr(12)) dir := os.TempDir() db, err := NewDB(name, backend, dir) @@ -317,6 +328,8 @@ func testDBIterator(t *testing.T, backend BackendType) { } func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) { + t.Helper() + var list []int64 for itr.Valid() { key := itr.Key() @@ -335,6 +348,8 @@ func TestDBBatch(t *testing.T) { } func testDBBatch(t *testing.T, backend BackendType) { + t.Helper() + name := fmt.Sprintf("test_%x", randStr(12)) dir := os.TempDir() db, err := NewDB(name, backend, dir) @@ -396,8 +411,12 @@ func testDBBatch(t *testing.T, backend BackendType) { // it should be possible to close an empty batch, and to re-close a closed batch batch = db.NewBatch() - batch.Close() - batch.Close() + if err := batch.Close(); err != nil { + require.NoError(t, err) + } + if err := batch.Close(); err != nil { + require.NoError(t, err) + } // all other operations on a closed batch should error require.Error(t, batch.Set([]byte("a"), []byte{9})) @@ -407,6 +426,7 @@ func testDBBatch(t *testing.T, backend BackendType) { } func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) { + t.Helper() iter, err := db.Iterator(nil, nil) require.NoError(t, err) defer iter.Close() diff --git a/badger_db.go b/badger_db.go index ecdfca1..08f201f 100644 --- a/badger_db.go +++ b/badger_db.go @@ -12,7 +12,7 @@ import ( "github.com/dgraph-io/badger/v2" ) -func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } +func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator) } func badgerDBCreator(dbName, dir string) (DB, error) { return NewBadgerDB(dbName, dir) diff --git a/boltdb.go b/boltdb.go index ffebbca..e452105 100644 --- a/boltdb.go +++ b/boltdb.go @@ -19,7 +19,7 @@ var ( func init() { registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) { return NewBoltDB(name, dir) - }, false) + }) } // BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt). diff --git a/cleveldb.go b/cleveldb.go index 7896730..3a42a31 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -14,7 +14,7 @@ func init() { dbCreator := func(name string, dir string) (DB, error) { return NewCLevelDB(name, dir) } - registerDBCreator(CLevelDBBackend, dbCreator, false) + registerDBCreator(CLevelDBBackend, dbCreator) } // CLevelDB uses the C LevelDB database via a Go wrapper. diff --git a/common_test.go b/common_test.go index fe2df32..f4e97c2 100644 --- a/common_test.go +++ b/common_test.go @@ -11,21 +11,24 @@ import ( "github.com/stretchr/testify/require" ) -//---------------------------------------- +// ---------------------------------------- // Helper functions. func checkValue(t *testing.T, db DB, key []byte, valueWanted []byte) { + t.Helper() valueGot, err := db.Get(key) assert.NoError(t, err) assert.Equal(t, valueWanted, valueGot) } func checkValid(t *testing.T, itr Iterator, expected bool) { + t.Helper() valid := itr.Valid() require.Equal(t, expected, valid) } func checkNext(t *testing.T, itr Iterator, expected bool) { + t.Helper() itr.Next() // assert.NoError(t, err) TODO: look at fixing this valid := itr.Valid() @@ -33,16 +36,19 @@ func checkNext(t *testing.T, itr Iterator, expected bool) { } func checkNextPanics(t *testing.T, itr Iterator) { + t.Helper() assert.Panics(t, func() { itr.Next() }, "checkNextPanics expected an error but didn't") } func checkDomain(t *testing.T, itr Iterator, start, end []byte) { + t.Helper() ds, de := itr.Domain() assert.Equal(t, start, ds, "checkDomain domain start incorrect") assert.Equal(t, end, de, "checkDomain domain end incorrect") } func checkItem(t *testing.T, itr Iterator, key []byte, value []byte) { + t.Helper() v := itr.Value() k := itr.Key() @@ -52,6 +58,7 @@ func checkItem(t *testing.T, itr Iterator, key []byte, value []byte) { } func checkInvalid(t *testing.T, itr Iterator) { + t.Helper() checkValid(t, itr, false) checkKeyPanics(t, itr) checkValuePanics(t, itr) @@ -59,14 +66,17 @@ func checkInvalid(t *testing.T, itr Iterator) { } func checkKeyPanics(t *testing.T, itr Iterator) { + t.Helper() assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") } func checkValuePanics(t *testing.T, itr Iterator) { + t.Helper() assert.Panics(t, func() { itr.Value() }) } func newTempDB(t *testing.T, backend BackendType) (db DB, dbDir string) { + t.Helper() dirname, err := os.MkdirTemp("", "db_common_test") require.NoError(t, err) db, err = NewDB("testdb", backend, dirname) @@ -75,6 +85,7 @@ func newTempDB(t *testing.T, backend BackendType) (db DB, dbDir string) { } func benchmarkRangeScans(b *testing.B, db DB, dbSize int64) { + b.Helper() b.StopTimer() rangeSize := int64(10000) @@ -83,8 +94,8 @@ func benchmarkRangeScans(b *testing.B, db DB, dbSize int64) { } for i := int64(0); i < dbSize; i++ { - bytes := int642Bytes(i) - err := db.Set(bytes, bytes) + int64bytes := int642Bytes(i) + err := db.Set(int64bytes, int64bytes) if err != nil { // require.NoError() is very expensive (according to profiler), so check manually b.Fatal(b, err) @@ -101,12 +112,14 @@ func benchmarkRangeScans(b *testing.B, db DB, dbSize int64) { for ; iter.Valid(); iter.Next() { count++ } - iter.Close() + err = iter.Close() + require.NoError(b, err) require.EqualValues(b, rangeSize, count) } } func benchmarkRandomReadsWrites(b *testing.B, db DB) { + b.Helper() b.StopTimer() // create dummy data diff --git a/db.go b/db.go index 4d518c0..fd6616e 100644 --- a/db.go +++ b/db.go @@ -35,15 +35,20 @@ const ( RocksDBBackend BackendType = "rocksdb" BadgerDBBackend BackendType = "badgerdb" + + // PebbleDBDBBackend represents pebble (uses github.com/cockroachdb/pebble) + // - EXPERIMENTAL + // - use pebble build tag (go build -tags pebbledb) + PebbleDBBackend BackendType = "pebbledb" ) type dbCreator func(name string, dir string) (DB, error) var backends = map[BackendType]dbCreator{} -func registerDBCreator(backend BackendType, creator dbCreator, force bool) { +func registerDBCreator(backend BackendType, creator dbCreator) { _, ok := backends[backend] - if !force && ok { + if ok { return } backends[backend] = creator diff --git a/go.mod b/go.mod index 634453e..44ca59f 100644 --- a/go.mod +++ b/go.mod @@ -15,21 +15,41 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/getsentry/sentry-go v0.26.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect +) + +require ( + github.com/DataDog/zstd v1.5.5 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20240112000813-0effd2429fca + github.com/cockroachdb/redact v1.1.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/klauspost/compress v1.12.3 // indirect - github.com/pkg/errors v0.8.1 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9509fe0..6e2da10 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,34 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= +github.com/cockroachdb/pebble v0.0.0-20240112000813-0effd2429fca h1:tUpLbX8fSYal8hba2v98y7VFIYb8aDH0oIgIkgxLVCU= +github.com/cockroachdb/pebble v0.0.0-20240112000813-0effd2429fca/go.mod h1:BHuaMa/lK7fUe75BlsteiiTu8ptIG+qSAuDtGMArP18= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -25,6 +46,10 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA= +github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -57,13 +82,16 @@ github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/linxGnu/grocksdb v1.8.10 h1:6FAhBThErRfJaevGOZISYvkG7RD4gfzeq452X4r8pes= github.com/linxGnu/grocksdb v1.8.10/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -79,10 +107,25 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -110,6 +153,8 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -119,12 +164,14 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -137,8 +184,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -153,8 +200,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:gphdwh0npgs8elJ4T6J+DQJHPVF7RsuJHCfwztUb4J4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -165,17 +212,19 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/goleveldb.go b/goleveldb.go index fd1bffd..518c729 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -14,7 +14,7 @@ func init() { dbCreator := func(name string, dir string) (DB, error) { return NewGoLevelDB(name, dir) } - registerDBCreator(GoLevelDBBackend, dbCreator, false) + registerDBCreator(GoLevelDBBackend, dbCreator) } type GoLevelDB struct { @@ -28,7 +28,7 @@ func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { } func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { - dbPath := filepath.Join(dir, name+".db") + dbPath := filepath.Join(dir, name+".db") //nolint:goconst db, err := leveldb.OpenFile(dbPath, o) if err != nil { return nil, err diff --git a/goleveldb_test.go b/goleveldb_test.go index e1c879f..54b1d22 100644 --- a/goleveldb_test.go +++ b/goleveldb_test.go @@ -17,7 +17,8 @@ func TestGoLevelDBNewGoLevelDB(t *testing.T) { require.Nil(t, err) _, err = NewGoLevelDB(name, "") require.NotNil(t, err) - wr1.Close() // Close the db to release the lock + err = wr1.Close() // Close the db to release the lock + require.Nil(t, err) // Test we can open the db twice for reading only ro1, err := NewGoLevelDBWithOpts(name, "", &opt.Options{ReadOnly: true}) @@ -35,7 +36,8 @@ func BenchmarkGoLevelDBRandomReadsWrites(b *testing.B) { b.Fatal(err) } defer func() { - db.Close() + err = db.Close() + require.NoError(b, err) cleanupDBDir("", name) }() diff --git a/memdb.go b/memdb.go index 2bfa0d1..ba66c7b 100644 --- a/memdb.go +++ b/memdb.go @@ -16,7 +16,7 @@ const ( func init() { registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) { return NewMemDB(), nil - }, false) + }) } // item is a btree.Item with byte slices as keys and values @@ -137,7 +137,7 @@ func (db *MemDB) DeleteSync(key []byte) error { } // Close implements DB. -func (db *MemDB) Close() error { +func (*MemDB) Close() error { // Close is a noop since for an in-memory database, we don't have a destination to flush // contents to nor do we want any data loss on invoking Close(). return nil @@ -149,7 +149,10 @@ func (db *MemDB) Print() error { defer db.mtx.RUnlock() db.btree.Ascend(func(i btree.Item) bool { - item := i.(*item) + item, ok := i.(*item) + if !ok { + return false // or handle the error as appropriate + } fmt.Printf("[%X]:\t[%X]\n", item.key, item.value) return true }) diff --git a/memdb_iterator.go b/memdb_iterator.go index ebd104f..b56320a 100644 --- a/memdb_iterator.go +++ b/memdb_iterator.go @@ -56,7 +56,10 @@ func newMemDBIteratorMtxChoice(db *MemDB, start []byte, end []byte, reverse bool abortLessThan []byte ) visitor := func(i btree.Item) bool { - item := i.(*item) + item, ok := i.(*item) + if !ok { + return false // or handle the error as appropriate + } if skipEqual != nil && bytes.Equal(item.key, skipEqual) { skipEqual = nil return true @@ -105,7 +108,7 @@ func newMemDBIteratorMtxChoice(db *MemDB, start []byte, end []byte, reverse bool // Close implements Iterator. func (i *memDBIterator) Close() error { i.cancel() - for range i.ch { // drain channel + for range i.ch { //nolint:revive // drain channel } i.item = nil return nil @@ -134,7 +137,7 @@ func (i *memDBIterator) Next() { } // Error implements Iterator. -func (i *memDBIterator) Error() error { +func (*memDBIterator) Error() error { return nil // famous last words } diff --git a/pebble.go b/pebble.go new file mode 100644 index 0000000..27980e4 --- /dev/null +++ b/pebble.go @@ -0,0 +1,525 @@ +package db + +import ( + "bytes" + "fmt" + "path/filepath" + "runtime" + "sync" + + "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/bloom" +) + +// ForceSync +/* +This is set at compile time. Could be 0 or 1, defaults is 0. +It will force using Sync for NoSync functions (Set, Delete, Write) + +Used as a workaround for chain-upgrade issue: At the upgrade-block, the sdk will panic without flushing data to disk or +closing dbs properly. + +Upgrade guide: + 1. After seeing `UPGRADE "xxxx" NEED at height....`, restart current version with `-X github.com/tendermint/tm-db.ForceSync=1` + 2. Restart new version as normal + + +Example: Upgrading sifchain from v0.14.0 to v0.15.0 + +# log: +panic: UPGRADE "0.15.0" NEEDED at height: 8170210: {"binaries":{"linux/amd64":"https://github.com/Sifchain/sifnode/releases/download/v0.15.0/sifnoded-v0.15.0-linux-amd64.zip?checksum=0c03b5846c5a13dcc0d9d3127e4f0cee0aeddcf2165177b2f2e0d60dbcf1a5ea"}} + +# step1 +git reset --hard +git checkout v0.14.0 +go mod edit -replace github.com/tendermint/tm-db=github.com/baabeetaa/tm-db@pebble +go mod tidy +go install -tags pebbledb -ldflags "-w -s -X github.com/cosmos/cosmos-sdk/types.DBBackend=pebbledb -X github.com/tendermint/tm-db.ForceSync=1" ./cmd/sifnoded + +$HOME/go/bin/sifnoded start --db_backend=pebbledb + + +# step 2 +git reset --hard +git checkout v0.15.0 +go mod edit -replace github.com/tendermint/tm-db=github.com/baabeetaa/tm-db@pebble +go mod tidy +go install -tags pebbledb -ldflags "-w -s -X github.com/cosmos/cosmos-sdk/types.DBBackend=pebbledb" ./cmd/sifnoded + +$HOME/go/bin/sifnoded start --db_backend=pebbledb + +*/ + +const ( + // minCache is the minimum amount of memory in megabytes to allocate to pebble + // read and write caching, split half and half. + minCache = 4096 + + // minHandles is the minimum number of files handles to allocate to the open + // database files. + minHandles = 8192 +) + +var ( + ForceSync = "0" + isForceSync = false +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewPebbleDB(name, dir) + } + registerDBCreator(PebbleDBBackend, dbCreator) + + if ForceSync == "1" { + isForceSync = true + } +} + +// PebbleDB is a PebbleDB backend. +type PebbleDB struct { + db *pebble.DB +} + +var iteratorPool = sync.Pool{ + New: func() interface{} { + return &pebbleDBIterator{} + }, +} + +var _ DB = (*PebbleDB)(nil) + +func NewPebbleDB(name string, dir string) (*PebbleDB, error) { + opts := &pebble.Options{} + opts.EnsureDefaults() + return NewPebbleDBWithOpts(name, dir) +} + +func NewPebbleDBWithOpts(name string, dir string) (*PebbleDB, error) { + dbPath := filepath.Join(dir, name+".db") + // Two memory tables is configured which is identical to leveldb, + // including a frozen memory table and another live one. + + memTableLimit := 2 + cache := int(1 << 13) + if cache < minCache { + cache = minCache + } + handles := minHandles + memTableSize := cache * 1024 * 1024 / 2 / memTableLimit + opts := &pebble.Options{ + Cache: pebble.NewCache(1 << 30), // 4GB + MaxOpenFiles: handles, + MemTableSize: uint64(memTableSize), + + // MemTableStopWritesThreshold places a hard limit on the size + // of the existent MemTables(including the frozen one). + // Note, this must be the number of tables not the size of all memtables + // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742 + // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903. + MemTableStopWritesThreshold: memTableLimit, + // The default compaction concurrency(1 thread), + // Here use all available CPUs for faster compaction. + MaxConcurrentCompactions: func() int { return runtime.NumCPU() * 3 }, // Use twice the number of CPUs, // Use twice the number of CPUs + + // Per-level options. Options for at least one level must be specified. The + // options for the last level are used for all subsequent levels. + Levels: []pebble.LevelOptions{ + {FilterPolicy: bloom.FilterPolicy(10)}, + }, + } + + opts.Experimental.ReadSamplingMultiplier = -1 // borrowed from the ethereum pebble db + + p, err := pebble.Open(dbPath, opts) + if err != nil { + return nil, err + } + return &PebbleDB{ + db: p, + }, nil +} + +// Get implements DB. +func (db *PebbleDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + + res, closer, err := db.db.Get(key) + if err != nil { + if err == pebble.ErrNotFound { + return nil, nil + } + return nil, err + } + defer closer.Close() + + return cp(res), nil +} + +// Has implements DB. +func (db *PebbleDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, errKeyEmpty + } + bytesPeb, err := db.Get(key) + if err != nil { + return false, err + } + return bytesPeb != nil, nil +} + +// Set implements DB. +func (db *PebbleDB) Set(key []byte, value []byte) error { + wopts := pebble.NoSync + if isForceSync { + wopts = pebble.Sync + } + return db.set(key, value, *wopts) +} + +// SetSync implements DB. +func (db *PebbleDB) SetSync(key []byte, value []byte) error { + return db.set(key, value, *pebble.Sync) +} + +// Delete implements DB. +func (db *PebbleDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + + wopts := pebble.NoSync + if isForceSync { + wopts = pebble.Sync + } + err := db.db.Delete(key, wopts) + if err != nil { + return err + } + return nil +} + +// DeleteSync implements DB. +func (db PebbleDB) DeleteSync(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + err := db.db.Delete(key, pebble.Sync) + if err != nil { + return nil + } + return nil +} + +func (db *PebbleDB) DB() *pebble.DB { + return db.db +} + +// Close implements DB. +func (db PebbleDB) Close() error { + db.db.Close() + return nil +} + +// Print implements DB. +func (db *PebbleDB) Print() error { + itr, err := db.Iterator(nil, nil) + if err != nil { + return err + } + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } + return nil +} + +// Stats implements DB. +func (db *PebbleDB) Stats() map[string]string { + m := db.db.Metrics() + stats := make(map[string]string) + + stats["BlockCacheSize"] = fmt.Sprintf("%d", m.BlockCache.Size) + stats["BlockCacheHits"] = fmt.Sprintf("%d", m.BlockCache.Hits) + stats["BlockCacheMisses"] = fmt.Sprintf("%d", m.BlockCache.Misses) + stats["MemTableSize"] = fmt.Sprintf("%d", m.MemTable.Size) + stats["Flushes"] = fmt.Sprintf("%d", m.Flush.Count) + stats["Compactions"] = fmt.Sprintf("%d", m.Compact.Count) + return stats +} + +// NewBatch implements DB. +func (db *PebbleDB) NewBatch() Batch { + return newPebbleDBBatch(db) +} + +func newPebbleDBBatch(db *PebbleDB) *pebbleDBBatch { + return &pebbleDBBatch{ + batch: db.db.NewBatch(), + } +} + +// Iterator implements DB. +func (db *PebbleDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + o := pebble.IterOptions{ + LowerBound: start, + UpperBound: end, + } + itr, err := db.db.NewIter(&o) + if err != nil { + return nil, err + } + itr.First() + + return newPebbleDBIterator(itr, start, end, false), nil +} + +// ReverseIterator implements DB. +func (db *PebbleDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + o := pebble.IterOptions{ + LowerBound: start, + UpperBound: end, + } + itr, err := db.db.NewIter(&o) + if err != nil { + return nil, err + } + itr.Last() + return newPebbleDBIterator(itr, start, end, true), nil +} + +var _ Batch = (*pebbleDBBatch)(nil) + +type pebbleDBBatch struct { + batch *pebble.Batch +} + +var _ Batch = (*pebbleDBBatch)(nil) + +func (b *pebbleDBBatch) Write() error { + if b.batch == nil { + return errBatchClosed + } + + wopts := pebble.NoSync + if isForceSync { + wopts = pebble.Sync + } + err := b.batch.Commit(wopts) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + + return b.Close() +} + +// Set implements Batch. +func (b *pebbleDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.batch == nil { + return errBatchClosed + } + + return b.batch.Set(key, value, nil) +} + +// Delete implements Batch. +func (b *pebbleDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.batch == nil { + return errBatchClosed + } + + return b.batch.Delete(key, nil) +} + +// Write implements Batch. + +// WriteSync implements Batch. +func (b *pebbleDBBatch) WriteSync() error { + if b.batch == nil { + return errBatchClosed + } + err := b.batch.Commit(pebble.Sync) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// Close implements Batch. +func (b *pebbleDBBatch) Close() error { + if b.batch != nil { + err := b.batch.Close() + if err != nil { + return err + } + b.batch = nil + } + + return nil +} + +type pebbleDBIterator struct { + source *pebble.Iterator + start, end []byte + isReverse bool + isInvalid bool +} + +var _ Iterator = (*pebbleDBIterator)(nil) + +func newPebbleDBIterator(source *pebble.Iterator, start, end []byte, isReverse bool) *pebbleDBIterator { + item := iteratorPool.Get() + itr, ok := item.(*pebbleDBIterator) + if !ok { + panic("item in iteratorPool is not of type *pebbleDBIterator") + } + itr.source = source + itr.start = start + itr.end = end + itr.isReverse = isReverse + itr.isInvalid = false + + if isReverse { + if end == nil { + source.Last() + } + } else { + if start == nil { + source.First() + } + } + return itr +} + +// Domain implements Iterator. +func (itr *pebbleDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Valid implements Iterator. +func (itr *pebbleDBIterator) Valid() bool { + // Once invalid, forever invalid. + if itr.isInvalid { + return false + } + + // If source has error, invalid. + if err := itr.source.Error(); err != nil { + itr.isInvalid = true + + return false + } + + // If source is invalid, invalid. + if !itr.source.Valid() { + itr.isInvalid = true + + return false + } + + // If key is end or past it, invalid. + start := itr.start + end := itr.end + key := itr.source.Key() + if itr.isReverse { + if start != nil && bytes.Compare(key, start) < 0 { + itr.isInvalid = true + + return false + } + } else { + if end != nil && bytes.Compare(end, key) <= 0 { + itr.isInvalid = true + + return false + } + } + + // It's valid. + return true +} + +// Key implements Iterator. +func (itr *pebbleDBIterator) Key() []byte { + itr.assertIsValid() + return cp(itr.source.Key()) +} + +// Value implements Iterator. +func (itr *pebbleDBIterator) Value() []byte { + itr.assertIsValid() + return cp(itr.source.Value()) +} + +// Next implements Iterator. +func (itr pebbleDBIterator) Next() { + itr.assertIsValid() + if itr.isReverse { + itr.source.Prev() + } else { + itr.source.Next() + } +} + +// Error implements Iterator. +func (itr *pebbleDBIterator) Error() error { + return itr.source.Error() +} + +// Close implements Iterator. +func (itr *pebbleDBIterator) Close() error { + err := itr.source.Close() + if err != nil { + return err + } + itr.source = nil + itr.start = nil + itr.end = nil + itr.isReverse = false + itr.isInvalid = true + iteratorPool.Put(itr) + return nil +} + +func (itr *pebbleDBIterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} + +// set function with unnecessary dereference removed +func (db *PebbleDB) set(key []byte, value []byte, pebbleSync pebble.WriteOptions) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + + err := db.db.Set(key, value, &pebbleSync) + if err != nil { + return err + } + return nil +} diff --git a/pebble_test.go b/pebble_test.go new file mode 100644 index 0000000..5d27bd5 --- /dev/null +++ b/pebble_test.go @@ -0,0 +1,206 @@ +package db + +import ( + "fmt" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPebbleDBBackend(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, PebbleDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + _, ok := db.(*PebbleDB) + assert.True(t, ok) +} + +// TODO: Add tests for pebble + +func TestPebbleDBStats(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, PebbleDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + stats := db.Stats() + assert.NotEmpty(t, stats) + + assert.Contains(t, stats, "BlockCacheSize") + assert.Contains(t, stats, "BlockCacheHits") + assert.Contains(t, stats, "BlockCacheMisses") + assert.Contains(t, stats, "MemTableSize") + assert.Contains(t, stats, "Flushes") + assert.Contains(t, stats, "Compactions") +} + +type BenchmarkCase struct { + Name string + Backend BackendType + BenchFunc func(b *testing.B, db DB) +} + +type BenchmarkVariationsCase struct { + Name string + Backend BackendType + NumKeys int + ValueSize int +} + +func BenchmarkDB(b *testing.B) { + var variationCases []BenchmarkVariationsCase + + // Define the range of keys and values + keys := []int{10000, 25000, 50000, 75000, 100000} // 10k, 100k - things we figure would happen in prod + values := []int{512, 1024, 2048, 4096} // 2KB, 4KB, 8KB - things we figure would happen in prod + + // Define the backends + backends := []BackendType{PebbleDBBackend, GoLevelDBBackend} + + // Generate the test cases + for _, key := range keys { + for _, value := range values { + if value > 1<<19 { // Limit the value size to 500KB + break + } + for _, backend := range backends { + variationCases = append(variationCases, BenchmarkVariationsCase{ + Name: fmt.Sprintf("Keys_%d_Values_%d", key, value), + Backend: backend, + NumKeys: key, + ValueSize: value, + }) + } + } + } + + // Create a map to store the benchmark results + results := make(map[string]BenchmarkResult) + + for _, vc := range variationCases { + b.Run(fmt.Sprintf("%s_%s", vc.Name, vc.Backend), func(b *testing.B) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, vc.Backend, dir) + require.NoError(b, err) + defer cleanupDBDir(dir, name) + + result := benchmarkDBVariations(b, db, vc.NumKeys, vc.ValueSize) + + // Store the benchmark result + results[fmt.Sprintf("%s_%s", vc.Name, vc.Backend)] = result + }) + } + + // Print out the comparison table + fmt.Println("Test Case\tBackend\tSetTime\tReadTime\tWriteTime\tConcurrentTime") + for _, vc := range variationCases { + var resultsArr []BenchmarkResult + for _, backend := range backends { + result := results[fmt.Sprintf("%s_%s", vc.Name, backend)] + resultsArr = append(resultsArr, result) + } + + // Calculate total time for each backend + totalTime0 := resultsArr[0].SetTime.Seconds() + resultsArr[0].ReadTime.Seconds() + resultsArr[0].WriteTime.Seconds() + totalTime1 := resultsArr[1].SetTime.Seconds() + resultsArr[1].ReadTime.Seconds() + resultsArr[1].WriteTime.Seconds() + + // Determine which backend is faster + fasterBackend := backends[0] + if totalTime0 > totalTime1 { + fasterBackend = backends[1] + } + + // Print total times and faster backend + fmt.Printf("Test Case: %s\n", vc.Name) + fmt.Printf("Total time for %s: %.2f seconds\n", backends[0], totalTime0) + fmt.Printf("Total time for %s: %.2f seconds\n", backends[1], totalTime1) + fmt.Printf("Faster Backend: %s\n", fasterBackend) + fmt.Println("-----------------------------") + } +} + +func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) BenchmarkResult { + b.Helper() + // Generate a large value + largeValue := make([]byte, valueSize) + for i := range largeValue { + largeValue[i] = 'a' + } + + var wg sync.WaitGroup + const numRoutines = 100 + numOpsPerRoutine := numKeys / numRoutines + + // Set keys and values + setTime := time.Now() + for i := 0; i < numRoutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < numOpsPerRoutine; j++ { + key := fmt.Sprintf("key_%d", j) + err := db.Set([]byte(key), largeValue) + require.NoError(b, err) + } + }() + } + wg.Wait() + setDuration := time.Since(setTime) + + b.ResetTimer() + + // Random reads + readTime := time.Now() + for i := 0; i < numRoutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < numOpsPerRoutine; j++ { + key := fmt.Sprintf("key_%d", rand.Intn(numKeys)) + _, err := db.Get([]byte(key)) + require.NoError(b, err) + } + }() + } + wg.Wait() + readDuration := time.Since(readTime) + + // Random writes + writeTime := time.Now() + for i := 0; i < numRoutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < numOpsPerRoutine; j++ { + key := fmt.Sprintf("key_%d", rand.Intn(numKeys)) + err := db.Set([]byte(key), largeValue) + require.NoError(b, err) + } + }() + } + wg.Wait() + writeDuration := time.Since(writeTime) + + // Return the durations + return BenchmarkResult{ + SetTime: setDuration, + ReadTime: readDuration, + WriteTime: writeDuration, + } +} + +type BenchmarkResult struct { + SetTime time.Duration + ReadTime time.Duration + WriteTime time.Duration +} diff --git a/prefixdb_test.go b/prefixdb_test.go index 3fc53ee..4f7194d 100644 --- a/prefixdb_test.go +++ b/prefixdb_test.go @@ -7,6 +7,7 @@ import ( ) func mockDBWithStuff(t *testing.T) DB { + t.Helper() db := NewMemDB() // Under "key" prefix require.NoError(t, db.Set(bz("key"), bz("value"))) @@ -51,7 +52,8 @@ func TestPrefixDBIterator1(t *testing.T) { checkItem(t, itr, bz("3"), bz("value3")) checkNext(t, itr, false) checkInvalid(t, itr) - itr.Close() + err = itr.Close() + require.NoError(t, err) } func TestPrefixDBReverseIterator1(t *testing.T) { @@ -68,7 +70,8 @@ func TestPrefixDBReverseIterator1(t *testing.T) { checkItem(t, itr, bz("1"), bz("value1")) checkNext(t, itr, false) checkInvalid(t, itr) - itr.Close() + err = itr.Close() + require.NoError(t, err) } func TestPrefixDBReverseIterator5(t *testing.T) { @@ -85,7 +88,8 @@ func TestPrefixDBReverseIterator5(t *testing.T) { checkItem(t, itr, bz("1"), bz("value1")) checkNext(t, itr, false) checkInvalid(t, itr) - itr.Close() + err = itr.Close() + require.NoError(t, err) } func TestPrefixDBReverseIterator6(t *testing.T) { @@ -100,7 +104,8 @@ func TestPrefixDBReverseIterator6(t *testing.T) { checkItem(t, itr, bz("2"), bz("value2")) checkNext(t, itr, false) checkInvalid(t, itr) - itr.Close() + err = itr.Close() + require.NoError(t, err) } func TestPrefixDBReverseIterator7(t *testing.T) { @@ -113,5 +118,6 @@ func TestPrefixDBReverseIterator7(t *testing.T) { checkItem(t, itr, bz("1"), bz("value1")) checkNext(t, itr, false) checkInvalid(t, itr) - itr.Close() + err = itr.Close() + require.NoError(t, err) } diff --git a/rocksdb.go b/rocksdb.go index 945491c..d508b82 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -15,7 +15,7 @@ func init() { dbCreator := func(name string, dir string) (DB, error) { return NewRocksDB(name, dir) } - registerDBCreator(RocksDBBackend, dbCreator, false) + registerDBCreator(RocksDBBackend, dbCreator) } // RocksDB is a RocksDB backend. diff --git a/test_helpers.go b/test_helpers.go index a930ac4..2d03345 100644 --- a/test_helpers.go +++ b/test_helpers.go @@ -22,13 +22,13 @@ MAIN_LOOP: if v >= 62 { // only 62 characters in strChars val >>= 6 continue - } else { - chars = append(chars, strChars[v]) - if len(chars) == length { - break MAIN_LOOP - } - val >>= 6 } + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } } diff --git a/util.go b/util.go index da0b635..7ba3a74 100644 --- a/util.go +++ b/util.go @@ -23,7 +23,7 @@ func cpIncr(bz []byte) (ret []byte) { for i := len(bz) - 1; i >= 0; i-- { if ret[i] < byte(0xFF) { ret[i]++ - return + return ret } ret[i] = byte(0x00) if i == 0 {