From dd683167c5942bfaa2fb6a10f9de7bacab4f1b0d Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Mon, 4 May 2020 15:47:58 -0700 Subject: [PATCH 1/2] walletdb/bdb: accept *bbolt.Options as main config param In this commit, we modify the Create and Open methods for bbolt to accept the bbolt config struct directly. With this change, we don't need to update this file each time we want to expose a new config option for the caller to set. --- go.mod | 1 + go.sum | 4 ---- waddrmgr/common_test.go | 11 +++++++++-- waddrmgr/manager_test.go | 6 +++++- wallet/loader.go | 11 +++++++++-- walletdb/bdb/db.go | 8 +++----- walletdb/bdb/driver.go | 25 +++++++++++++------------ walletdb/bdb/driver_test.go | 21 +++++++++++++++------ walletdb/bdb/interface_test.go | 7 ++++++- walletdb/db_test.go | 6 +++++- walletdb/example_test.go | 16 +++++++++++++--- walletdb/go.mod | 4 +--- walletdb/go.sum | 4 ---- wtxmgr/go.mod | 1 + wtxmgr/tx_test.go | 11 +++++++++-- 15 files changed, 90 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index a20589cb62..cdfbe9ccd1 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec // indirect github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf github.com/lightninglabs/neutrino v0.11.0 + go.etcd.io/bbolt v1.3.3 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 // indirect diff --git a/go.sum b/go.sum index 3f267ec86b..e42b7918dd 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495 h1:6IyqGr3fnd0tM3YxipK27TUskaOVUjU2nG45yzwcQKY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -89,8 +87,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= diff --git a/waddrmgr/common_test.go b/waddrmgr/common_test.go index 20493fe129..a0148cfe6b 100644 --- a/waddrmgr/common_test.go +++ b/waddrmgr/common_test.go @@ -15,6 +15,7 @@ import ( "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcwallet/walletdb" _ "github.com/btcsuite/btcwallet/walletdb/bdb" + "go.etcd.io/bbolt" ) var ( @@ -238,7 +239,10 @@ func emptyDB(t *testing.T) (tearDownFunc func(), db walletdb.DB) { t.Fatalf("Failed to create db temp dir: %v", err) } dbPath := filepath.Join(dirName, "mgrtest.db") - db, err = walletdb.Create("bdb", dbPath, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err = walletdb.Create("bdb", dbPath, opts) if err != nil { _ = os.RemoveAll(dirName) t.Fatalf("createDbNamespace: unexpected error: %v", err) @@ -259,7 +263,10 @@ func setupManager(t *testing.T) (tearDownFunc func(), db walletdb.DB, mgr *Manag t.Fatalf("Failed to create db temp dir: %v", err) } dbPath := filepath.Join(dirName, "mgrtest.db") - db, err = walletdb.Create("bdb", dbPath, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err = walletdb.Create("bdb", dbPath, opts) if err != nil { _ = os.RemoveAll(dirName) t.Fatalf("createDbNamespace: unexpected error: %v", err) diff --git a/waddrmgr/manager_test.go b/waddrmgr/manager_test.go index 0db0a88cb7..efd26df68d 100644 --- a/waddrmgr/manager_test.go +++ b/waddrmgr/manager_test.go @@ -21,6 +21,7 @@ import ( "github.com/btcsuite/btcwallet/snacl" "github.com/btcsuite/btcwallet/walletdb" "github.com/davecgh/go-spew/spew" + "go.etcd.io/bbolt" ) // failingCryptoKey is an implementation of the EncryptorDecryptor interface @@ -1683,7 +1684,10 @@ func testConvertWatchingOnly(tc *testContext) bool { defer os.Remove(woMgrName) // Open the new database copy and get the address manager namespace. - db, err := walletdb.Open("bdb", woMgrName, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Open("bdb", woMgrName, opts) if err != nil { tc.t.Errorf("openDbNamespace: unexpected error: %v", err) return false diff --git a/wallet/loader.go b/wallet/loader.go index 17f761f922..778fd6392f 100644 --- a/wallet/loader.go +++ b/wallet/loader.go @@ -15,6 +15,7 @@ import ( "github.com/btcsuite/btcwallet/internal/prompt" "github.com/btcsuite/btcwallet/waddrmgr" "github.com/btcsuite/btcwallet/walletdb" + "go.etcd.io/bbolt" ) const ( @@ -140,7 +141,10 @@ func (l *Loader) createNewWallet(pubPassphrase, privPassphrase, if err != nil { return nil, err } - db, err := walletdb.Create("bdb", dbPath, l.noFreelistSync) + opts := &bbolt.Options{ + NoFreelistSync: l.noFreelistSync, + } + db, err := walletdb.Create("bdb", dbPath, opts) if err != nil { return nil, err } @@ -196,7 +200,10 @@ func (l *Loader) OpenExistingWallet(pubPassphrase []byte, canConsolePrompt bool) // Open the database using the boltdb backend. dbPath := filepath.Join(l.dbDirPath, walletDbName) - db, err := walletdb.Open("bdb", dbPath, l.noFreelistSync) + opts := &bbolt.Options{ + NoFreelistSync: l.noFreelistSync, + } + db, err := walletdb.Open("bdb", dbPath, opts) if err != nil { log.Errorf("Failed to open database: %v", err) return nil, err diff --git a/walletdb/bdb/db.go b/walletdb/bdb/db.go index cf3fc2e2d4..c96d14685d 100644 --- a/walletdb/bdb/db.go +++ b/walletdb/bdb/db.go @@ -9,7 +9,7 @@ import ( "os" "github.com/btcsuite/btcwallet/walletdb" - "github.com/coreos/bbolt" + "go.etcd.io/bbolt" ) // convertErr converts some bolt errors to the equivalent walletdb error. @@ -367,16 +367,14 @@ func fileExists(name string) bool { // openDB opens the database at the provided path. walletdb.ErrDbDoesNotExist // is returned if the database doesn't exist and the create flag is not set. -func openDB(dbPath string, noFreelistSync bool, create bool) (walletdb.DB, error) { +func openDB(dbPath string, create bool, options *bbolt.Options) (walletdb.DB, error) { if !create && !fileExists(dbPath) { return nil, walletdb.ErrDbDoesNotExist } // Specify bbolt freelist options to reduce heap pressure in case the // freelist grows to be very large. - options := &bbolt.Options{ - NoFreelistSync: noFreelistSync, - FreelistType: bbolt.FreelistMapType, + options.FreelistType = bbolt.FreelistMapType } boltDB, err := bbolt.Open(dbPath, 0600, options) diff --git a/walletdb/bdb/driver.go b/walletdb/bdb/driver.go index 78442fd3f6..cfe9a2ccca 100644 --- a/walletdb/bdb/driver.go +++ b/walletdb/bdb/driver.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/btcsuite/btcwallet/walletdb" + "go.etcd.io/bbolt" ) const ( @@ -15,50 +16,50 @@ const ( ) // parseArgs parses the arguments from the walletdb Open/Create methods. -func parseArgs(funcName string, args ...interface{}) (string, bool, error) { +func parseArgs(funcName string, args ...interface{}) (string, *bbolt.Options, error) { if len(args) != 2 { - return "", false, fmt.Errorf("invalid arguments to %s.%s -- "+ - "expected database path and no-freelist-sync option", + return "", nil, fmt.Errorf("invalid arguments to %s.%s -- "+ + "expected database path and *bbolt.Option option", dbType, funcName) } dbPath, ok := args[0].(string) if !ok { - return "", false, fmt.Errorf("first argument to %s.%s is "+ + return "", nil, fmt.Errorf("first argument to %s.%s is "+ "invalid -- expected database path string", dbType, funcName) } - noFreelistSync, ok := args[1].(bool) + options, ok := args[1].(*bbolt.Options) if !ok { - return "", false, fmt.Errorf("second argument to %s.%s is "+ - "invalid -- expected no-freelist-sync bool", dbType, + return "", nil, fmt.Errorf("second argument to %s.%s is "+ + "invalid -- expected *bbolt.Option", dbType, funcName) } - return dbPath, noFreelistSync, nil + return dbPath, options, nil } // openDBDriver is the callback provided during driver registration that opens // an existing database for use. func openDBDriver(args ...interface{}) (walletdb.DB, error) { - dbPath, noFreelistSync, err := parseArgs("Open", args...) + dbPath, options, err := parseArgs("Open", args...) if err != nil { return nil, err } - return openDB(dbPath, noFreelistSync, false) + return openDB(dbPath, false, options) } // createDBDriver is the callback provided during driver registration that // creates, initializes, and opens a database for use. func createDBDriver(args ...interface{}) (walletdb.DB, error) { - dbPath, noFreelistSync, err := parseArgs("Create", args...) + dbPath, options, err := parseArgs("Create", args...) if err != nil { return nil, err } - return openDB(dbPath, noFreelistSync, true) + return openDB(dbPath, true, options) } func init() { diff --git a/walletdb/bdb/driver_test.go b/walletdb/bdb/driver_test.go index aa4d87210f..327d1dbf02 100644 --- a/walletdb/bdb/driver_test.go +++ b/walletdb/bdb/driver_test.go @@ -14,6 +14,7 @@ import ( "github.com/btcsuite/btcwallet/walletdb" _ "github.com/btcsuite/btcwallet/walletdb/bdb" + "go.etcd.io/bbolt" ) // dbType is the database type name for this driver. @@ -22,10 +23,14 @@ const dbType = "bdb" // TestCreateOpenFail ensures that errors related to creating and opening a // database are handled properly. func TestCreateOpenFail(t *testing.T) { + opts := &bbolt.Options{ + NoFreelistSync: true, + } + // Ensure that attempting to open a database that doesn't exist returns // the expected error. wantErr := walletdb.ErrDbDoesNotExist - if _, err := walletdb.Open(dbType, "noexist.db", true); err != wantErr { + if _, err := walletdb.Open(dbType, "noexist.db", opts); err != wantErr { t.Errorf("Open: did not receive expected error - got %v, "+ "want %v", err, wantErr) return @@ -34,7 +39,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure that attempting to open a database with the wrong number of // parameters returns the expected error. wantErr = fmt.Errorf("invalid arguments to %s.Open -- expected "+ - "database path and no-freelist-sync option", dbType) + "database path and *bbolt.Option option", dbType) if _, err := walletdb.Open(dbType, 1, 2, 3); err.Error() != wantErr.Error() { t.Errorf("Open: did not receive expected error - got %v, "+ "want %v", err, wantErr) @@ -54,7 +59,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure that attempting to create a database with the wrong number of // parameters returns the expected error. wantErr = fmt.Errorf("invalid arguments to %s.Create -- expected "+ - "database path and no-freelist-sync option", dbType) + "database path and *bbolt.Option option", dbType) if _, err := walletdb.Create(dbType, 1, 2, 3); err.Error() != wantErr.Error() { t.Errorf("Create: did not receive expected error - got %v, "+ "want %v", err, wantErr) @@ -81,7 +86,7 @@ func TestCreateOpenFail(t *testing.T) { defer os.Remove(tempDir) dbPath := filepath.Join(tempDir, "db") - db, err := walletdb.Create(dbType, dbPath, true) + db, err := walletdb.Create(dbType, dbPath, opts) if err != nil { t.Errorf("Create: unexpected error: %v", err) return @@ -99,6 +104,10 @@ func TestCreateOpenFail(t *testing.T) { // TestPersistence ensures that values stored are still valid after closing and // reopening the database. func TestPersistence(t *testing.T) { + opts := &bbolt.Options{ + NoFreelistSync: true, + } + // Create a new database to run tests against. tempDir, err := ioutil.TempDir("", "persistencetest") if err != nil { @@ -108,7 +117,7 @@ func TestPersistence(t *testing.T) { defer os.Remove(tempDir) dbPath := filepath.Join(tempDir, "db") - db, err := walletdb.Create(dbType, dbPath, true) + db, err := walletdb.Create(dbType, dbPath, opts) if err != nil { t.Errorf("Failed to create test database (%s) %v", dbType, err) return @@ -144,7 +153,7 @@ func TestPersistence(t *testing.T) { // Close and reopen the database to ensure the values persist. db.Close() - db, err = walletdb.Open(dbType, dbPath, true) + db, err = walletdb.Open(dbType, dbPath, opts) if err != nil { t.Errorf("Failed to open test database (%s) %v", dbType, err) return diff --git a/walletdb/bdb/interface_test.go b/walletdb/bdb/interface_test.go index 3755b1d690..52de418143 100644 --- a/walletdb/bdb/interface_test.go +++ b/walletdb/bdb/interface_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/btcsuite/btcwallet/walletdb/walletdbtest" + "go.etcd.io/bbolt" ) // TestInterface performs all interfaces tests for this database driver. @@ -32,5 +33,9 @@ func TestInterface(t *testing.T) { dbPath := filepath.Join(tempDir, "db") defer os.RemoveAll(dbPath) - walletdbtest.TestInterface(t, dbType, dbPath, true) + + opts := &bbolt.Options{ + NoFreelistSync: true, + } + walletdbtest.TestInterface(t, dbType, dbPath, opts) } diff --git a/walletdb/db_test.go b/walletdb/db_test.go index 30c4341bc8..f47ac3769e 100644 --- a/walletdb/db_test.go +++ b/walletdb/db_test.go @@ -13,6 +13,7 @@ import ( "github.com/btcsuite/btcwallet/walletdb" _ "github.com/btcsuite/btcwallet/walletdb/bdb" + "go.etcd.io/bbolt" ) var ( @@ -64,7 +65,10 @@ func TestAddDuplicateDriver(t *testing.T) { defer os.Remove(tempDir) dbPath := filepath.Join(tempDir, "db") - db, err := walletdb.Create(dbType, dbPath, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Create(dbType, dbPath, opts) if err != nil { t.Errorf("failed to create database: %v", err) return diff --git a/walletdb/example_test.go b/walletdb/example_test.go index 4cb513425d..c85e41c3b0 100644 --- a/walletdb/example_test.go +++ b/walletdb/example_test.go @@ -12,6 +12,7 @@ import ( "github.com/btcsuite/btcwallet/walletdb" _ "github.com/btcsuite/btcwallet/walletdb/bdb" + "go.etcd.io/bbolt" ) // This example demonstrates creating a new database. @@ -28,7 +29,10 @@ func ExampleCreate() { // this, but it's done here in the example to ensure the example cleans // up after itself. dbPath := filepath.Join(os.TempDir(), "examplecreate.db") - db, err := walletdb.Create("bdb", dbPath, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Create("bdb", dbPath, opts) if err != nil { fmt.Println(err) return @@ -47,7 +51,10 @@ var exampleNum = 0 func exampleLoadDB() (walletdb.DB, func(), error) { dbName := fmt.Sprintf("exampleload%d.db", exampleNum) dbPath := filepath.Join(os.TempDir(), dbName) - db, err := walletdb.Create("bdb", dbPath, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Create("bdb", dbPath, opts) if err != nil { return nil, nil, err } @@ -111,7 +118,10 @@ func Example_basicUsage() { // this, but it's done here in the example to ensure the example cleans // up after itself. dbPath := filepath.Join(os.TempDir(), "exampleusage.db") - db, err := walletdb.Create("bdb", dbPath, true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Create("bdb", dbPath, opts) if err != nil { fmt.Println(err) return diff --git a/walletdb/go.mod b/walletdb/go.mod index 59f6db3d65..87aa5836b0 100644 --- a/walletdb/go.mod +++ b/walletdb/go.mod @@ -4,9 +4,7 @@ go 1.12 require ( github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f - github.com/coreos/bbolt v1.3.3 github.com/davecgh/go-spew v1.1.1 - go.etcd.io/bbolt v1.3.3 // indirect - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + go.etcd.io/bbolt v1.3.3 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect ) diff --git a/walletdb/go.sum b/walletdb/go.sum index ca356ac361..45d75650cd 100644 --- a/walletdb/go.sum +++ b/walletdb/go.sum @@ -1,12 +1,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/wtxmgr/go.mod b/wtxmgr/go.mod index 9d04e242cd..d69178c993 100644 --- a/wtxmgr/go.mod +++ b/wtxmgr/go.mod @@ -7,4 +7,5 @@ require ( github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/btcsuite/btcwallet/walletdb v1.2.0 + go.etcd.io/bbolt v1.3.3 ) diff --git a/wtxmgr/tx_test.go b/wtxmgr/tx_test.go index 84a60233d1..a40fbb9770 100644 --- a/wtxmgr/tx_test.go +++ b/wtxmgr/tx_test.go @@ -19,6 +19,7 @@ import ( "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwallet/walletdb" _ "github.com/btcsuite/btcwallet/walletdb/bdb" + "go.etcd.io/bbolt" ) // Received transaction output for mainnet outpoint @@ -51,7 +52,10 @@ func testDB() (walletdb.DB, func(), error) { if err != nil { return nil, func() {}, err } - db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), opts) return db, func() { os.RemoveAll(tmpDir) }, err } @@ -63,7 +67,10 @@ func testStore() (*Store, walletdb.DB, func(), error) { return nil, nil, func() {}, err } - db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), true) + opts := &bbolt.Options{ + NoFreelistSync: true, + } + db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), opts) if err != nil { os.RemoveAll(tmpDir) return nil, nil, nil, err From d0cfd3720fbcb1789bd6e88e53440a29a9df967c Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Mon, 4 May 2020 15:49:39 -0700 Subject: [PATCH 2/2] walletdb/bdb: attempt to set initial memory map size when opening In this commit, we start to set the initial memory map size when opening the database. This helps with bulk insertions or migrations of the database, as if we set this to a larger value, then bbolt needs to remap less often. Remapping can be memory intensive as it needs to copy over the entire existing database. For 32-bit systems, we take care to clamp this value to ensure we don't exceed the largest addressable memory on such systems. --- walletdb/bdb/db.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/walletdb/bdb/db.go b/walletdb/bdb/db.go index c96d14685d..430c963065 100644 --- a/walletdb/bdb/db.go +++ b/walletdb/bdb/db.go @@ -365,6 +365,18 @@ func fileExists(name string) bool { return true } +const ( + // is64Bit tells us if we're running on a 64-bit system or not. If uintptr is + // 32 bits, then its complement will be 0xffffffff rather than + // 0xffffffffffffffff. The complement of uint64(0) will always be + // 0xffffffffffffffff. + is64Bit = uint64(^uintptr(0)) == ^uint64(0) + + // max32BitMapSize is the largest mmap size we can support on 32-bit + // systems. (2^31)-1 == 0x7FFFFFFF. + max32BitMapSize = 0x7FFFFFFF +) + // openDB opens the database at the provided path. walletdb.ErrDbDoesNotExist // is returned if the database doesn't exist and the create flag is not set. func openDB(dbPath string, create bool, options *bbolt.Options) (walletdb.DB, error) { @@ -375,6 +387,30 @@ func openDB(dbPath string, create bool, options *bbolt.Options) (walletdb.DB, er // Specify bbolt freelist options to reduce heap pressure in case the // freelist grows to be very large. options.FreelistType = bbolt.FreelistMapType + + if !create { + // The other value that we want to set is the initial memory + // map size. When bolt expands the mmap, it actually copies + // over the entire database. By setting this to a large value + // than zero, then we can avoid some of that heap pressure due + // to the copies. + dbFileInfo, err := os.Stat(dbPath) + if err != nil { + return nil, err + } + + // We'll aim to set the initial memory map to 2x the size of + // the database as it exists on disk, meaning the DB size can + // double without us having to remap everything. + mmapSize := dbFileInfo.Size() * 2 + + // If we're on a 32-bit system, then we'll need to limit this + // value to ensure we don't run into errors down the line. + if !is64Bit { + mmapSize = max32BitMapSize + } + + options.InitialMmapSize = int(mmapSize) } boltDB, err := bbolt.Open(dbPath, 0600, options)