Add some basic rocksdb options to command line

These options are there mainly to drive experiments, and are therefore
hidden.

nimbus/config.nim

@@ -26,7 +26,8 @@ import
   ],
   eth/[common, net/utils, net/nat, p2p/bootnodes, p2p/enode, p2p/discoveryv5/enr],
   "."/[constants, vm_compile_info, version],
-  common/chain_config
+  common/chain_config,
+  db/opts
 
 export net
 
@@ -380,6 +381,30 @@ type
       defaultValueDesc: $ProtocolFlag.Eth
       name: "protocols" .}: seq[string]
 
+    rocksdbMaxOpenFiles {.
+      hidden
+      defaultValue: defaultMaxOpenFiles
+      defaultValueDesc: $defaultMaxOpenFiles
+      name: "debug-rocksdb-max-open-files".}: int
+
+    rocksdbWriteBufferSize {.
+      hidden
+      defaultValue: defaultWriteBufferSize
+      defaultValueDesc: $defaultWriteBufferSize
+      name: "debug-rocksdb-write-buffer-size".}: int
+
+    rocksdbRowCacheSize {.
+      hidden
+      defaultValue: defaultRowCacheSize
+      defaultValueDesc: $defaultRowCacheSize
+      name: "debug-rocksdb-row-cache-size".}: int
+
+    rocksdbBlockCacheSize {.
+      hidden
+      defaultValue: defaultBlockCacheSize
+      defaultValueDesc: $defaultBlockCacheSize
+      name: "debug-rocksdb-block-cache-size".}: int
+
     case cmd* {.
       command
       defaultValue: NimbusCmd.noCommand }: NimbusCmd
@@ -751,6 +776,14 @@ func httpServerEnabled*(conf: NimbusConf): bool =
 func era1Dir*(conf: NimbusConf): OutDir =
   conf.era1DirOpt.get(OutDir(conf.dataDir.string & "/era1"))
 
+func dbOptions*(conf: NimbusConf): DbOptions =
+  DbOptions.init(
+    maxOpenFiles = conf.rocksdbMaxOpenFiles,
+    writeBufferSize = conf.rocksdbWriteBufferSize,
+    rowCacheSize = conf.rocksdbRowCacheSize,
+    blockCacheSize = conf.rocksdbBlockCacheSize,
+  )
+
 # KLUDGE: The `load()` template does currently not work within any exception
 #         annotated environment.
 {.pop.}

nimbus/db/aristo/aristo_init/persistent.nim

@@ -23,7 +23,8 @@ import
   rocksdb,
   ../aristo_desc,
   ./rocks_db/rdb_desc,
-  "."/[rocks_db, memory_only]
+  "."/[rocks_db, memory_only],
+  ../../opts
 
 export
   RdbBackendRef,
@@ -36,9 +37,10 @@ export
 proc newAristoRdbDbRef(
     basePath: string;
     qidLayout: QidLayoutRef;
+    opts: DbOptions
       ): Result[AristoDbRef, AristoError]=
   let
-    be = ? rocksDbBackend(basePath, qidLayout)
+    be = ? rocksDbBackend(basePath, qidLayout, opts)
     vGen = block:
       let rc = be.getIdgFn()
       if rc.isErr:
@@ -60,6 +62,7 @@ proc init*[W: RdbBackendRef](
     B: type W;
     basePath: string;
     qidLayout: QidLayoutRef;
+    opts: DbOptions
       ): Result[T, AristoError] =
   ## Generic constructor, `basePath` argument is ignored for memory backend
   ## databases (which also unconditionally succeed initialising.)
@@ -70,17 +73,18 @@ proc init*[W: RdbBackendRef](
   ## layouts might render the filter history data unmanageable.
   ##
   when B is RdbBackendRef:
-    basePath.newAristoRdbDbRef qidLayout
+    basePath.newAristoRdbDbRef qidLayout, opts
 
 proc init*[W: RdbBackendRef](
     T: type AristoDbRef;
     B: type W;
     basePath: string;
+    opts: DbOptions
       ): Result[T, AristoError] =
   ## Variant of `init()` using default schedule.
   ##
   when B is RdbBackendRef:
-    basePath.newAristoRdbDbRef DEFAULT_QID_QUEUES.to(QidLayoutRef)
+    basePath.newAristoRdbDbRef DEFAULT_QID_QUEUES.to(QidLayoutRef), opts
 
 proc getRocksDbFamily*(
     gdb: GuestDbRef;

nimbus/db/aristo/aristo_init/rocks_db.nim

@@ -36,11 +36,10 @@ import
   ../aristo_desc/desc_backend,
   ../aristo_blobify,
   ./init_common,
-  ./rocks_db/[rdb_desc, rdb_get, rdb_init, rdb_put, rdb_walk]
+  ./rocks_db/[rdb_desc, rdb_get, rdb_init, rdb_put, rdb_walk],
+  ../../opts
 
 const
-  maxOpenFiles = 512          ## Rocks DB setup, open files limit
-
   extraTraceMessages = false
     ## Enabled additional logging noise
 
@@ -377,13 +376,14 @@ proc closeFn(db: RdbBackendRef): CloseFn =
 proc rocksDbBackend*(
     path: string;
     qidLayout: QidLayoutRef;
+    opts: DbOptions
       ): Result[BackendRef,AristoError] =
   let db = RdbBackendRef(
     beKind: BackendRocksDB)
 
   # Initialise RocksDB
   block:
-    let rc = db.rdb.init(path, maxOpenFiles)
+    let rc = db.rdb.init(path, opts)
     if rc.isErr:
       when extraTraceMessages:
         trace logTxt "constructor failed",

nimbus/db/aristo/aristo_init/rocks_db/rdb_init.nim

@@ -18,7 +18,8 @@ import
   rocksdb,
   results,
   ../../aristo_desc,
-  ./rdb_desc
+  ./rdb_desc,
+  ../../../opts
 
 const
   extraTraceMessages = false
@@ -38,7 +39,7 @@ when extraTraceMessages:
 proc init*(
     rdb: var RdbInst;
     basePath: string;
-    openMax: int;
+    opts: DbOptions;
       ): Result[void,(AristoError,string)] =
   ## Constructor c ode inspired by `RocksStoreRef.init()` from
   ## kvstore_rocksdb.nim
@@ -53,13 +54,29 @@ proc init*(
     return err((RdbBeCantCreateDataDir, ""))
 
   let
-    cfs = @[initColFamilyDescriptor AristoFamily] &
-          RdbGuest.mapIt(initColFamilyDescriptor $it)
-    opts = defaultDbOptions()
-  opts.setMaxOpenFiles(openMax)
+    cfOpts = defaultColFamilyOptions()
+
+  if opts.writeBufferSize > 0:
+    cfOpts.setWriteBufferSize(opts.writeBufferSize)
+
+  let
+    cfs = @[initColFamilyDescriptor(AristoFamily, cfOpts)] &
+          RdbGuest.mapIt(initColFamilyDescriptor($it, cfOpts))
+    dbOpts = defaultDbOptions()
+
+  dbOpts.setMaxOpenFiles(opts.maxOpenFiles)
+  dbOpts.setMaxBytesForLevelBase(opts.writeBufferSize)
+
+  if opts.rowCacheSize > 0:
+    dbOpts.setRowCache(cacheCreateLRU(opts.rowCacheSize))
+
+  if opts.blockCacheSize > 0:
+    let tableOpts = defaultTableOptions()
+    tableOpts.setBlockCache(cacheCreateLRU(opts.rowCacheSize))
+    dbOpts.setBlockBasedTableFactory(tableOpts)
 
   # Reserve a family corner for `Aristo` on the database
-  let baseDb = openRocksDb(dataDir, opts, columnFamilies=cfs).valueOr:
+  let baseDb = openRocksDb(dataDir, dbOpts, columnFamilies=cfs).valueOr:
     let errSym = RdbBeDriverInitError
     when extraTraceMessages:
       trace logTxt "init failed", dataDir, openMax, error=errSym, info=error

nimbus/db/core_db/backend/aristo_rocksdb.nim

@@ -20,7 +20,8 @@ import
   ../../kvt/kvt_persistent as use_kvt,
   ../base,
   ./aristo_db,
-  ./aristo_db/[common_desc, handlers_aristo]
+  ./aristo_db/[common_desc, handlers_aristo],
+  ../../opts
 
 include
   ./aristo_db/aristo_replicate
@@ -37,10 +38,10 @@ const
 # Public constructor
 # ------------------------------------------------------------------------------
 
-proc newAristoRocksDbCoreDbRef*(path: string): CoreDbRef =
+proc newAristoRocksDbCoreDbRef*(path: string, opts: DbOptions): CoreDbRef =
   let
     qlr = QidLayoutRef(nil)
-    adb = AristoDbRef.init(use_ari.RdbBackendRef, path, qlr).expect aristoFail
+    adb = AristoDbRef.init(use_ari.RdbBackendRef, path, qlr, opts).expect aristoFail
     gdb = adb.guestDb().valueOr: GuestDbRef(nil)
     kdb = KvtDbRef.init(use_kvt.RdbBackendRef, path, gdb).expect kvtFail
   AristoDbRocks.create(kdb, adb)

nimbus/db/core_db/persistent.nim

@@ -25,7 +25,8 @@ import
   ../aristo,
   ./memory_only,
   base_iterators_persistent,
-  ./backend/aristo_rocksdb
+  ./backend/aristo_rocksdb,
+  ../opts
 
 export
   memory_only,
@@ -34,13 +35,14 @@ export
 proc newCoreDbRef*(
     dbType: static[CoreDbType];      # Database type symbol
     path: string;                    # Storage path for database
+    opts: DbOptions;
       ): CoreDbRef =
   ## Constructor for persistent type DB
   ##
   ## Note: Using legacy notation `newCoreDbRef()` rather than
   ## `CoreDbRef.init()` because of compiler coughing.
   when dbType == AristoDbRocks:
-    newAristoRocksDbCoreDbRef path
+    newAristoRocksDbCoreDbRef path, opts
 
   else:
     {.error: "Unsupported dbType for persistent newCoreDbRef()".}

nimbus/db/opts.nim

@@ -0,0 +1,32 @@
+import results
+
+export results
+
+const
+  # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning
+  defaultMaxOpenFiles* = 512
+  defaultWriteBufferSize* = 64 * 1024 * 1024
+  defaultRowCacheSize* = 512 * 1024 * 1024
+  defaultBlockCacheSize* = 256 * 1024 * 1024
+
+type
+  DbOptions* = object
+    # Options that are transported to the database layer
+    maxOpenFiles*: int
+    writeBufferSize*: int
+    rowCacheSize*: int
+    blockCacheSize*: int
+
+func init*(
+    T: type DbOptions,
+    maxOpenFiles = defaultMaxOpenFiles,
+    writeBufferSize = defaultWriteBufferSize,
+    rowCacheSize = defaultRowCacheSize,
+    blockCacheSize = defaultBlockCacheSize,
+): T =
+  T(
+    maxOpenFiles: maxOpenFiles,
+    writeBufferSize: writeBufferSize,
+    rowCacheSize: rowCacheSize,
+    blockCacheSize: blockCacheSize,
+  )

nimbus/nimbus.nim

@@ -239,7 +239,8 @@ proc run(nimbus: NimbusNode, conf: NimbusConf) =
     # Resolve statically for database type
     case conf.chainDbMode:
     of Aristo,AriPrune:
-      AristoDbRocks.newCoreDbRef(string conf.dataDir)
+      AristoDbRocks.newCoreDbRef(string conf.dataDir, conf.dbOptions())
+
   let com = CommonRef.new(
     db = coreDB,
     pruneHistory = (conf.chainDbMode == AriPrune),

tests/test_aristo/test_backend.nim

@@ -17,6 +17,7 @@ import
   unittest2,
   stew/endians2,
   ../../nimbus/sync/protocol,
+  ../../nimbus/db/opts,
   ../../nimbus/db/aristo/[
     aristo_blobify,
     aristo_debug,
@@ -28,8 +29,7 @@ import
     aristo_layers,
     aristo_merge,
     aristo_persistent,
-    aristo_tx,
-    aristo_vid],
+    aristo_tx],
   ../replay/xcheck,
   ./test_helpers
 
@@ -279,7 +279,7 @@ proc testBackendConsistency*(
         filTab.clear
       rdb.finish(flush=true)
       if 0 < rdbPath.len:
-        let rc = AristoDbRef.init(RdbBackendRef, rdbPath)
+        let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init())
         xCheckRc rc.error == 0
         rdb = rc.value
       else:

tests/test_aristo/test_filter.nim

@@ -16,6 +16,7 @@ import
   eth/common,
   results,
   unittest2,
+  ../../nimbus/db/opts,
   ../../nimbus/db/aristo/[
     aristo_blobify,
     aristo_check,
@@ -173,7 +174,7 @@ iterator quadripartite(td: openArray[ProofTrieData]): LeafQuartet =
 proc dbTriplet(w: LeafQuartet; rdbPath: string): Result[DbTriplet,AristoError] =
   let db = block:
     if 0 < rdbPath.len:
-      let rc = AristoDbRef.init(RdbBackendRef, rdbPath)
+      let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init())
       xCheckRc rc.error == 0
       rc.value
     else:
@@ -690,7 +691,7 @@ proc testFilterFifo*(
       ): bool =
   let
     db = if 0 < rdbPath.len:
-      let rc = AristoDbRef.init(RdbBackendRef, rdbPath, layout.to(QidLayoutRef))
+      let rc = AristoDbRef.init(RdbBackendRef, rdbPath, layout.to(QidLayoutRef), DbOptions.init())
       xCheckRc rc.error == 0
       rc.value
     else:
@@ -776,7 +777,7 @@ proc testFilterBacklog*(
        ): bool =
   let
     db = if 0 < rdbPath.len:
-      let rc = AristoDbRef.init(RdbBackendRef, rdbPath, layout.to(QidLayoutRef))
+      let rc = AristoDbRef.init(RdbBackendRef, rdbPath, layout.to(QidLayoutRef), DbOptions.init())
       xCheckRc rc.error == 0
       rc.value
     else:

tests/test_aristo/test_tx.nim

@@ -16,6 +16,7 @@ import
   results,
   unittest2,
   stew/endians2,
+  ../../nimbus/db/opts,
   ../../nimbus/db/aristo/[
     aristo_check,
     aristo_debug,
@@ -349,7 +350,8 @@ proc testTxMergeAndDeleteOneByOne*(
     # Start with brand new persistent database.
     db = block:
       if 0 < rdbPath.len:
-        let rc = AristoDbRef.init(RdbBackendRef, rdbPath, qidLayout=TxQidLyo)
+        let rc = AristoDbRef.init(
+          RdbBackendRef, rdbPath, qidLayout=TxQidLyo, DbOptions.init())
         xCheckRc rc.error == 0
         rc.value
       else:
@@ -457,7 +459,8 @@ proc testTxMergeAndDeleteSubTree*(
     # Start with brand new persistent database.
     db = block:
       if 0 < rdbPath.len:
-        let rc = AristoDbRef.init(RdbBackendRef, rdbPath, qidLayout=TxQidLyo)
+        let rc = AristoDbRef.init(
+          RdbBackendRef, rdbPath, qidLayout=TxQidLyo, DbOptions.init())
         xCheckRc rc.error == 0
         rc.value
       else:
@@ -559,7 +562,8 @@ proc testTxMergeProofAndKvpList*(
       db = block:
         # New DB with disabled filter slots management
         if 0 < rdbPath.len:
-          let rc = AristoDbRef.init(RdbBackendRef, rdbPath, QidLayoutRef(nil))
+          let rc = AristoDbRef.init(
+            RdbBackendRef, rdbPath, QidLayoutRef(nil), DbOptions.init())
           xCheckRc rc.error == 0
           rc.value
         else:

tests/test_coredb.nim

@@ -16,6 +16,7 @@ import
   eth/common,
   results,
   unittest2,
+  ../nimbus/db/opts,
   ../nimbus/db/core_db/persistent,
   ../nimbus/core/chain,
   ./replay/pp,
@@ -157,7 +158,7 @@ proc initRunnerDB(
     # Resolve for static `dbType`
     case dbType:
     of AristoDbMemory: AristoDbMemory.newCoreDbRef()
-    of AristoDbRocks: AristoDbRocks.newCoreDbRef path
+    of AristoDbRocks: AristoDbRocks.newCoreDbRef(path, DbOptions.init())
     of AristoDbVoid: AristoDbVoid.newCoreDbRef()
     else: raiseAssert "Oops"