From 439fa72741e195b273eaa88a32a676a5a37d2249 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Wed, 12 Jun 2024 19:35:11 +0100 Subject: [PATCH 1/6] Expose function for listing existing column families --- rocksdb/rocksdb.nim | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index 531387f..510a414 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -63,6 +63,40 @@ type writeOpts: WriteOptionsRef ingestOptsPtr: IngestExternalFilesOptionsPtr +proc listRocksDbCFs*( + path: string; + dbOpts = defaultDbOptions(); + ): RocksDBResult[seq[string]] = + ## List exisiting column families on disk. This might be used to find out + ## whether there were some columns missing with the version on disk. + ## + ## Column families previously used must be declared when re-opening an + ## existing database. So this function can be used to add some CFs + ## on-the-fly to the opener list of CFs + ## + ## Note that the on-the-fly adding might not be needed in the way described + ## above once rocksdb has been upgraded to the latest version, see comments + ## at the end of ./columnfamily/cfhandle.nim. + ## + var + lencf: csize_t + errors: cstring + let + cList = rocksdb_list_column_families( + dbOpts.cPtr, + path.cstring, + addr lencf, + cast[cstringArray](errors.addr)) + bailOnErrors(errors) + + var cfs: seq[string] + for n in 0 ..< lencf: + if cList[n].isNil: + return err("short reply") + cfs.add $cList[n] + + ok cfs + proc openRocksDb*( path: string, dbOpts = defaultDbOptions(), From 7930eacaf3c0d208dfd01ea3bccaa8c27701e213 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 13 Jun 2024 12:21:43 +0100 Subject: [PATCH 2/6] Free CF list after use --- rocksdb/rocksdb.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index 510a414..a04afe3 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -90,10 +90,12 @@ proc listRocksDbCFs*( bailOnErrors(errors) var cfs: seq[string] - for n in 0 ..< lencf: - if cList[n].isNil: - return err("short reply") - cfs.add $cList[n] + if not cList.isNil: + for n in 0 ..< lencf: + if cList[n].isNil: + return err("short reply") + cfs.add $cList[n] + rocksdb_free(cList) ok cfs From 29e5c616ba065cfbc919ca5b503f0cf577d6802c Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 13 Jun 2024 12:24:32 +0100 Subject: [PATCH 3/6] Update, fix error case --- rocksdb/rocksdb.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index a04afe3..8a49a19 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -91,11 +91,11 @@ proc listRocksDbCFs*( var cfs: seq[string] if not cList.isNil: + defer: rocksdb_free(cList) for n in 0 ..< lencf: if cList[n].isNil: return err("short reply") cfs.add $cList[n] - rocksdb_free(cList) ok cfs From 03a1ee527b640756e687d0a5725829eb2db5bad8 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 13 Jun 2024 12:29:16 +0100 Subject: [PATCH 4/6] Rename listRocksDbCFs -> listColumnFamilies --- rocksdb/rocksdb.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index 8a49a19..1f1242e 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -63,7 +63,7 @@ type writeOpts: WriteOptionsRef ingestOptsPtr: IngestExternalFilesOptionsPtr -proc listRocksDbCFs*( +proc listColumnFamilies*( path: string; dbOpts = defaultDbOptions(); ): RocksDBResult[seq[string]] = From 7d117086edab7c7ead1103ab9439773231ed0d4c Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 13 Jun 2024 13:26:20 +0100 Subject: [PATCH 5/6] Need to free each item separately --- rocksdb/rocksdb.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index 1f1242e..f16a363 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -92,10 +92,17 @@ proc listColumnFamilies*( var cfs: seq[string] if not cList.isNil: defer: rocksdb_free(cList) + for n in 0 ..< lencf: if cList[n].isNil: + # Clean up the rest + for z in n+1 ..< lencf: + if not cList[z].isNil: + rocksdb_free(cList[z]) return err("short reply") + cfs.add $cList[n] + rocksdb_free(cList[n]) ok cfs From 04b16a723c3ff5afb41a67028d7113f6bd3ad7ea Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 13 Jun 2024 15:52:49 +0100 Subject: [PATCH 6/6] Handle default option objects, close after use --- rocksdb/rocksdb.nim | 54 +++++++++++++++++++++++++++------------ rocksdb/sstfilewriter.nim | 6 +++-- rocksdb/transactiondb.nim | 34 ++++++++++++++++-------- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index f16a363..949de77 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -65,7 +65,7 @@ type proc listColumnFamilies*( path: string; - dbOpts = defaultDbOptions(); + dbOpts = DbOptionsRef(nil); ): RocksDBResult[seq[string]] = ## List exisiting column families on disk. This might be used to find out ## whether there were some columns missing with the version on disk. @@ -78,12 +78,17 @@ proc listColumnFamilies*( ## above once rocksdb has been upgraded to the latest version, see comments ## at the end of ./columnfamily/cfhandle.nim. ## + let + useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) + defer: + if dbOpts.isNil: useDbOpts.close() + var lencf: csize_t errors: cstring let cList = rocksdb_list_column_families( - dbOpts.cPtr, + useDbOpts.cPtr, path.cstring, addr lencf, cast[cstringArray](errors.addr)) @@ -107,17 +112,22 @@ proc listColumnFamilies*( ok cfs proc openRocksDb*( - path: string, - dbOpts = defaultDbOptions(), - readOpts = defaultReadOptions(), - writeOpts = defaultWriteOptions(), - columnFamilies: openArray[ColFamilyDescriptor] = []): RocksDBResult[RocksDbReadWriteRef] = + path: string; + dbOpts = DbOptionsRef(nil); + readOpts = ReadOptionsRef(nil); + writeOpts = WriteOptionsRef(nil); + columnFamilies: openArray[ColFamilyDescriptor] = []; + ): RocksDBResult[RocksDbReadWriteRef] = ## Open a RocksDB instance in read-write mode. If `columnFamilies` is empty ## then it will open the default column family. If `dbOpts`, `readOpts`, or ## `writeOpts` are not supplied then the default options will be used. ## By default, column families will be created if they don't yet exist. ## All existing column families must be specified if the database has ## previously created any column families. + let + useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) + defer: + if dbOpts.isNil: useDbOpts.close() var cfs = columnFamilies.toSeq() if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): @@ -129,7 +139,7 @@ proc openRocksDb*( cfHandles = newSeq[ColFamilyHandlePtr](cfs.len) errors: cstring let rocksDbPtr = rocksdb_open_column_families( - dbOpts.cPtr, + useDbOpts.cPtr, path.cstring, cfNames.len().cint, cast[cstringArray](cfNames[0].addr), @@ -138,7 +148,11 @@ proc openRocksDb*( cast[cstringArray](errors.addr)) bailOnErrors(errors) - let db = RocksDbReadWriteRef( + let + dbOpts = useDbOpts # don't close on exit + readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) + writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts) + db = RocksDbReadWriteRef( lock: createLock(), cPtr: rocksDbPtr, path: path, @@ -151,17 +165,22 @@ proc openRocksDb*( ok(db) proc openRocksDbReadOnly*( - path: string, - dbOpts = defaultDbOptions(), - readOpts = defaultReadOptions(), - columnFamilies: openArray[ColFamilyDescriptor] = [], - errorIfWalFileExists = false): RocksDBResult[RocksDbReadOnlyRef] = + path: string; + dbOpts = DbOptionsRef(nil); + readOpts = ReadOptionsRef(nil); + columnFamilies: openArray[ColFamilyDescriptor] = []; + errorIfWalFileExists = false; + ): RocksDBResult[RocksDbReadOnlyRef] = ## Open a RocksDB instance in read-only mode. If `columnFamilies` is empty ## then it will open the default column family. If `dbOpts` or `readOpts` are ## not supplied then the default options will be used. By default, column ## families will be created if they don't yet exist. If the database already ## contains any column families, then all or a subset of the existing column ## families can be opened for reading. + let + useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) + defer: + if dbOpts.isNil: useDbOpts.close() var cfs = columnFamilies.toSeq() if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): @@ -173,7 +192,7 @@ proc openRocksDbReadOnly*( cfHandles = newSeq[ColFamilyHandlePtr](cfs.len) errors: cstring let rocksDbPtr = rocksdb_open_for_read_only_column_families( - dbOpts.cPtr, + useDbOpts.cPtr, path.cstring, cfNames.len().cint, cast[cstringArray](cfNames[0].addr), @@ -183,7 +202,10 @@ proc openRocksDbReadOnly*( cast[cstringArray](errors.addr)) bailOnErrors(errors) - let db = RocksDbReadOnlyRef( + let + dbOpts = useDbOpts # don't close on exit + readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) + db = RocksDbReadOnlyRef( lock: createLock(), cPtr: rocksDbPtr, path: path, diff --git a/rocksdb/sstfilewriter.nim b/rocksdb/sstfilewriter.nim index 6e7e276..2a78631 100644 --- a/rocksdb/sstfilewriter.nim +++ b/rocksdb/sstfilewriter.nim @@ -30,9 +30,11 @@ type dbOpts: DbOptionsRef proc openSstFileWriter*( - filePath: string, - dbOpts = defaultDbOptions()): RocksDBResult[SstFileWriterRef] = + filePath: string; + dbOpts = DbOptionsRef(nil); + ): RocksDBResult[SstFileWriterRef] = ## Creates a new `SstFileWriterRef` and opens the file at the given `filePath`. + let dbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) doAssert not dbOpts.isClosed() let envOptsPtr = rocksdb_envoptions_create() diff --git a/rocksdb/transactiondb.nim b/rocksdb/transactiondb.nim index 8bfe896..9f072d2 100644 --- a/rocksdb/transactiondb.nim +++ b/rocksdb/transactiondb.nim @@ -47,12 +47,17 @@ type proc openTransactionDb*( path: string, - dbOpts = defaultDbOptions(), - txDbOpts = defaultTransactionDbOptions(), - columnFamilies: openArray[ColFamilyDescriptor] = []): RocksDBResult[TransactionDbRef] = + dbOpts = DbOptionsRef(nil); + txDbOpts = TransactionDbOptionsRef(nil); + columnFamilies: openArray[ColFamilyDescriptor] = []; + ): RocksDBResult[TransactionDbRef] = ## Open a `TransactionDbRef` with the given options and column families. ## If no column families are provided the default column family will be used. ## If no options are provided the default options will be used. + let + useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) + defer: + if dbOpts.isNil: useDbOpts.close() var cfs = columnFamilies.toSeq() if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): @@ -65,7 +70,7 @@ proc openTransactionDb*( errors: cstring let txDbPtr = rocksdb_transactiondb_open_column_families( - dbOpts.cPtr, + useDbOpts.cPtr, txDbOpts.cPtr, path.cstring, cfNames.len().cint, @@ -75,7 +80,10 @@ proc openTransactionDb*( cast[cstringArray](errors.addr)) bailOnErrors(errors) - let db = TransactionDbRef( + let + dbOpts = useDbOpts # don't close on exit + txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts) + db = TransactionDbRef( lock: createLock(), cPtr: txDbPtr, path: path, @@ -89,15 +97,21 @@ proc isClosed*(db: TransactionDbRef): bool {.inline.} = db.cPtr.isNil() proc beginTransaction*( - db: TransactionDbRef, - readOpts = defaultReadOptions(), - writeOpts = defaultWriteOptions(), - txOpts = defaultTransactionOptions(), - columnFamily = DEFAULT_COLUMN_FAMILY_NAME): TransactionRef = + db: TransactionDbRef; + readOpts = ReadOptionsRef(nil); + writeOpts = WriteOptionsRef(nil); + txDbOpts = TransactionDbOptionsRef(nil); + txOpts = defaultTransactionOptions(); + columnFamily = DEFAULT_COLUMN_FAMILY_NAME; + ): TransactionRef = ## Begin a new transaction against the database. The transaction will default ## to using the specified column family. If no column family is specified ## then the default column family will be used. doAssert not db.isClosed() + let + txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts) + readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) + writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts) let txPtr = rocksdb_transaction_begin( db.cPtr,