From a5c23fa71a26b27fda245971a0d86a79cebd792d Mon Sep 17 00:00:00 2001 From: Mandar Harshe Date: Sun, 14 Nov 2021 21:10:11 +0100 Subject: [PATCH] Add metadata class --- rocksdb/_rocksdb.pyx | 5 ++-- rocksdb/db.pxd | 54 ++++++++++------------------------------ rocksdb/metadata.pxd | 59 ++++++++++++++++++++++++++++++++++++++++++++ rocksdb/options.pxd | 19 +++++++++++++- 4 files changed, 93 insertions(+), 44 deletions(-) create mode 100644 rocksdb/metadata.pxd diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index 0863fc7..18ac906 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -30,6 +30,7 @@ from . cimport env from . cimport table_factory from . cimport memtablerep from . cimport universal_compaction +from . cimport metadata # Enums are the only exception for direct imports # Their name als already unique enough @@ -2120,7 +2121,7 @@ cdef class DB(object): # return None def get_live_files_metadata(self): - cdef vector[db.LiveFileMetaData] metadata + cdef vector[metadata.LiveFileMetaData] metadata with nogil: self.db.GetLiveFilesMetaData(cython.address(metadata)) @@ -2141,7 +2142,7 @@ cdef class DB(object): return ret def get_column_family_meta_data(self, ColumnFamilyHandle column_family=None): - cdef db.ColumnFamilyMetaData metadata + cdef metadata.ColumnFamilyMetaData metadata cdef db.ColumnFamilyHandle* cf_handle = self.db.DefaultColumnFamily() if column_family: diff --git a/rocksdb/db.pxd b/rocksdb/db.pxd index 186d64c..b365ffe 100644 --- a/rocksdb/db.pxd +++ b/rocksdb/db.pxd @@ -11,6 +11,9 @@ from .slice_ cimport Slice from .snapshot cimport Snapshot from .iterator cimport Iterator from .env cimport Env +from .metadata cimport ColumnFamilyMetaData +from .metadata cimport LiveFileMetaData +from .metadata cimport ExportImportFilesMetaData cdef extern from "rocksdb/write_batch.h" namespace "rocksdb": cdef cppclass WriteBatch: @@ -49,37 +52,6 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": # todo TableProperties # ctypedef unordered_map[string, shared_ptr[const TableProperties]] TablePropertiesCollection - cdef struct LiveFileMetaData: - string name - int level - uint64_t size - string smallestkey - string largestkey - SequenceNumber smallest_seqno - SequenceNumber largest_seqno - - # cdef struct SstFileMetaData: - # uint64_t size - # string name - # uint64_t file_number - # string db_path - # string smallestkey - # string largestkey - # SequenceNumber smallest_seqno - # SequenceNumber largest_seqno - - # cdef struct LevelMetaData: - # int level - # uint64_t size - # string largestkey - # LiveFileMetaData files - - cdef struct ColumnFamilyMetaData: - uint64_t size - uint64_t file_count - # string largestkey - # LevelMetaData levels - cdef struct GetMergeOperandsOptions: uint64_t expected_max_number_of_operands @@ -271,16 +243,16 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+ void GetColumnFamilyMetaData(ColumnFamilyHandle*, ColumnFamilyMetaData*) nogil except+ - #TODO Status IngestExternalFile( - # ColumnFamilyHandle*, - # const vector[string]&, - # const options.IngestExternalFileOptions&) nogil except+ - #TODO Status CreateColumnFamilyWithImport( - # const options.ColumnFamilyOptions&, - # const string&, - # const options.ImportColumnFamilyOptions&, - # const options.ExportImportFilesMetaData&, - # ColumnFamilyHandle**) nogil except+ + Status IngestExternalFile( + ColumnFamilyHandle*, + const vector[string]&, + const options.IngestExternalFileOptions&) nogil except+ + Status CreateColumnFamilyWithImport( + const options.ColumnFamilyOptions&, + const string&, + const options.ImportColumnFamilyOptions&, + const ExportImportFilesMetaData&, + ColumnFamilyHandle**) nogil except+ Status VerifyChecksum(const options.ReadOptions&) nogil except+ DB* GetRootDB() nogil except+ diff --git a/rocksdb/metadata.pxd b/rocksdb/metadata.pxd new file mode 100644 index 0000000..9f1c780 --- /dev/null +++ b/rocksdb/metadata.pxd @@ -0,0 +1,59 @@ +from libcpp cimport bool as cpp_bool +from libcpp.string cimport string +from libcpp.vector cimport vector +from libc.stdint cimport uint64_t +from libc.stdint cimport uint32_t +from libc.stdint cimport int64_t +from libc.stdint cimport int32_t +from .logger cimport Logger +from .slice_ cimport Slice +from .snapshot cimport Snapshot +from .db cimport SequenceNumber + +cdef extern from "rocksdb/metadata.h" namespace "rocksdb": + cdef cppclass ColumnFamilyMetaData: + ColumnFamilyMetaData() except+ + ColumnFamilyMetaData(const string&, uint64_t, + const vector[LevelMetaData]&&) except+ + uint64_t size + uint64_t file_count + string name + vector[LevelMetaData] levels + + cdef cppclass LevelMetaData: + int level + uint64_t size + vector[SstFileMetaData] files + + cdef cppclass SstFileMetaData: + uint64_t size + string name + uint64_t file_number + string db_path + SequenceNumber smallest_seqno + SequenceNumber largest_seqno + string smallestkey + string largestkey + uint64_t num_reads_sampled + cpp_bool being_compacted + uint64_t num_entries + uint64_t num_deletions + uint64_t oldest_blob_file_number + uint64_t oldest_ancester_time + uint64_t file_creation_time + string file_checksum + string file_checksum_func_name + + cdef cppclass LiveFileMetaData(SstFileMetaData): + string column_family_name + int level + uint64_t size + string smallestkey + string largestkey + SequenceNumber smallest_seqno + SequenceNumber largest_seqno + + cdef cppclass ExportImportFilesMetaData: + string db_comparator_name + vector[LiveFileMetaData] files + diff --git a/rocksdb/options.pxd b/rocksdb/options.pxd index 8cac3d0..1dcc3b6 100644 --- a/rocksdb/options.pxd +++ b/rocksdb/options.pxd @@ -252,7 +252,6 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": cpp_bool allow_write_stall uint32_t max_subcompactions - cdef cppclass IngestExternalFileOptions: cpp_bool move_files cpp_bool failed_move_fall_back_to_copy @@ -264,3 +263,21 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": cpp_bool verify_checksums_before_ingest size_t verify_checksums_readahead_size cpp_bool verify_file_checksum + + ctypedef enum TraceFilterType: + kTraceFilterNone + kTraceFilterGet + kTraceFilterWrite + + cdef cppclass TraceOptions: + uint64_t max_trace_file_size + uint64_t sampling_frequency + uint64_t filter + + cdef cppclass ImportColumnFamilyOptions: + cpp_bool move_files + + cdef cppclass SizeApproximationOptions: + cpp_bool include_memtabtles + cpp_bool include_files + double files_size_error_margin