Add more options from 6.11.4 API

rocksdb/_rocksdb.pyx

@@ -3,6 +3,7 @@ import cython
 from libcpp.string cimport string
 from libcpp.deque cimport deque
 from libcpp.vector cimport vector
+from libcpp.map cimport map
 from cpython cimport bool as py_bool
 from libcpp cimport bool as cpp_bool
 from libc.stdint cimport uint32_t
@@ -31,6 +32,7 @@ from . cimport table_factory
 from . cimport memtablerep
 from . cimport universal_compaction
 from . cimport transaction_db
+from . cimport metadata
 
 # Enums are the only exception for direct imports
 # Their name als already unique enough
@@ -1113,6 +1115,24 @@ cdef class ColumnFamilyOptions(object):
         def __set__(self, value):
             self.copts.max_bytes_for_level_multiplier_additional = value
 
+    property soft_rate_limit:
+        def __get__(self):
+            return self.copts.soft_rate_limit
+        def __set__(self, value):
+            self.copts.soft_rate_limit = value
+
+    property hard_rate_limit:
+        def __get__(self):
+            return self.copts.hard_rate_limit
+        def __set__(self, value):
+            self.copts.hard_rate_limit = value
+
+    property rate_limit_delay_max_milliseconds:
+        def __get__(self):
+            return self.copts.rate_limit_delay_max_milliseconds
+        def __set__(self, value):
+            self.copts.rate_limit_delay_max_milliseconds = value
+
     property arena_block_size:
         def __get__(self):
             return self.copts.arena_block_size
@@ -1125,6 +1145,12 @@ cdef class ColumnFamilyOptions(object):
         def __set__(self, value):
             self.copts.disable_auto_compactions = value
 
+    property purge_redundant_kvs_while_flush:
+        def __get__(self):
+            return self.copts.purge_redundant_kvs_while_flush
+        def __set__(self, value):
+            self.copts.purge_redundant_kvs_while_flush = value
+
     # FIXME: remove to util/options_helper.h
     #  property allow_os_buffer:
         #  def __get__(self):
@@ -1400,12 +1426,6 @@ cdef class Options(ColumnFamilyOptions):
         def __set__(self, value):
             self.opts.base_background_compactions = value
 
-    property max_background_compactions:
-        def __get__(self):
-            return self.opts.max_background_compactions
-        def __set__(self, value):
-            self.opts.max_background_compactions = value
-
     property max_subcompactions:
         def __get__(self):
             return self.opts.max_subcompactions
@@ -1472,6 +1492,18 @@ cdef class Options(ColumnFamilyOptions):
         def __set__(self, value):
             self.opts.manifest_preallocation_size = value
 
+    property enable_write_thread_adaptive_yield:
+        def __get__(self):
+            return self.opts.enable_write_thread_adaptive_yield
+        def __set__(self, value):
+            self.opts.enable_write_thread_adaptive_yield = value
+
+    property allow_concurrent_memtable_write:
+        def __get__(self):
+            return self.opts.allow_concurrent_memtable_write
+        def __set__(self, value):
+            self.opts.allow_concurrent_memtable_write = value
+
     property allow_mmap_reads:
         def __get__(self):
             return self.opts.allow_mmap_reads
@@ -1508,6 +1540,12 @@ cdef class Options(ColumnFamilyOptions):
         def __set__(self, value):
             self.opts.is_fd_close_on_exec = value
 
+    property skip_log_error_on_recovery:
+        def __get__(self):
+            return self.opts.skip_log_error_on_recovery
+        def __set__(self, value):
+            self.opts.skip_log_error_on_recovery = value
+
     property stats_dump_period_sec:
         def __get__(self):
             return self.opts.stats_dump_period_sec
@@ -2464,8 +2502,26 @@ cdef class DB(object):
         else:
             return None
 
+    # def get_map_property(self, prop, ColumnFamilyHandle column_family=None):
+    #     cdef map[string, string] value
+    #     cdef Slice c_prop = bytes_to_slice(prop)
+    #     cdef cpp_bool ret = False
+    #     cdef db.ColumnFamilyHandle* cf_handle = NULL
+    #     if column_family:
+    #         cf_handle = column_family.get_handle()
+    #     else:
+    #         cf_handle = self.db.DefaultColumnFamily()
+
+    #     with nogil:
+    #         ret = self.db.GetMapProperty(cf_handle, c_prop, cython.address(value))
+
+    #     if ret:
+    #         return value
+    #     else:
+    #         return None
+
     def get_live_files_metadata(self):
-        cdef vector[db.LiveFileMetaData] metadata
+        cdef vector[metadata.LiveFileMetaData] metadata
 
         with nogil:
             self.wrapped_db.GetLiveFilesMetaData(cython.address(metadata))
@@ -2486,7 +2542,7 @@ cdef class DB(object):
         return ret
 
     def get_column_family_meta_data(self, ColumnFamilyHandle column_family=None):
-        cdef db.ColumnFamilyMetaData metadata
+        cdef metadata.ColumnFamilyMetaData metadata
 
         cdef db.ColumnFamilyHandle* cf_handle = self.wrapped_db.DefaultColumnFamily()
         if column_family:
@@ -2644,6 +2700,18 @@ def repair_db(db_name, Options opts):
     st = db.RepairDB(db_path, deref(opts.opts))
     check_status(st)
 
+# TODO Figure out API to add descriptors. See constructor which deals with vector of ColumnFamilyDescriptor
+# def repair_db(db_name, Options opts, descriptors):
+#     cdef Status st
+#     cdef string db_path
+#     cdef vector[db.ColumnFamilyDescriptor] c_descriptors
+
+#     for d in descriptors:
+#         c_descriptors.push_back(<db.ColumnFamilyDescriptor>(d))
+
+#     db_path = path_to_string(db_name)
+#     st = db.RepairDB(db_path, deref(opts.opts), c_descriptors)
+#     check_status(st)
 
 def list_column_families(db_name, Options opts):
     cdef Status st

rocksdb/db.pxd

@@ -5,9 +5,18 @@ from libcpp cimport bool as cpp_bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 from .types cimport SequenceNumber
+from libcpp.map cimport map
+from libcpp.unordered_map cimport unordered_map
+from libcpp.memory cimport shared_ptr
+from .types cimport SequenceNumber
 from .slice_ cimport Slice
 from .snapshot cimport Snapshot
 from .iterator cimport Iterator
+from .env cimport Env
+from .metadata cimport ColumnFamilyMetaData
+from .metadata cimport LiveFileMetaData
+from .metadata cimport ExportImportFilesMetaData
+from .table_properties cimport TableProperties
 
 cdef extern from "rocksdb/write_batch.h" namespace "rocksdb":
     cdef cppclass WriteBatch:
@@ -40,8 +49,15 @@ cdef extern from "cpp/write_batch_iter_helper.hpp" namespace "py_rocks":
 
 
 cdef extern from "rocksdb/db.h" namespace "rocksdb":
+    ctypedef uint64_t SequenceNumber
     string kDefaultColumnFamilyName
 
+    #todo TableProperties
+    ctypedef unordered_map[string, shared_ptr[const TableProperties]] TablePropertiesCollection
+
+    cdef struct GetMergeOperandsOptions:
+        uint64_t expected_max_number_of_operands
+
     cdef struct LiveFileMetaData:
         string name
         int level
@@ -67,12 +83,6 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
     #     string largestkey
     #     LiveFileMetaData files
 
-    cdef struct ColumnFamilyMetaData:
-        uint64_t size
-        uint64_t file_count
-        # string largestkey
-        # LevelMetaData levels
-
     cdef cppclass Range:
         Range(const Slice&, const Slice&)
 
@@ -88,6 +98,12 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
             ColumnFamilyHandle*,
             const Slice&) nogil except+
 
+        Status DeleteRange(
+            const options.WriteOptions&,
+            ColumnFamilyHandle*,
+            const Slice&,
+            const Slice&) nogil except+
+
         Status Merge(
             const options.WriteOptions&,
             ColumnFamilyHandle*,
@@ -104,6 +120,14 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
             const Slice&,
             string*) nogil except+
 
+        # Status GetMergeOperands(
+        #     const options.ReadOptions&,
+        #     ColumnFamilyHandle*,
+        #     const Slice&,
+        #     PinnableSlice*,
+        #     GetMergeOperandsOptions*,
+        #     uint64_t*) nogil except+;
+
         vector[Status] MultiGet(
             const options.ReadOptions&,
             const vector[ColumnFamilyHandle*]&,
@@ -141,6 +165,22 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
             const Slice&,
             string*) nogil except+
 
+        cpp_bool GetMapProperty(
+            ColumnFamilyHandle*,
+            const Slice&,
+            map[string, string]*) nogil except+
+
+        cpp_bool GetIntProperty(
+            ColumnFamilyHandle*,
+            const Slice&,
+            uint64_t*) nogil except+
+
+        Status ResetStats() nogil except+
+
+        cpp_bool GetAggregatedIntProperty(
+            const Slice&,
+            uint64_t*) nogil except+
+
         void GetApproximateSizes(
             ColumnFamilyHandle*,
             const Range*
@@ -153,35 +193,116 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
             const Slice*,
             const Slice*) nogil except+
 
+        Status SetOptions(
+            ColumnFamilyHandle*,
+            const unordered_map[string, string]&) nogil except+
+
+        Status EnableAutoCompaction(
+            const vector[ColumnFamilyHandle*]&) nogil except+
+
+        void DisableManualCompaction() nogil except+
+        void EnableManualCompaction() nogil except+
+
         Status CreateColumnFamily(
             const options.ColumnFamilyOptions&,
             const string&,
             ColumnFamilyHandle**) nogil except+
 
+        Status CreateColumnFamilies(
+            const options.ColumnFamilyOptions&,
+            const vector[string]&,
+            vector[ColumnFamilyHandle*]*) nogil except+
+
+        Status CreateColumnFamilies(
+            const vector[ColumnFamilyDescriptor]&,
+            vector[ColumnFamilyHandle*]*) nogil except+
+
         Status DropColumnFamily(
             ColumnFamilyHandle*) nogil except+
 
+        Status DropColumnFamilies(
+            vector[ColumnFamilyHandle*]*) nogil except+
+
+        Status DestroyColumnFamilyHandle(
+            ColumnFamilyHandle*) nogil except+
+
         int NumberLevels(ColumnFamilyHandle*) nogil except+
         int MaxMemCompactionLevel(ColumnFamilyHandle*) nogil except+
         int Level0StopWriteTrigger(ColumnFamilyHandle*) nogil except+
         const string& GetName() nogil except+
+        Env* GetEnv() nogil except+
+        # TODO Mandar FileSystem* GetFileSystem()  nogil except+
         const options.Options& GetOptions(ColumnFamilyHandle*) nogil except+
         Status Flush(const options.FlushOptions&, ColumnFamilyHandle*) nogil except+
+        Status Flush(
+            const options.FlushOptions&,
+            const vector[ColumnFamilyHandle*]&) nogil except+
+        Status FlushWAL(bool_cpp) nogil except+
+        Status SyncWAL() nogil except+
+        Status LockWAL() nogil except+
+        Status UnlockWAL() nogil except+
+
+        SequenceNumber GetLatestSequenceNumber()
+        cpp_bool SetPreserveDeletesSequenceNumber(SequenceNumber) nogil except+
+
         Status DisableFileDeletions() nogil except+
         Status EnableFileDeletions() nogil except+
+
         Status Close() nogil except+
+        Status Resume() nogil except+
+        Status PauseBackgroundWork() nogil except+
+        Status ContinueBackgroundWork() nogil except+
 
+        Status GetDbIdentity(string&) nogil except+
+        ColumnFamilyHandle* DefaultColumnFamily()
+
+        # Following defined for #ifndef ROCKDDB_LITE
+        Status GetLiveFiles(vector[string]&,
+                            uint64_t*,
+                            cpp_bool) nogil except+
         # TODO: Status GetSortedWalFiles(VectorLogPtr& files)
-        # TODO: SequenceNumber GetLatestSequenceNumber()
+        # TODO: Status GetCurrentWalFile(std::unique_ptr<LogFile>*)
         # TODO: Status GetUpdatesSince(
-                  # SequenceNumber seq_number,
-                  # unique_ptr[TransactionLogIterator]*)
-
+        #    SequenceNumber seq_number,
+        #    unique_ptr[TransactionLogIterator]*,
+        #    const TransactionLogIterator::ReadOptions&)
+        Status GetCreationTimeOfOldestFile(uint64_t*) nogil except+
         Status DeleteFile(string) nogil except+
         void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+
         void GetColumnFamilyMetaData(ColumnFamilyHandle*, ColumnFamilyMetaData*) nogil except+
-        ColumnFamilyHandle* DefaultColumnFamily()
 
+        Status IngestExternalFile(
+            ColumnFamilyHandle*,
+            const vector[string]&,
+            const options.IngestExternalFileOptions&) nogil except+
+        Status CreateColumnFamilyWithImport(
+            const options.ColumnFamilyOptions&,
+            const string&,
+            const options.ImportColumnFamilyOptions&,
+            const ExportImportFilesMetaData&,
+            ColumnFamilyHandle**) nogil except+
+        Status VerifyChecksum(const options.ReadOptions&) nogil except+
+
+        DB* GetRootDB() nogil except+
+        Status GetPropertiesOfAllTables(
+            ColumnFamilyHandle*,
+            TablePropertiesCollection*) nogil except+
+        Status GetPropertiesOfTablesInRange(
+            ColumnFamilyHandle*, const Range*, size_t,
+            TablePropertiesCollection*) nogil except+
+
+        Status SuggestCompactRange(ColumnFamilyHandle*,
+                                     const Slice*,
+                                     const Slice*) nogil except+
+        Status PromoteL0(ColumnFamilyHandle*, int) nogil except+
+        #TODO Status StartTrace(const TraceOptions&,
+        #                     std::unique_ptr<TraceWriter>&&) nogil except+
+        Status EndTrace() nogil except+
+        # TODO Status StartBlockCacheTrace(
+        #     const TraceOptions&,
+        #     std::unique_ptr<TraceWriter>&&) nogil except+
+        Status EndBlockCacheTrace() nogil except+
+        Status TryCatchUpWithPrimary() nogil except+
 
     cdef Status DB_Open "rocksdb::DB::Open"(
         const options.Options&,
@@ -209,7 +330,13 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
         DB**,
         cpp_bool) nogil except+
 
+    cdef Status DestroyDB(
+            const string&,
+            const options.Options&,
+            const vector[ColumnFamilyDescriptor]&) nogil except+
+
     cdef Status RepairDB(const string& dbname, const options.Options&)
+    cdef Status RepairDB(const string& dbname, const options.Options&, const vector[ColumnFamilyDescriptor]&)
 
     cdef Status ListColumnFamilies "rocksdb::DB::ListColumnFamilies" (
         const options.Options&,
@@ -223,7 +350,7 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
     cdef cppclass ColumnFamilyDescriptor:
         ColumnFamilyDescriptor() nogil except+
         ColumnFamilyDescriptor(
-	    const string&,
+	        const string&,
             const options.ColumnFamilyOptions&) nogil except+
         string name
         options.ColumnFamilyOptions options