diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index cc4caae..c9ae11e 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -969,7 +969,6 @@ cdef class ColumnFamilyOptions(object): copts.parallel_threads = value['parallel_threads'] if 'enabled' in value: copts.enabled = value['enabled'] - property compaction_pri: def __get__(self): @@ -1368,6 +1367,18 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.max_open_files = value + property max_file_opening_threads: + def __get__(self): + return self.opts.max_file_opening_threads + def __set__(self, value): + self.opts.max_file_opening_threads = value + + property max_total_wal_size: + def __get__(self): + return self.opts.max_total_wal_size + def __set__(self, value): + self.opts.max_total_wal_size = value + property use_fsync: def __get__(self): return self.opts.use_fsync @@ -1392,17 +1403,29 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.delete_obsolete_files_period_micros = value + property max_background_jobs: + def __get__(self): + return self.opts.max_background_jobs + def __set__(self, value): + self.opts.max_background_jobs = value + + property base_background_compactions: + def __get__(self): + return self.opts.base_background_compactions + def __set__(self, value): + self.opts.base_background_compactions = value + property max_background_compactions: def __get__(self): return self.opts.max_background_compactions def __set__(self, value): self.opts.max_background_compactions = value - property max_background_jobs: + property max_subcompactions: def __get__(self): - return self.opts.max_background_jobs + return self.opts.max_subcompactions def __set__(self, value): - self.opts.max_background_jobs = value + self.opts.max_subcompactions = value property max_background_flushes: def __get__(self): @@ -1428,6 +1451,12 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.keep_log_file_num = value + property recycle_log_file_num: + def __get__(self): + return self.opts.recycle_log_file_num + def __set__(self, value): + self.opts.recycle_log_file_num = value + property max_manifest_file_size: def __get__(self): return self.opts.max_manifest_file_size @@ -1458,18 +1487,6 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.manifest_preallocation_size = value - property enable_write_thread_adaptive_yield: - def __get__(self): - return self.opts.enable_write_thread_adaptive_yield - def __set__(self, value): - self.opts.enable_write_thread_adaptive_yield = value - - property allow_concurrent_memtable_write: - def __get__(self): - return self.opts.allow_concurrent_memtable_write - def __set__(self, value): - self.opts.allow_concurrent_memtable_write = value - property allow_mmap_reads: def __get__(self): return self.opts.allow_mmap_reads @@ -1482,6 +1499,24 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.allow_mmap_writes = value + property use_direct_reads: + def __get__(self): + return self.opts.use_direct_reads + def __set__(self, value): + self.opts.use_direct_reads = value + + property use_direct_io_for_flush_and_compaction: + def __get__(self): + return self.opts.use_direct_io_for_flush_and_compaction + def __set__(self, value): + self.opts.use_direct_io_for_flush_and_compaction = value + + property allow_fallocate: + def __get__(self): + return self.opts.allow_fallocate + def __set__(self, value): + self.opts.allow_fallocate = value + property is_fd_close_on_exec: def __get__(self): return self.opts.is_fd_close_on_exec @@ -1500,12 +1535,36 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.stats_dump_period_sec = value + property stats_persist_period_sec: + def __get__(self): + return self.opts.stats_persist_period_sec + def __set__(self, value): + self.opts.stats_persist_period_sec = value + + property persist_stats_to_disk: + def __get__(self): + return self.opts.persist_stats_to_disk + def __set__(self, value): + self.opts.persist_stats_to_disk = value + + property stats_history_buffer_size: + def __get__(self): + return self.opts.stats_history_buffer_size + def __set__(self, value): + self.opts.stats_history_buffer_size = value + property advise_random_on_open: def __get__(self): return self.opts.advise_random_on_open def __set__(self, value): self.opts.advise_random_on_open = value + property db_write_buffer_size: + def __get__(self): + return self.opts.db_write_buffer_size + def __set__(self, value): + self.opts.db_write_buffer_size = value + # TODO: need to remove -Wconversion to make this work # property access_hint_on_compaction_start: # def __get__(self): @@ -1513,6 +1572,30 @@ cdef class Options(ColumnFamilyOptions): # def __set__(self, AccessHint value): # self.opts.access_hint_on_compaction_start = value + property new_table_reader_for_compaction_inputs: + def __get__(self): + return self.opts.new_table_reader_for_compaction_inputs + def __set__(self, value): + self.opts.new_table_reader_for_compaction_inputs = value + + property compaction_readahead_size: + def __get__(self): + return self.opts.compaction_readahead_size + def __set__(self, value): + self.opts.compaction_readahead_size = value + + property random_access_max_buffer_size: + def __get__(self): + return self.opts.random_access_max_buffer_size + def __set__(self, value): + self.opts.random_access_max_buffer_size = value + + property writable_file_max_buffer_size: + def __get__(self): + return self.opts.writable_file_max_buffer_size + def __set__(self, value): + self.opts.writable_file_max_buffer_size = value + property use_adaptive_mutex: def __get__(self): return self.opts.use_adaptive_mutex @@ -1525,6 +1608,90 @@ cdef class Options(ColumnFamilyOptions): def __set__(self, value): self.opts.bytes_per_sync = value + property wal_bytes_per_sync: + def __get__(self): + return self.opts.wal_bytes_per_sync + def __set__(self, value): + self.opts.wal_bytes_per_sync = value + + property strict_bytes_per_sync: + def __get__(self): + return self.opts.strict_bytes_per_sync + def __set__(self, value): + self.opts.strict_bytes_per_sync = value + + property enable_thread_tracking: + def __get__(self): + return self.opts.enable_thread_tracking + def __set__(self, value): + self.opts.enable_thread_tracking = value + + property delayed_write_rate: + def __get__(self): + return self.opts.delayed_write_rate + def __set__(self, value): + self.opts.delayed_write_rate = value + + property enable_pipelined_write: + def __get__(self): + return self.opts.enable_pipelined_write + def __set__(self, value): + self.opts.enable_pipelined_write = value + + property unordered_write: + def __get__(self): + return self.opts.unordered_write + def __set__(self, value): + self.opts.unordered_write = value + + property allow_concurrent_memtable_write: + def __get__(self): + return self.opts.allow_concurrent_memtable_write + def __set__(self, value): + self.opts.allow_concurrent_memtable_write = value + + property enable_write_thread_adaptive_yield: + def __get__(self): + return self.opts.enable_write_thread_adaptive_yield + def __set__(self, value): + self.opts.enable_write_thread_adaptive_yield = value + + property max_write_batch_group_size_bytes: + def __get__(self): + return self.opts.max_write_batch_group_size_bytes + def __set__(self, value): + self.opts.max_write_batch_group_size_bytes = value + + property write_thread_max_yield_usec: + def __get__(self): + return self.opts.write_thread_max_yield_usec + def __set__(self, value): + self.opts.write_thread_max_yield_usec = value + + property write_thread_slow_yield_usec: + def __get__(self): + return self.opts.write_thread_slow_yield_usec + def __set__(self, value): + self.opts.write_thread_slow_yield_usec = value + + property skip_stats_update_on_db_open: + def __get__(self): + return self.opts.skip_stats_update_on_db_open + def __set__(self, value): + self.opts.skip_stats_update_on_db_open = value + + property skip_checking_sst_file_sizes_on_db_open: + def __get__(self): + return self.opts.skip_checking_sst_file_sizes_on_db_open + def __set__(self, value): + self.opts.skip_checking_sst_file_sizes_on_db_open = value + + property allow_2pc: + def __get__(self): + return self.opts.allow_2pc + def __set__(self, value): + self.opts.allow_2pc = value + property row_cache: def __get__(self): return self.py_row_cache @@ -1539,6 +1706,84 @@ cdef class Options(ColumnFamilyOptions): self.py_row_cache = value self.opts.row_cache = self.py_row_cache.get_cache() + property fail_if_options_file_error: + def __get__(self): + return self.opts.fail_if_options_file_error + def __set__(self, value): + self.opts.fail_if_options_file_error = value + + property dump_malloc_stats: + def __get__(self): + return self.opts.dump_malloc_stats + def __set__(self, value): + self.opts.dump_malloc_stats = value + + property avoid_flush_during_recovery: + def __get__(self): + return self.opts.avoid_flush_during_recovery + def __set__(self, value): + self.opts.avoid_flush_during_recovery = value + + property avoid_flush_during_shutdown: + def __get__(self): + return self.opts.avoid_flush_during_shutdown + def __set__(self, value): + self.opts.avoid_flush_during_shutdown = value + + property allow_ingest_behind: + def __get__(self): + return self.opts.allow_ingest_behind + def __set__(self, value): + self.opts.allow_ingest_behind = value + + property preserve_deletes: + def __get__(self): + return self.opts.preserve_deletes + def __set__(self, value): + self.opts.preserve_deletes = value + + property two_write_queues: + def __get__(self): + return self.opts.two_write_queues + def __set__(self, value): + self.opts.two_write_queues = value + + property manual_wal_flush: + def __get__(self): + return self.opts.manual_wal_flush + def __set__(self, value): + self.opts.manual_wal_flush = value + + property atomic_flush: + def __get__(self): + return self.opts.atomic_flush + def __set__(self, value): + self.opts.atomic_flush = value + + property avoid_unnecessary_blocking_io: + def __get__(self): + return self.opts.avoid_unnecessary_blocking_io + def __set__(self, value): + self.opts.avoid_unnecessary_blocking_io = value + + property write_dbid_to_manifest: + def __get__(self): + return self.opts.write_dbid_to_manifest + def __set__(self, value): + self.opts.write_dbid_to_manifest = value + + property log_readahead_size: + def __get__(self): + return self.opts.log_readahead_size + def __set__(self, value): + self.opts.log_readahead_size = value + + property best_efforts_recovery: + def __get__(self): + return self.opts.best_efforts_recovery + def __set__(self, value): + self.opts.best_efforts_recovery = value + # Forward declaration cdef class Snapshot diff --git a/rocksdb/tests/test_options.py b/rocksdb/tests/test_options.py index 7d5cc8a..d61cb8d 100644 --- a/rocksdb/tests/test_options.py +++ b/rocksdb/tests/test_options.py @@ -76,15 +76,18 @@ def test_compression_opts(self): with self.assertRaises(TypeError): opts.compression_opts = list(1, 2) - opts.compression_opts = {'window_bits': 1, 'level': 2, 'strategy': 3, 'max_dict_bytes': 4, 'zstd_max_train_bytes': 15, 'parallel_threads': 4, 'enabled': True} - compression_opts = opts.compression_opts - self.assertEqual(compression_opts['window_bits'], 1) - self.assertEqual(compression_opts['level'], 2) - self.assertEqual(compression_opts['strategy'], 3) - self.assertEqual(compression_opts['max_dict_bytes'], 4) - self.assertEqual(compression_opts['zstd_max_train_bytes'], 15) - self.assertEqual(compression_opts['parallel_threads'], 4) - self.assertEqual(compression_opts['enabled'], True) + new_opts = { + 'window_bits': 1, + 'level': 2, + 'strategy': 3, + 'max_dict_bytes': 4, + 'zstd_max_train_bytes': 15, + 'parallel_threads': 4, + 'enabled': True} + opts.compression_opts = new_opts + self.assertIsNot(new_opts, opts.compression_opts) + for key, value in new_opts.items(): + self.assertEqual(opts.compression_opts[key], value) def test_bottommost_compression_opts(self): opts = rocksdb.Options() @@ -102,15 +105,19 @@ def test_bottommost_compression_opts(self): with self.assertRaises(TypeError): opts.compression_opts = list(1, 2) - opts.bottommost_compression_opts = {'window_bits': 1, 'level': 2, 'strategy': 3, 'max_dict_bytes': 4, 'zstd_max_train_bytes': 15, 'parallel_threads': 4, 'enabled': True} - bottommost_compression_opts = opts.bottommost_compression_opts - self.assertEqual(bottommost_compression_opts['window_bits'], 1) - self.assertEqual(bottommost_compression_opts['level'], 2) - self.assertEqual(bottommost_compression_opts['strategy'], 3) - self.assertEqual(bottommost_compression_opts['max_dict_bytes'], 4) - self.assertEqual(bottommost_compression_opts['zstd_max_train_bytes'], 15) - self.assertEqual(bottommost_compression_opts['parallel_threads'], 4) - self.assertEqual(bottommost_compression_opts['enabled'], True) + new_opts = { + 'window_bits': 1, + 'level': 2, + 'strategy': 3, + 'max_dict_bytes': 4, + 'zstd_max_train_bytes': 15, + 'parallel_threads': 4, + 'enabled': True, + } + opts.bottommost_compression_opts = new_opts + self.assertIsNot(new_opts, opts.bottommost_compression_opts) + for key, value in new_opts.items(): + self.assertEqual(opts.bottommost_compression_opts[key], value) def test_simple(self): opts = rocksdb.Options() @@ -195,7 +202,89 @@ def test_compaction_opts_universal(self): self.assertEqual(2, uopts['min_merge_width']) self.assertEqual(30, uopts['max_merge_width']) - def test_row_cache(self): + def test_rockdb_options(self): + NOTNONE = object() + UNSETTABLE = object() + for option, def_value, new_value in ( + ('max_open_files', NOTNONE, 10), + ('row_cache', None, UNSETTABLE), + ('max_file_opening_threads', NOTNONE, 10), + ('max_total_wal_size', NOTNONE, 10), + ('max_background_jobs', NOTNONE, 10), + ('base_background_compactions', NOTNONE, 10), + ('max_background_compactions', NOTNONE, 10), + ('max_subcompactions', NOTNONE, 10), + ('max_background_flushes', NOTNONE, 10), + ('max_log_file_size', NOTNONE, 10), + ('log_file_time_to_roll', NOTNONE, 10), + ('keep_log_file_num', 1000, 10), + ('recycle_log_file_num', NOTNONE, 10), + ('max_manifest_file_size', NOTNONE, 10), + ('table_cache_numshardbits', NOTNONE, 10), + ('wal_ttl_seconds', NOTNONE, 10), + ('wal_size_limit_mb', NOTNONE, 10), + ('manifest_preallocation_size', NOTNONE, 10), + ('allow_mmap_reads', False, True), + ('allow_mmap_writes', False, True), + ('use_direct_reads', False, True), + ('use_direct_io_for_flush_and_compaction', False, True), + ('allow_fallocate', True, False), + ('is_fd_close_on_exec', True, False), + ('skip_log_error_on_recovery', False, True), + ('stats_dump_period_sec', 600, 3600), + ('stats_persist_period_sec', 600, 3600), + ('persist_stats_to_disk', False, True), + ('stats_history_buffer_size', 1024*1024, 1024), + ('advise_random_on_open', True, False), + ('db_write_buffer_size', 0, 100), + ('new_table_reader_for_compaction_inputs', False, True), + ('compaction_readahead_size', 0, 10), + ('random_access_max_buffer_size', 1024*1024, 100), + ('writable_file_max_buffer_size', 1024*1024, 100), + ('use_adaptive_mutex', False, True), + ('bytes_per_sync', 0, 10), + ('wal_bytes_per_sync', 0, 10), + ('strict_bytes_per_sync', False, True), + ('enable_thread_tracking', False, True), + ('delayed_write_rate', 0, 10), + ('enable_pipelined_write', False, True), + ('unordered_write', False, True), + ('allow_concurrent_memtable_write', True, False), + ('enable_write_thread_adaptive_yield', True, False), + ('max_write_batch_group_size_bytes', 1 << 20, 10), + ('write_thread_max_yield_usec', 100, 200), + ('write_thread_slow_yield_usec', 3, 2000), + ('skip_stats_update_on_db_open', False, True), + ('skip_checking_sst_file_sizes_on_db_open', False, True), + ('allow_2pc', False, True), + ('fail_if_options_file_error', False, True), + ('dump_malloc_stats', False, True), + ('avoid_flush_during_recovery', False, True), + ('avoid_flush_during_shutdown', False, True), + ('allow_ingest_behind', False, True), + ('preserve_deletes', False, True), + ('two_write_queues', False, True), + ('manual_wal_flush', False, True), + ('atomic_flush', False, True), + ('avoid_unnecessary_blocking_io', False, True), + ('write_dbid_to_manifest', False, True), + ('log_readahead_size', 0, 10), + ('best_efforts_recovery', False, True), + ): + with self.subTest(option=option): + opts = rocksdb.Options() + if def_value is NOTNONE: + self.assertIsNotNone(getattr(opts, option)) + else: + self.assertEqual(def_value, getattr(opts, option)) + if new_value is UNSETTABLE: + self.assertRaises( + Exception, setattr, opts, option, new_value) + else: + setattr(opts, option, new_value) + self.assertEqual(getattr(opts, option), new_value) + + def test_row_cache_set(self): opts = rocksdb.Options() self.assertIsNone(opts.row_cache) opts.row_cache = cache = rocksdb.LRUCache(2*1024*1024)