From 58472813218b6c1ca4c816708cadbacbfb54ae68 Mon Sep 17 00:00:00 2001 From: chenjiong Date: Tue, 21 Mar 2023 10:09:12 +0800 Subject: [PATCH 1/7] feat(option): add max_subcompactions option support --- options.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/options.go b/options.go index 07000215..d3ed9705 100644 --- a/options.go +++ b/options.go @@ -1196,6 +1196,17 @@ func (opts *Options) SetOptimizeFiltersForHits(value bool) { C.rocksdb_options_set_optimize_filters_for_hits(opts.c, C.int(btoi(value))) } +// SetMaxSubCompactions set max_subcompactions +// This value represents the maximum number of threads that will +// concurrently perform a compaction job by breaking it into multiple, +// smaller ones that are run simultaneously. +// Default: 1 (i.e. no subcompactions) +// +// Dynamically changeable through SetDBOptions() API. +func (opts *Options) SetMaxSubCompactions(value int) { + C.rocksdb_options_set_max_subcompactions(opts.c, C.uint32_t(value)) +} + // Destroy deallocates the Options object. func (opts *Options) Destroy() { C.rocksdb_options_destroy(opts.c) From 218a8a1808d43b557737bbbaeb5b371f9aab61b5 Mon Sep 17 00:00:00 2001 From: chenjiong Date: Tue, 21 Mar 2023 10:09:43 +0800 Subject: [PATCH 2/7] feat(db): add FlushCF function support --- db.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/db.go b/db.go index 64735c61..918d3ee8 100755 --- a/db.go +++ b/db.go @@ -752,6 +752,17 @@ func (db *DB) Flush(opts *FlushOptions) error { return nil } +// FlushCF triggers a manuel column family flush for the database. +func (db *DB) FlushCF(opts *FlushOptions, cf *ColumnFamilyHandle) error { + var cErr *C.char + C.rocksdb_flush_cf(db.c, opts.c, cf.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + // DisableFileDeletions disables file deletions and should be used when backup the database. func (db *DB) DisableFileDeletions() error { var cErr *C.char From f6dbac5c4de44a331ba2b833e5c753c592ab06ee Mon Sep 17 00:00:00 2001 From: chenjiong Date: Thu, 23 Mar 2023 15:45:33 +0800 Subject: [PATCH 3/7] feat(option): add write buffer manager and rate limiter feature support --- options.go | 19 +++++++++++++++++++ ratelimiter.go | 5 +++++ write_buffer_manager.go | 21 +++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 write_buffer_manager.go diff --git a/options.go b/options.go index d3ed9705..daa8d9ae 100644 --- a/options.go +++ b/options.go @@ -1207,6 +1207,25 @@ func (opts *Options) SetMaxSubCompactions(value int) { C.rocksdb_options_set_max_subcompactions(opts.c, C.uint32_t(value)) } +// SetWriteBufferManager set write_bufffer_manager forthe option +// The memory usage of memtable will report to this object. The same object +// can be passed into multiple DBs and it will track the sum of size of all +// the DBs. If the total size of all live memtables of all the DBs exceeds +// a limit, a flush will be triggered in the next DB to which the next write +// is issued. +// +// If the object is only passed to one DB, the behavior is the same as +// db_write_buffer_size. When write_buffer_manager is set, the value set will +// override db_write_buffer_size. +// +// This feature is disabled by default. Specify a non-zero value +// to enable it. +// +// Default: null +func (opts *Options) SetWriteBufferManager(w *WriteBufferManager) { + C.rocksdb_options_set_write_buffer_manager(opts.c, w.c) +} + // Destroy deallocates the Options object. func (opts *Options) Destroy() { C.rocksdb_options_destroy(opts.c) diff --git a/ratelimiter.go b/ratelimiter.go index 72b53951..7bf44410 100644 --- a/ratelimiter.go +++ b/ratelimiter.go @@ -24,6 +24,11 @@ func NewNativeRateLimiter(c *C.rocksdb_ratelimiter_t) *RateLimiter { return &RateLimiter{c} } +// SetBytesPerSecond set rate limiter bytes per second dynamically +func (self *RateLimiter) SetBytesPerSecond(val int64) { + C.rocksdb_ratelimiter_set_bytes_per_second(self.c, C.int64_t(val)) +} + // Destroy deallocates the RateLimiter object. func (self *RateLimiter) Destroy() { C.rocksdb_ratelimiter_destroy(self.c) diff --git a/write_buffer_manager.go b/write_buffer_manager.go new file mode 100644 index 00000000..bf613a51 --- /dev/null +++ b/write_buffer_manager.go @@ -0,0 +1,21 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +// WriteBufferManager manager the whole write buffer between multiple column families or multiple db +type WriteBufferManager struct { + c *C.rocksdb_write_buffer_manager_t +} + +// NewWriteBufferManager creates a WriteBufferManager object. +func NewWriteBufferManager(bufferSize uint64) *WriteBufferManager { + return &WriteBufferManager{c: C.rocksdb_write_buffer_manager_create(C.size_t(bufferSize))} +} + +// Destroy deallocates the WriterBufferManager object. +func (w *WriteBufferManager) Destroy() { + C.rocksdb_write_buffer_manager_destory(w.c) + w.c = nil +} From af0fb7e0273552d151f084841a1163424b7baee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E7=82=AF?= Date: Mon, 10 Apr 2023 15:15:47 +0800 Subject: [PATCH 4/7] feat(option): add set db option feature support --- db.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/db.go b/db.go index 918d3ee8..edd6a927 100755 --- a/db.go +++ b/db.go @@ -691,6 +691,37 @@ func (db *DB) SetOptions(keys, values []string) error { return nil } +// SetDBOptions dynamically changes options through the SetDBOptions API. +func (db *DB) SetDBOptions(keys, values []string) error { + num_keys := len(keys) + + if num_keys == 0 { + return nil + } + + cKeys := make([]*C.char, num_keys) + cValues := make([]*C.char, num_keys) + for i := range keys { + cKeys[i] = C.CString(keys[i]) + cValues[i] = C.CString(values[i]) + } + + var cErr *C.char + + C.rocksdb_set_dboptions( + db.c, + C.int(num_keys), + &cKeys[0], + &cValues[0], + &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + // LiveFileMetadata is a metadata which is associated with each SST file. type LiveFileMetadata struct { Name string From 37c0131c65545b7ab3a1347ea515bb8734a9a75f Mon Sep 17 00:00:00 2001 From: unknown <80212630@adc.com> Date: Thu, 11 May 2023 10:41:05 +0800 Subject: [PATCH 5/7] fix(option): fix set option and set db option without free c string problem --- db.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/db.go b/db.go index edd6a927..9a5bc17c 100755 --- a/db.go +++ b/db.go @@ -674,6 +674,12 @@ func (db *DB) SetOptions(keys, values []string) error { cKeys[i] = C.CString(keys[i]) cValues[i] = C.CString(values[i]) } + defer func() { + for i := range cKeys { + C.free(unsafe.Pointer(cKeys[i])) + C.free(unsafe.Pointer(cValues[i])) + } + }() var cErr *C.char @@ -705,6 +711,12 @@ func (db *DB) SetDBOptions(keys, values []string) error { cKeys[i] = C.CString(keys[i]) cValues[i] = C.CString(values[i]) } + defer func() { + for i := range cKeys { + C.free(unsafe.Pointer(cKeys[i])) + C.free(unsafe.Pointer(cValues[i])) + } + }() var cErr *C.char From 9dc7a98e7e4e8a67b5f15579d0363d88248abec2 Mon Sep 17 00:00:00 2001 From: unknown <80212630@adc.com> Date: Mon, 29 May 2023 15:40:08 +0800 Subject: [PATCH 6/7] feat(option): add set stats persist option feature support --- options.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/options.go b/options.go index daa8d9ae..430a5683 100644 --- a/options.go +++ b/options.go @@ -918,6 +918,14 @@ func (opts *Options) SetStatsDumpPeriodSec(value uint) { C.rocksdb_options_set_stats_dump_period_sec(opts.c, C.uint(value)) } +// SetStatsPersistPeriodSec sets the stats persist period in seconds. +// +// If not zero, persist stats to LOG every stats_persist_period_sec +// Default: 300 (5 min) +func (opts *Options) SetStatsPersistPeriodSec(value uint) { + C.rocksdb_options_set_stats_persist_period_sec(opts.c, C.uint(value)) +} + // SetAdviseRandomOnOpen specifies whether we will hint the underlying // file system that the file access pattern is random, when a sst file is opened. // Default: true From db8e5fd208949cccece266a1a495467b5db58b40 Mon Sep 17 00:00:00 2001 From: unknown <80212630@adc.com> Date: Mon, 29 May 2023 18:32:18 +0800 Subject: [PATCH 7/7] feat(option): add sst file manager option feature support --- options.go | 21 ++++++++++++++++++++- sst_file_manager.go | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 sst_file_manager.go diff --git a/options.go b/options.go index 430a5683..337498a4 100644 --- a/options.go +++ b/options.go @@ -1215,7 +1215,7 @@ func (opts *Options) SetMaxSubCompactions(value int) { C.rocksdb_options_set_max_subcompactions(opts.c, C.uint32_t(value)) } -// SetWriteBufferManager set write_bufffer_manager forthe option +// SetWriteBufferManager set write_bufffer_manager for the option // The memory usage of memtable will report to this object. The same object // can be passed into multiple DBs and it will track the sum of size of all // the DBs. If the total size of all live memtables of all the DBs exceeds @@ -1234,6 +1234,25 @@ func (opts *Options) SetWriteBufferManager(w *WriteBufferManager) { C.rocksdb_options_set_write_buffer_manager(opts.c, w.c) } +// SetSstFileManager set sst_file_manager for the option +// Use to track SST files and control their file deletion rate. +// +// Features: +// - Throttle the deletion rate of the SST files. +// - Keep track the total size of all SST files. +// - Set a maximum allowed space limit for SST files that when reached +// the DB wont do any further flushes or compactions and will set the +// background error. +// - Can be shared between multiple dbs. +// Limitations: +// - Only track and throttle deletes of SST files in +// first db_path (db_name if db_paths is empty). +// +// Default: null +func (opts *Options) SetSstFileManager(s *SstFileManager) { + C.rocksdb_options_set_sst_file_manager(opts.c, s.c) +} + // Destroy deallocates the Options object. func (opts *Options) Destroy() { C.rocksdb_options_destroy(opts.c) diff --git a/sst_file_manager.go b/sst_file_manager.go new file mode 100644 index 00000000..41ab0927 --- /dev/null +++ b/sst_file_manager.go @@ -0,0 +1,21 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +// SstFileManager manager sst file deletion between multiple column families or multiple db +type SstFileManager struct { + c *C.rocksdb_sst_file_manager_t +} + +// NewSstFileManager creates a SstFileManager object. +func NewSstFileManager(env *Env) *SstFileManager { + return &SstFileManager{c: C.rocksdb_sst_file_manager_create(env.c)} +} + +// Destroy deallocates the SstFileManager object. +func (w *SstFileManager) Destroy() { + C.rocksdb_sst_file_manager_destory(w.c) + w.c = nil +}