diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c11da1..ce37df1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.16) -project(tidesdb_cpp VERSION 2.0.1 LANGUAGES CXX) +project(tidesdb_cpp VERSION 2.1.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -13,7 +13,7 @@ if(MSVC) elseif(MINGW) add_compile_options(-Wall -Wextra) else() - add_compile_options(-Wall -Wextra -Wpedantic) + add_compile_options(-Wall -Wextra -Wpedantic -fpermissive) endif() find_library(TIDESDB_LIBRARY NAMES tidesdb REQUIRED) diff --git a/include/tidesdb/tidesdb.hpp b/include/tidesdb/tidesdb.hpp index 94cc658..3b9b70b 100644 --- a/include/tidesdb/tidesdb.hpp +++ b/include/tidesdb/tidesdb.hpp @@ -41,10 +41,13 @@ namespace tidesdb */ enum class CompressionAlgorithm { - None = NO_COMPRESSION, - LZ4 = LZ4_COMPRESSION, - Zstd = ZSTD_COMPRESSION, - LZ4Fast = LZ4_FAST_COMRESSION + None = TDB_COMPRESS_NONE, +#ifndef __sun + Snappy = TDB_COMPRESS_SNAPPY, +#endif + LZ4 = TDB_COMPRESS_LZ4, + Zstd = TDB_COMPRESS_ZSTD, + LZ4Fast = TDB_COMPRESS_LZ4_FAST }; /** @@ -190,6 +193,24 @@ struct ColumnFamilyConfig * @brief Get default column family configuration from TidesDB */ static ColumnFamilyConfig defaultConfig(); + + /** + * @brief Load column family configuration from an INI file + * @param iniFile Path to the INI file + * @param sectionName Section name in the INI file + * @return Loaded configuration + */ + static ColumnFamilyConfig loadFromIni(const std::string& iniFile, + const std::string& sectionName); + + /** + * @brief Save column family configuration to an INI file + * @param iniFile Path to the INI file + * @param sectionName Section name in the INI file + * @param config Configuration to save + */ + static void saveToIni(const std::string& iniFile, const std::string& sectionName, + const ColumnFamilyConfig& config); }; /** @@ -215,6 +236,13 @@ struct Stats std::vector levelSizes; std::vector levelNumSSTables; std::optional config; + std::uint64_t totalKeys = 0; + std::uint64_t totalDataSize = 0; + double avgKeySize = 0.0; + double avgValueSize = 0.0; + std::vector levelKeyCounts; + double readAmp = 0.0; + double hitRate = 0.0; }; /** @@ -258,6 +286,25 @@ class ColumnFamily */ void flushMemtable(); + /** + * @brief Check if a flush operation is in progress + * @return true if flushing, false otherwise + */ + [[nodiscard]] bool isFlushing() const; + + /** + * @brief Check if a compaction operation is in progress + * @return true if compacting, false otherwise + */ + [[nodiscard]] bool isCompacting() const; + + /** + * @brief Update runtime-safe configuration settings + * @param config New configuration (only runtime-safe fields are applied) + * @param persistToDisk If true, save changes to config.ini + */ + void updateRuntimeConfig(const ColumnFamilyConfig& config, bool persistToDisk = true); + /** * @brief Get the underlying C handle (for internal use) */ @@ -503,9 +550,39 @@ class TidesDB /** * @brief Register a custom comparator * @param name Comparator name + * @param fn Comparator function * @param ctxStr Context string (optional) + * @param ctx Context pointer (optional) + */ + void registerComparator(const std::string& name, tidesdb_comparator_fn fn = nullptr, + const std::string& ctxStr = "", void* ctx = nullptr); + + /** + * @brief Get a registered comparator + * @param name Comparator name + * @param fn Output comparator function + * @param ctx Output context pointer + */ + void getComparator(const std::string& name, tidesdb_comparator_fn* fn, void** ctx); + + /** + * @brief Rename a column family + * @param oldName Current name of the column family + * @param newName New name for the column family + */ + void renameColumnFamily(const std::string& oldName, const std::string& newName); + + /** + * @brief Create a backup of the database + * @param dir Backup directory (must be empty or non-existent) + */ + void backup(const std::string& dir); + + /** + * @brief Get default database configuration + * @return Default Config struct */ - void registerComparator(const std::string& name, const std::string& ctxStr = ""); + static Config defaultConfig(); private: tidesdb_t* db_ = nullptr; diff --git a/src/tidesdb.cpp b/src/tidesdb.cpp index 44c4629..cea7d7d 100644 --- a/src/tidesdb.cpp +++ b/src/tidesdb.cpp @@ -52,7 +52,8 @@ ColumnFamilyConfig ColumnFamilyConfig::defaultConfig() config.minLevels = cConfig.min_levels; config.dividingLevelOffset = cConfig.dividing_level_offset; config.klogValueThreshold = cConfig.klog_value_threshold; - config.compressionAlgorithm = static_cast(cConfig.compression_algo); + config.compressionAlgorithm = + static_cast(static_cast(cConfig.compression_algorithm)); config.enableBloomFilter = cConfig.enable_bloom_filter != 0; config.bloomFPR = cConfig.bloom_fpr; config.enableBlockIndexes = cConfig.enable_block_indexes != 0; @@ -71,6 +72,79 @@ ColumnFamilyConfig ColumnFamilyConfig::defaultConfig() return config; } +ColumnFamilyConfig ColumnFamilyConfig::loadFromIni(const std::string& iniFile, + const std::string& sectionName) +{ + tidesdb_column_family_config_t cConfig; + int result = tidesdb_cf_config_load_from_ini(iniFile.c_str(), sectionName.c_str(), &cConfig); + checkResult(result, "failed to load config from INI"); + + ColumnFamilyConfig config; + config.writeBufferSize = cConfig.write_buffer_size; + config.levelSizeRatio = cConfig.level_size_ratio; + config.minLevels = cConfig.min_levels; + config.dividingLevelOffset = cConfig.dividing_level_offset; + config.klogValueThreshold = cConfig.klog_value_threshold; + config.compressionAlgorithm = + static_cast(static_cast(cConfig.compression_algorithm)); + config.enableBloomFilter = cConfig.enable_bloom_filter != 0; + config.bloomFPR = cConfig.bloom_fpr; + config.enableBlockIndexes = cConfig.enable_block_indexes != 0; + config.indexSampleRatio = cConfig.index_sample_ratio; + config.blockIndexPrefixLen = cConfig.block_index_prefix_len; + config.syncMode = static_cast(cConfig.sync_mode); + config.syncIntervalUs = cConfig.sync_interval_us; + config.comparatorName = cConfig.comparator_name; + config.skipListMaxLevel = cConfig.skip_list_max_level; + config.skipListProbability = cConfig.skip_list_probability; + config.defaultIsolationLevel = static_cast(cConfig.default_isolation_level); + config.minDiskSpace = cConfig.min_disk_space; + config.l1FileCountTrigger = cConfig.l1_file_count_trigger; + config.l0QueueStallThreshold = cConfig.l0_queue_stall_threshold; + + return config; +} + +void ColumnFamilyConfig::saveToIni(const std::string& iniFile, const std::string& sectionName, + const ColumnFamilyConfig& config) +{ + tidesdb_column_family_config_t cConfig; + cConfig.write_buffer_size = config.writeBufferSize; + cConfig.level_size_ratio = config.levelSizeRatio; + cConfig.min_levels = config.minLevels; + cConfig.dividing_level_offset = config.dividingLevelOffset; + cConfig.klog_value_threshold = config.klogValueThreshold; + cConfig.compression_algorithm = + static_cast<::compression_algorithm>(config.compressionAlgorithm); + cConfig.enable_bloom_filter = config.enableBloomFilter ? 1 : 0; + cConfig.bloom_fpr = config.bloomFPR; + cConfig.enable_block_indexes = config.enableBlockIndexes ? 1 : 0; + cConfig.index_sample_ratio = config.indexSampleRatio; + cConfig.block_index_prefix_len = config.blockIndexPrefixLen; + cConfig.sync_mode = static_cast(config.syncMode); + cConfig.sync_interval_us = config.syncIntervalUs; + cConfig.skip_list_max_level = config.skipListMaxLevel; + cConfig.skip_list_probability = config.skipListProbability; + cConfig.default_isolation_level = + static_cast(config.defaultIsolationLevel); + cConfig.min_disk_space = config.minDiskSpace; + cConfig.l1_file_count_trigger = config.l1FileCountTrigger; + cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold; + + std::memset(cConfig.comparator_name, 0, TDB_MAX_COMPARATOR_NAME); + if (!config.comparatorName.empty()) + { + std::strncpy(cConfig.comparator_name, config.comparatorName.c_str(), + TDB_MAX_COMPARATOR_NAME - 1); + } + std::memset(cConfig.comparator_ctx_str, 0, TDB_MAX_COMPARATOR_CTX); + cConfig.comparator_fn_cached = nullptr; + cConfig.comparator_ctx_cached = nullptr; + + int result = tidesdb_cf_config_save_to_ini(iniFile.c_str(), sectionName.c_str(), &cConfig); + checkResult(result, "failed to save config to INI"); +} + //----------------------------------------------------------------------------- // ColumnFamily //----------------------------------------------------------------------------- @@ -118,6 +192,23 @@ Stats ColumnFamily::getStats() const } } + // New stats fields + stats.totalKeys = cStats->total_keys; + stats.totalDataSize = cStats->total_data_size; + stats.avgKeySize = cStats->avg_key_size; + stats.avgValueSize = cStats->avg_value_size; + stats.readAmp = cStats->read_amp; + stats.hitRate = cStats->hit_rate; + + if (cStats->num_levels > 0 && cStats->level_key_counts != nullptr) + { + stats.levelKeyCounts.resize(cStats->num_levels); + for (int i = 0; i < cStats->num_levels; ++i) + { + stats.levelKeyCounts[i] = cStats->level_key_counts[i]; + } + } + if (cStats->config != nullptr) { ColumnFamilyConfig cfConfig; @@ -126,8 +217,8 @@ Stats ColumnFamily::getStats() const cfConfig.minLevels = cStats->config->min_levels; cfConfig.dividingLevelOffset = cStats->config->dividing_level_offset; cfConfig.klogValueThreshold = cStats->config->klog_value_threshold; - cfConfig.compressionAlgorithm = - static_cast(cStats->config->compression_algo); + cfConfig.compressionAlgorithm = static_cast( + static_cast(cStats->config->compression_algorithm)); cfConfig.enableBloomFilter = cStats->config->enable_bloom_filter != 0; cfConfig.bloomFPR = cStats->config->bloom_fpr; cfConfig.enableBlockIndexes = cStats->config->enable_block_indexes != 0; @@ -162,6 +253,55 @@ void ColumnFamily::flushMemtable() checkResult(result, "failed to flush memtable"); } +bool ColumnFamily::isFlushing() const +{ + return tidesdb_is_flushing(cf_) != 0; +} + +bool ColumnFamily::isCompacting() const +{ + return tidesdb_is_compacting(cf_) != 0; +} + +void ColumnFamily::updateRuntimeConfig(const ColumnFamilyConfig& config, bool persistToDisk) +{ + tidesdb_column_family_config_t cConfig; + cConfig.write_buffer_size = config.writeBufferSize; + cConfig.level_size_ratio = config.levelSizeRatio; + cConfig.min_levels = config.minLevels; + cConfig.dividing_level_offset = config.dividingLevelOffset; + cConfig.klog_value_threshold = config.klogValueThreshold; + cConfig.compression_algorithm = + static_cast<::compression_algorithm>(config.compressionAlgorithm); + cConfig.enable_bloom_filter = config.enableBloomFilter ? 1 : 0; + cConfig.bloom_fpr = config.bloomFPR; + cConfig.enable_block_indexes = config.enableBlockIndexes ? 1 : 0; + cConfig.index_sample_ratio = config.indexSampleRatio; + cConfig.block_index_prefix_len = config.blockIndexPrefixLen; + cConfig.sync_mode = static_cast(config.syncMode); + cConfig.sync_interval_us = config.syncIntervalUs; + cConfig.skip_list_max_level = config.skipListMaxLevel; + cConfig.skip_list_probability = config.skipListProbability; + cConfig.default_isolation_level = + static_cast(config.defaultIsolationLevel); + cConfig.min_disk_space = config.minDiskSpace; + cConfig.l1_file_count_trigger = config.l1FileCountTrigger; + cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold; + + std::memset(cConfig.comparator_name, 0, TDB_MAX_COMPARATOR_NAME); + if (!config.comparatorName.empty()) + { + std::strncpy(cConfig.comparator_name, config.comparatorName.c_str(), + TDB_MAX_COMPARATOR_NAME - 1); + } + std::memset(cConfig.comparator_ctx_str, 0, TDB_MAX_COMPARATOR_CTX); + cConfig.comparator_fn_cached = nullptr; + cConfig.comparator_ctx_cached = nullptr; + + int result = tidesdb_cf_update_runtime_config(cf_, &cConfig, persistToDisk ? 1 : 0); + checkResult(result, "failed to update runtime config"); +} + //----------------------------------------------------------------------------- // Iterator //----------------------------------------------------------------------------- @@ -449,7 +589,8 @@ void TidesDB::createColumnFamily(const std::string& name, const ColumnFamilyConf cConfig.min_levels = config.minLevels; cConfig.dividing_level_offset = config.dividingLevelOffset; cConfig.klog_value_threshold = config.klogValueThreshold; - cConfig.compression_algo = static_cast(config.compressionAlgorithm); + cConfig.compression_algorithm = + static_cast<::compression_algorithm>(config.compressionAlgorithm); cConfig.enable_bloom_filter = config.enableBloomFilter ? 1 : 0; cConfig.bloom_fpr = config.bloomFPR; cConfig.enable_block_indexes = config.enableBlockIndexes ? 1 : 0; @@ -551,11 +692,45 @@ CacheStats TidesDB::getCacheStats() return stats; } -void TidesDB::registerComparator(const std::string& name, const std::string& ctxStr) +void TidesDB::registerComparator(const std::string& name, tidesdb_comparator_fn fn, + const std::string& ctxStr, void* ctx) { const char* ctxStrPtr = ctxStr.empty() ? nullptr : ctxStr.c_str(); - int result = tidesdb_register_comparator(db_, name.c_str(), nullptr, ctxStrPtr, nullptr); + int result = tidesdb_register_comparator(db_, name.c_str(), fn, ctxStrPtr, ctx); checkResult(result, "failed to register comparator"); } +void TidesDB::getComparator(const std::string& name, tidesdb_comparator_fn* fn, void** ctx) +{ + int result = tidesdb_get_comparator(db_, name.c_str(), fn, ctx); + checkResult(result, "failed to get comparator"); +} + +void TidesDB::renameColumnFamily(const std::string& oldName, const std::string& newName) +{ + int result = tidesdb_rename_column_family(db_, oldName.c_str(), newName.c_str()); + checkResult(result, "failed to rename column family"); +} + +void TidesDB::backup(const std::string& dir) +{ + int result = tidesdb_backup(db_, const_cast(dir.c_str())); + checkResult(result, "failed to create backup"); +} + +Config TidesDB::defaultConfig() +{ + tidesdb_config_t cConfig = tidesdb_default_config(); + + Config config; + config.dbPath = cConfig.db_path ? cConfig.db_path : ""; + config.numFlushThreads = cConfig.num_flush_threads; + config.numCompactionThreads = cConfig.num_compaction_threads; + config.logLevel = static_cast(cConfig.log_level); + config.blockCacheSize = cConfig.block_cache_size; + config.maxOpenSSTables = cConfig.max_open_sstables; + + return config; +} + } // namespace tidesdb diff --git a/tests/tidesdb_test.cpp b/tests/tidesdb_test.cpp index 0f8b6c1..60251aa 100644 --- a/tests/tidesdb_test.cpp +++ b/tests/tidesdb_test.cpp @@ -597,6 +597,158 @@ TEST_F(TidesDBTest, ByteVectorOperations) } } +TEST_F(TidesDBTest, RenameColumnFamily) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + db.createColumnFamily("old_cf", cfConfig); + + auto cf = db.getColumnFamily("old_cf"); + { + auto txn = db.beginTransaction(); + txn.put(cf, "key", "value", -1); + txn.commit(); + } + + db.renameColumnFamily("old_cf", "new_cf"); + + EXPECT_THROW(db.getColumnFamily("old_cf"), tidesdb::Exception); + + auto newCf = db.getColumnFamily("new_cf"); + { + auto txn = db.beginTransaction(); + auto value = txn.get(newCf, "key"); + std::string valueStr(value.begin(), value.end()); + ASSERT_EQ(valueStr, "value"); + } +} + +TEST_F(TidesDBTest, IsFlushingAndCompacting) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + db.createColumnFamily("test_cf", cfConfig); + + auto cf = db.getColumnFamily("test_cf"); + + bool isFlushing = cf.isFlushing(); + bool isCompacting = cf.isCompacting(); + + ASSERT_FALSE(isFlushing); + ASSERT_FALSE(isCompacting); +} + +TEST_F(TidesDBTest, Backup) +{ + std::string backupPath = testDbPath_ + "_backup"; + fs::remove_all(backupPath); + + { + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + db.createColumnFamily("test_cf", cfConfig); + + auto cf = db.getColumnFamily("test_cf"); + { + auto txn = db.beginTransaction(); + txn.put(cf, "backup_key", "backup_value", -1); + txn.commit(); + } + + db.backup(backupPath); + } + + ASSERT_TRUE(fs::exists(backupPath)); + + { + tidesdb::Config config; + config.dbPath = backupPath; + config.numFlushThreads = 2; + config.numCompactionThreads = 2; + config.logLevel = tidesdb::LogLevel::Info; + config.blockCacheSize = 64 * 1024 * 1024; + config.maxOpenSSTables = 256; + + tidesdb::TidesDB backupDb(config); + auto cf = backupDb.getColumnFamily("test_cf"); + auto txn = backupDb.beginTransaction(); + auto value = txn.get(cf, "backup_key"); + std::string valueStr(value.begin(), value.end()); + ASSERT_EQ(valueStr, "backup_value"); + } + + fs::remove_all(backupPath); +} + +TEST_F(TidesDBTest, ExtendedStats) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + cfConfig.writeBufferSize = 2 * 1024 * 1024; + db.createColumnFamily("test_cf", cfConfig); + + auto cf = db.getColumnFamily("test_cf"); + + { + auto txn = db.beginTransaction(); + for (int i = 0; i < 100; ++i) + { + std::string key = "key" + std::to_string(i); + std::string value = "value" + std::to_string(i); + txn.put(cf, key, value, -1); + } + txn.commit(); + } + + auto stats = cf.getStats(); + + ASSERT_GE(stats.numLevels, 0); + ASSERT_GE(stats.totalKeys, 0u); + ASSERT_GE(stats.totalDataSize, 0u); + ASSERT_GE(stats.avgKeySize, 0.0); + ASSERT_GE(stats.avgValueSize, 0.0); + ASSERT_GE(stats.readAmp, 0.0); + ASSERT_GE(stats.hitRate, 0.0); +} + +TEST_F(TidesDBTest, UpdateRuntimeConfig) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + cfConfig.writeBufferSize = 64 * 1024 * 1024; + db.createColumnFamily("test_cf", cfConfig); + + auto cf = db.getColumnFamily("test_cf"); + + auto newConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + newConfig.writeBufferSize = 128 * 1024 * 1024; + newConfig.skipListMaxLevel = 16; + newConfig.bloomFPR = 0.001; + + cf.updateRuntimeConfig(newConfig, false); + + auto stats = cf.getStats(); + if (stats.config.has_value()) + { + ASSERT_EQ(stats.config->writeBufferSize, 128 * 1024 * 1024u); + } +} + +TEST_F(TidesDBTest, DefaultConfig) +{ + auto defaultConfig = tidesdb::TidesDB::defaultConfig(); + + ASSERT_GE(defaultConfig.numFlushThreads, 0); + ASSERT_GE(defaultConfig.numCompactionThreads, 0); + ASSERT_GE(defaultConfig.blockCacheSize, 0u); + ASSERT_GE(defaultConfig.maxOpenSSTables, 0u); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv);