diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index f124c8ef9b0..6adecea4d60 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -299,21 +299,18 @@ struct Settings M(SettingUInt64, dt_storage_pool_log_gc_min_legacy_num, 3, "Min number of legacy page files to compact") \ M(SettingUInt64, dt_storage_pool_log_gc_min_bytes, 128 * Constant::MB, "Min bytes of page data to compact") \ M(SettingFloat, dt_storage_pool_log_gc_max_valid_rate, 0.35, "Max valid rate of deciding a page file can be compact") \ - M(SettingDouble, dt_storage_pool_log_gc_force_hardlink_rate, 2, "Max valid rate of deciding a page file can do hardlink") \ \ M(SettingUInt64, dt_storage_pool_data_write_slots, 1, "Max write concurrency for each StoragePool.data.") \ M(SettingUInt64, dt_storage_pool_data_gc_min_file_num, 10, "Min number of page files to compact") \ M(SettingUInt64, dt_storage_pool_data_gc_min_legacy_num, 3, "Min number of legacy page files to compact") \ M(SettingUInt64, dt_storage_pool_data_gc_min_bytes, 128 * Constant::MB, "Min bytes of page data to compact") \ M(SettingFloat, dt_storage_pool_data_gc_max_valid_rate, 0.35, "Max valid rate of deciding a page file can be compact") \ - M(SettingDouble, dt_storage_pool_data_gc_force_hardlink_rate, 2, "Max valid rate of deciding a page file can do hardlink") \ \ M(SettingUInt64, dt_storage_pool_meta_write_slots, 2, "Max write concurrency for each StoragePool.meta.") \ M(SettingUInt64, dt_storage_pool_meta_gc_min_file_num, 10, "Min number of page files to compact") \ M(SettingUInt64, dt_storage_pool_meta_gc_min_legacy_num, 3, "Min number of legacy page files to compact") \ M(SettingUInt64, dt_storage_pool_meta_gc_min_bytes, 128 * Constant::MB, "Min bytes of page data to compact") \ M(SettingFloat, dt_storage_pool_meta_gc_max_valid_rate, 0.35, "Max valid rate of deciding a page file can be compact") \ - M(SettingDouble, dt_storage_pool_meta_gc_force_hardlink_rate, 2, "Max valid rate of deciding a page file can do hardlink") \ \ M(SettingUInt64, dt_checksum_frame_size, DBMS_DEFAULT_BUFFER_SIZE, "Frame size for delta tree stable storage") \ M(SettingChecksumAlgorithm, dt_checksum_algorithm, ChecksumAlgo::XXH3, "Checksum algorithm for delta tree stable storage") \ diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 3d6fb73344b..fe82a7c58fb 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -42,8 +42,7 @@ PageStorage::Config extractConfig(const Settings & settings, StorageType subtype config.gc_min_files = settings.dt_storage_pool_##NAME##_gc_min_file_num; \ config.gc_min_bytes = settings.dt_storage_pool_##NAME##_gc_min_bytes; \ config.gc_min_legacy_num = settings.dt_storage_pool_##NAME##_gc_min_legacy_num; \ - config.gc_max_valid_rate = settings.dt_storage_pool_##NAME##_gc_max_valid_rate; \ - config.gc_force_hardlink_rate = settings.dt_storage_pool_##NAME##_gc_force_hardlink_rate; + config.gc_max_valid_rate = settings.dt_storage_pool_##NAME##_gc_max_valid_rate; PageStorage::Config config = getConfigFromSettings(settings); diff --git a/dbms/src/Storages/Page/V2/PageFile.cpp b/dbms/src/Storages/Page/V2/PageFile.cpp index b5e7d381b05..102a848ccc9 100644 --- a/dbms/src/Storages/Page/V2/PageFile.cpp +++ b/dbms/src/Storages/Page/V2/PageFile.cpp @@ -302,7 +302,7 @@ bool PageFile::LinkingMetaAdapter::hasNext() const return meta_file_offset < meta_size; } -void PageFile::LinkingMetaAdapter::linkToNewSequenceNext(WriteBatch::SequenceID sid, PageEntriesEdit & edit, UInt64 file_id, UInt64 level) +bool PageFile::LinkingMetaAdapter::linkToNewSequenceNext(WriteBatch::SequenceID sid, PageEntriesEdit & edit, UInt64 file_id, UInt64 level) { char * meta_data_end = meta_buffer + meta_size; char * pos = meta_buffer + meta_file_offset; @@ -311,7 +311,7 @@ void PageFile::LinkingMetaAdapter::linkToNewSequenceNext(WriteBatch::SequenceID LOG_WARNING(page_file.log, "[batch_start_pos=" << meta_file_offset << "] [meta_size=" << meta_size << "] [file=" << page_file.metaPath() << "] ignored."); - return; + return false; } const char * wb_start_pos = pos; @@ -321,7 +321,7 @@ void PageFile::LinkingMetaAdapter::linkToNewSequenceNext(WriteBatch::SequenceID LOG_WARNING(page_file.log, "[expect_batch_bytes=" << wb_bytes << "] [meta_size=" << meta_size << "] [file=" << page_file.metaPath() << "] ignored."); - return; + return false; } WriteBatch::SequenceID wb_sequence = 0; @@ -450,6 +450,8 @@ void PageFile::LinkingMetaAdapter::linkToNewSequenceNext(WriteBatch::SequenceID ErrorCodes::LOGICAL_ERROR); meta_file_offset = pos - meta_buffer; + + return true; } // ========================================================= @@ -735,7 +737,11 @@ void PageFile::Writer::hardlinkFrom(PageFile & linked_file, WriteBatch::Sequence // Move to the SequenceID item while (reader->hasNext()) { - reader->linkToNewSequenceNext(sid, edit, page_file.getFileId(), page_file.getLevel()); + if (!reader->linkToNewSequenceNext(sid, edit, page_file.getFileId(), page_file.getLevel())) + { + throw Exception(fmt::format("Failed to update [sid={}] into [file_id={}] , [file_level={}]", sid, page_file.getFileId(), page_file.getLevel()), + ErrorCodes::LOGICAL_ERROR); + } } char * linked_meta_data; diff --git a/dbms/src/Storages/Page/V2/PageFile.h b/dbms/src/Storages/Page/V2/PageFile.h index 90d5f33d72c..7f1c41f9e8f 100644 --- a/dbms/src/Storages/Page/V2/PageFile.h +++ b/dbms/src/Storages/Page/V2/PageFile.h @@ -221,7 +221,7 @@ class PageFile : public Allocator bool hasNext() const; - void linkToNewSequenceNext(WriteBatch::SequenceID sid, PageEntriesEdit & edit, UInt64 file_id, UInt64 level); + bool linkToNewSequenceNext(WriteBatch::SequenceID sid, PageEntriesEdit & edit, UInt64 file_id, UInt64 level); std::pair getMetaInfo() { return {meta_buffer, meta_size}; };