Add new parameters CompactionBlobThreshold and CompactionBlobThreshol…

…dSSD. It is thresholds in bytes, if data size less threshold it will be written to mixed channel and to merged channel in other case.
ydb-platform · Feb 14, 2025 · 8b2473b · 8b2473b
1 parent 0407cea
commit 8b2473b
Show file tree

Hide file tree

Showing 12 changed files with 658 additions and 62 deletions.
diff --git a/.gitignore b/.gitignore
@@ -73,6 +73,7 @@ test-results
 # Test binaries and generated dirs
 *-ut
 **/tests/tests
+**/tests/**/cloud-blockstore-tests-loadtest-compaction
 
 # Autogenerated util, library and contrib files
 /util/all_*.cpp

diff --git a/cloud/blockstore/config/storage.proto b/cloud/blockstore/config/storage.proto
@@ -1113,4 +1113,8 @@ message TStorageServiceConfig
 
     // Timeout for TDestroyVolumeActor (in milliseconds)
     optional uint32 DestroyVolumeTimeout = 405;
+
+    // Minimum compaction data size (in bytes) that lets us write the data to
+    // blobstorage (as a merged blob) else we will write it to mixed channel.
+    optional uint32 CompactionToMergedThresholdHDD = 406;
 }
diff --git a/cloud/blockstore/libs/storage/core/config.cpp b/cloud/blockstore/libs/storage/core/config.cpp
@@ -531,6 +531,7 @@ TDuration MSeconds(ui32 value)
     xxx(CalculateSplittedUsedQuotaMetric,                 bool,    false      )\
                                                                                \
     xxx(DestroyVolumeTimeout,                      TDuration, Seconds(30)     )\
+    xxx(CompactionToMergedThresholdHDD,            ui32,      0               )\
 // BLOCKSTORE_STORAGE_CONFIG_RW
 
 #define BLOCKSTORE_STORAGE_CONFIG(xxx)                                         \

diff --git a/cloud/blockstore/libs/storage/core/config.h b/cloud/blockstore/libs/storage/core/config.h
@@ -635,6 +635,8 @@ class TStorageConfig
     bool GetAutomaticallyEnableBufferCopyingAfterChecksumMismatch() const;
     [[nodiscard]] bool GetNonReplicatedVolumeDirectAcquireEnabled() const;
     [[nodiscard]] TDuration GetDestroyVolumeTimeout() const;
+
+    ui32 GetCompactionToMergedThresholdHDD() const;
 };
 
 ui64 GetAllocationUnit(

diff --git a/cloud/blockstore/libs/storage/partition/part_actor_addblobs.cpp b/cloud/blockstore/libs/storage/partition/part_actor_addblobs.cpp
@@ -87,17 +87,35 @@ class TAddBlobsExecutor
 
     void Execute(const TActorContext& ctx, TPartitionDatabase& db)
     {
-        for (const auto& blob: Args.MixedBlobs) {
+        if (Args.Mode == ADD_COMPACTION_RESULT) {
+            Y_ABORT_UNLESS(
+                Args.MixedBlobs.size() ==
+                Args.MixedBlockCompactionInfos.size());
+        }
+
+        for (ui32 i = 0; i < Args.MixedBlobs.size(); ++i) {
+            const auto& blob = Args.MixedBlobs[i];
             ProcessNewBlob(ctx, db, blob);
             UpdateCompactionCounters(blob);
             if (Args.Mode == EAddBlobMode::ADD_WRITE_RESULT) {
                 UpdateUsedBlocks(db, blob);
             }
+
+            if (Args.Mode == ADD_COMPACTION_RESULT) {
+                const auto& cm = State.GetCompactionMap();
+                const auto blockIndex = cm.GetRangeStart(BlockIndex(blob, 0));
+                auto& rangeInfo = CompactionCounters[blockIndex];
+                rangeInfo.BlobsSkippedByCompaction =
+                    Args.MixedBlockCompactionInfos[i].BlobsSkippedByCompaction;
+                rangeInfo.BlocksSkippedByCompaction =
+                    Args.MixedBlockCompactionInfos[i].BlocksSkippedByCompaction;
+            }
         }
 
         if (Args.Mode == ADD_COMPACTION_RESULT) {
-            Y_ABORT_UNLESS(Args.MergedBlobs.size()
-                    == Args.MergedBlobCompactionInfos.size());
+            Y_ABORT_UNLESS(
+                Args.MergedBlobs.size() ==
+                Args.MergedBlobCompactionInfos.size());
         }
 
         for (ui32 i = 0; i < Args.MergedBlobs.size(); ++i) {
@@ -111,7 +129,8 @@ class TAddBlobsExecutor
             if (Args.Mode == ADD_COMPACTION_RESULT) {
                 const auto& cm = State.GetCompactionMap();
                 const auto blockIndex = cm.GetRangeStart(blob.BlockRange.Start);
-                Y_DEBUG_ABORT_UNLESS(blockIndex == cm.GetRangeStart(blob.BlockRange.End));
+                Y_DEBUG_ABORT_UNLESS(
+                    blockIndex == cm.GetRangeStart(blob.BlockRange.End));
                 auto& rangeInfo = CompactionCounters[blockIndex];
                 rangeInfo.BlobsSkippedByCompaction =
                     Args.MergedBlobCompactionInfos[i].BlobsSkippedByCompaction;
@@ -641,6 +660,7 @@ void TPartitionActor::HandleAddBlobs(
         msg->Mode,
         std::move(msg->AffectedBlobs),
         std::move(msg->AffectedBlocks),
+        std::move(msg->MixedBlockBlobCompactionInfos),
         std::move(msg->MergedBlobCompactionInfos));
 }