fix deadlock caused by timeout flush (alibaba#1583)

henryzhx8 · Jul 3, 2024 · 71394f0 · 71394f0
1 parent 46099c4
commit 71394f0
Showing 1 changed file with 16 additions and 8 deletions.
diff --git a/core/batch/TimeoutFlushManager.cpp b/core/batch/TimeoutFlushManager.cpp
@@ -31,17 +31,25 @@ void TimeoutFlushManager::UpdateRecord(
 }
 
 void TimeoutFlushManager::FlushTimeoutBatch() {
-    lock_guard<mutex> lock(mMux);
-    for (auto& item : mTimeoutRecords) {
-        for (auto it = item.second.begin(); it != item.second.end();) {
-            if (time(nullptr) - it->second.mUpdateTime >= it->second.mTimeoutSecs) {
-                it->second.mFlusher->Flush(it->second.mKey);
-                it = item.second.erase(it);
-            } else {
-                ++it;
+    vector<pair<Flusher*, size_t>> records;
+    {
+        lock_guard<mutex> lock(mMux);
+        for (auto& item : mTimeoutRecords) {
+            for (auto it = item.second.begin(); it != item.second.end();) {
+                if (time(nullptr) - it->second.mUpdateTime >= it->second.mTimeoutSecs) {
+                    // cannot flush here, since flush may also update record, which will lead to both deadlock and map
+                    // iterator invalidation problems
+                    records.emplace_back(it->second.mFlusher, it->second.mKey);
+                    it = item.second.erase(it);
+                } else {
+                    ++it;
+                }
             }
         }
     }
+    for (auto& item : records) {
+        item.first->Flush(item.second);
+    }
 }
 
 void TimeoutFlushManager::ClearRecords(const string& config) {