ensure wal is synced into manifest in FindObsoleteFiles as well

Signed-off-by: Qi Xu <[email protected]>
tonyxuqqi · Mar 21, 2024 · 48eee8b · 48eee8b
1 parent a4674f0
commit 48eee8b
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 6 deletions.
diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc
@@ -1488,11 +1488,15 @@ void DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir,
     auto& wal = *it;
     assert(wal.IsSyncing());
     ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "Synced log %" PRIu64 " from logs_\n", wal.number);
+                   "Synced log %" PRIu64 " from logs_, last seq number %" PRIu64
+                   "\n",
+                   wal.number, wal.writer->GetLastSequence());
     if (logs_.size() > 1) {
       if (immutable_db_options_.track_and_verify_wals_in_manifest &&
           wal.GetPreSyncSize() > 0) {
-        synced_wals->AddWal(wal.number, WalMetadata(wal.GetPreSyncSize()));
+        synced_wals->AddWal(
+            wal.number,
+            WalMetadata(wal.GetPreSyncSize(), wal.writer->GetLastSequence()));
       }
       auto writer = wal.ReleaseWriter();
       ROCKS_LOG_INFO(immutable_db_options_.info_log,

diff --git a/db/db_impl/db_impl_files.cc b/db/db_impl/db_impl_files.cc
@@ -280,6 +280,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
     return;
   }
 
+  VersionEdit synced_wals;
   if (!alive_log_files_.empty() && !logs_.empty()) {
     uint64_t min_log_number = job_context->log_number;
     size_t num_alive_log_files = alive_log_files_.size();
@@ -319,6 +320,12 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
         // logs_ could have changed while we were waiting.
         continue;
       }
+      if (immutable_db_options_.track_and_verify_wals_in_manifest &&
+          log.GetPreSyncSize() > 0) {
+        synced_wals.AddWal(
+            log.number,
+            WalMetadata(log.GetPreSyncSize(), log.writer->GetLastSequence()));
+      }
       auto writer = log.ReleaseWriter();
       ROCKS_LOG_INFO(immutable_db_options_.info_log,
                      "deleting log %" PRIu64
@@ -338,6 +345,9 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   logs_to_free_.clear();
   log_write_mutex_.Unlock();
   mutex_.Lock();
+  if (synced_wals.IsWalAddition()) {
+    ApplyWALToManifest(&synced_wals);
+  }
   job_context->log_recycle_files.assign(log_recycle_files_.begin(),
                                         log_recycle_files_.end());
 }

diff --git a/db/wal_edit.cc b/db/wal_edit.cc
@@ -58,13 +58,15 @@ Status WalAddition::DecodeFrom(Slice* src) {
 
 JSONWriter& operator<<(JSONWriter& jw, const WalAddition& wal) {
   jw << "LogNumber" << wal.GetLogNumber() << "SyncedSizeInBytes"
-     << wal.GetMetadata().GetSyncedSizeInBytes();
+     << wal.GetMetadata().GetSyncedSizeInBytes() << "LastSeqNumber"
+     << wal.GetMetadata().GetLastSequence();
   return jw;
 }
 
 std::ostream& operator<<(std::ostream& os, const WalAddition& wal) {
   os << "log_number: " << wal.GetLogNumber()
-     << " synced_size_in_bytes: " << wal.GetMetadata().GetSyncedSizeInBytes();
+     << " synced_size_in_bytes: " << wal.GetMetadata().GetSyncedSizeInBytes()
+     << " last_seq_number: " << wal.GetMetadata().GetLastSequence();
   return os;
 }
 

diff --git a/db/wal_edit.h b/db/wal_edit.h
@@ -32,15 +32,19 @@ class WalMetadata {
  public:
   WalMetadata() = default;
 
-  explicit WalMetadata(uint64_t synced_size_bytes)
-      : synced_size_bytes_(synced_size_bytes) {}
+  explicit WalMetadata(uint64_t synced_size_bytes,
+                       uint64_t last_sequence_number)
+      : synced_size_bytes_(synced_size_bytes),
+        last_sequence_number_(last_sequence_number) {}
 
   bool HasSyncedSize() const { return synced_size_bytes_ != kUnknownWalSize; }
 
   void SetSyncedSizeInBytes(uint64_t bytes) { synced_size_bytes_ = bytes; }
 
   uint64_t GetSyncedSizeInBytes() const { return synced_size_bytes_; }
 
+  uint64_t GetLastSequence() const { return last_sequence_number_; }
+
  private:
   friend bool operator==(const WalMetadata& lhs, const WalMetadata& rhs);
   friend bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs);
@@ -50,6 +54,8 @@ class WalMetadata {
 
   // Size of the most recently synced WAL in bytes.
   uint64_t synced_size_bytes_ = kUnknownWalSize;
+
+  uint64_t last_sequence_number_ = 0;
 };
 
 inline bool operator==(const WalMetadata& lhs, const WalMetadata& rhs) {