feat(storage): support file cache recover mode (#18036)

Signed-off-by: MrCroxx <[email protected]>

Cargo.toml

@@ -77,7 +77,7 @@ license = "Apache-2.0"
 repository = "https://github.com/risingwavelabs/risingwave"
 
 [workspace.dependencies]
-foyer = { version = "0.10.1", features = ["nightly", "mtrace"] }
+foyer = { version = "0.10.4", features = ["nightly", "mtrace"] }
 apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "25113ba88234a9ae23296e981d8302c290fdaa4b", features = [
     "snappy",
     "zstandard",

src/common/src/config.rs

@@ -24,7 +24,7 @@ use std::num::NonZeroUsize;
 use anyhow::Context;
 use clap::ValueEnum;
 use educe::Educe;
-use foyer::{LfuConfig, LruConfig, S3FifoConfig};
+use foyer::{LfuConfig, LruConfig, RecoverMode, S3FifoConfig};
 use risingwave_common_proc_macro::ConfigDoc;
 pub use risingwave_common_proc_macro::OverrideConfig;
 use risingwave_pb::meta::SystemParams;
@@ -863,6 +863,18 @@ pub struct FileCacheConfig {
     #[serde(default = "default::file_cache::flush_buffer_threshold_mb")]
     pub flush_buffer_threshold_mb: Option<usize>,
 
+    /// Recover mode.
+    ///
+    /// Options:
+    ///
+    /// - "None": Do not recover disk cache.
+    /// - "Quiet": Recover disk cache and skip errors.
+    /// - "Strict": Recover disk cache and panic on errors.
+    ///
+    /// More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`],
+    #[serde(default = "default::file_cache::recover_mode")]
+    pub recover_mode: RecoverMode,
+
     #[serde(default, flatten)]
     #[config_doc(omitted)]
     pub unrecognized: Unrecognized<Self>,
@@ -1673,6 +1685,8 @@ pub mod default {
     }
 
     pub mod file_cache {
+        use foyer::RecoverMode;
+
         pub fn dir() -> String {
             "".to_string()
         }
@@ -1712,6 +1726,10 @@ pub mod default {
         pub fn flush_buffer_threshold_mb() -> Option<usize> {
             None
         }
+
+        pub fn recover_mode() -> RecoverMode {
+            RecoverMode::None
+        }
     }
 
     pub mod cache_refill {

src/config/example.toml

@@ -171,6 +171,7 @@ recover_concurrency = 8
 insert_rate_limit_mb = 0
 indexer_shards = 64
 compression = "none"
+recover_mode = "None"
 
 [storage.meta_file_cache]
 dir = ""
@@ -182,6 +183,7 @@ recover_concurrency = 8
 insert_rate_limit_mb = 0
 indexer_shards = 64
 compression = "none"
+recover_mode = "None"
 
 [storage.cache_refill]
 data_refill_levels = []

src/storage/src/opts.rs

@@ -86,6 +86,7 @@ pub struct StorageOpts {
     pub data_file_cache_file_capacity_mb: usize,
     pub data_file_cache_flushers: usize,
     pub data_file_cache_reclaimers: usize,
+    pub data_file_cache_recover_mode: foyer::RecoverMode,
     pub data_file_cache_recover_concurrency: usize,
     pub data_file_cache_insert_rate_limit_mb: usize,
     pub data_file_cache_indexer_shards: usize,
@@ -105,6 +106,7 @@ pub struct StorageOpts {
     pub meta_file_cache_file_capacity_mb: usize,
     pub meta_file_cache_flushers: usize,
     pub meta_file_cache_reclaimers: usize,
+    pub meta_file_cache_recover_mode: foyer::RecoverMode,
     pub meta_file_cache_recover_concurrency: usize,
     pub meta_file_cache_insert_rate_limit_mb: usize,
     pub meta_file_cache_indexer_shards: usize,
@@ -181,6 +183,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             data_file_cache_file_capacity_mb: c.storage.data_file_cache.file_capacity_mb,
             data_file_cache_flushers: c.storage.data_file_cache.flushers,
             data_file_cache_reclaimers: c.storage.data_file_cache.reclaimers,
+            data_file_cache_recover_mode: c.storage.data_file_cache.recover_mode,
             data_file_cache_recover_concurrency: c.storage.data_file_cache.recover_concurrency,
             data_file_cache_insert_rate_limit_mb: c.storage.data_file_cache.insert_rate_limit_mb,
             data_file_cache_indexer_shards: c.storage.data_file_cache.indexer_shards,
@@ -191,6 +194,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             meta_file_cache_file_capacity_mb: c.storage.meta_file_cache.file_capacity_mb,
             meta_file_cache_flushers: c.storage.meta_file_cache.flushers,
             meta_file_cache_reclaimers: c.storage.meta_file_cache.reclaimers,
+            meta_file_cache_recover_mode: c.storage.meta_file_cache.recover_mode,
             meta_file_cache_recover_concurrency: c.storage.meta_file_cache.recover_concurrency,
             meta_file_cache_insert_rate_limit_mb: c.storage.meta_file_cache.insert_rate_limit_mb,
             meta_file_cache_indexer_shards: c.storage.meta_file_cache.indexer_shards,

src/storage/src/store_impl.rs

@@ -669,6 +669,7 @@ impl StateStoreImpl {
                     .with_clean_region_threshold(
                         opts.meta_file_cache_reclaimers + opts.meta_file_cache_reclaimers / 2,
                     )
+                    .with_recover_mode(opts.meta_file_cache_recover_mode)
                     .with_recover_concurrency(opts.meta_file_cache_recover_concurrency)
                     .with_compression(
                         opts.meta_file_cache_compression
@@ -722,6 +723,7 @@ impl StateStoreImpl {
                     .with_clean_region_threshold(
                         opts.data_file_cache_reclaimers + opts.data_file_cache_reclaimers / 2,
                     )
+                    .with_recover_mode(opts.data_file_cache_recover_mode)
                     .with_recover_concurrency(opts.data_file_cache_recover_concurrency)
                     .with_compression(
                         opts.data_file_cache_compression