refactor(metrics): adapt to the new foyer metrics framework (#19580)

Signed-off-by: MrCroxx <[email protected]>

Cargo.toml

@@ -81,9 +81,10 @@ repository = "https://github.com/risingwavelabs/risingwave"
 
 [workspace.dependencies]
 # foyer = { version = "0.12.2", features = ["tracing", "nightly"] }
-foyer = { git = "https://github.com/mrcroxx/foyer", rev = "99005412eac1bca6aa7c73d750efc9cdb8ae176e", features = [
+foyer = { git = "https://github.com/mrcroxx/foyer", rev = "19bc2def14dfa8bb7d6581fb625584694bc7f4a0", features = [
     "tracing",
     "nightly",
+    "prometheus",
 ] }
 apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "25113ba88234a9ae23296e981d8302c290fdaa4b", features = [
     "snappy",

src/storage/Cargo.toml

@@ -35,7 +35,6 @@ itertools = { workspace = true }
 libc = "0.2"
 lz4 = "1.28.0"
 memcomparable = "0.2"
-metrics-prometheus = "0.8"
 moka = { version = "0.12.0", features = ["future", "sync"] }
 more-asserts = "0.3"
 num-integer = "0.1"

src/storage/src/store_impl.rs

@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use std::fmt::Debug;
-use std::ops::Deref;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 
 use enum_as_inner::EnumAsInner;
 use foyer::{
-    DirectFsDeviceOptions, Engine, HybridCacheBuilder, LargeEngineOptions, RateLimitPicker,
+    DirectFsDeviceOptions, Engine, HybridCacheBuilder, LargeEngineOptions,
+    PrometheusMetricsRegistry, RateLimitPicker,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common_service::RpcNotificationClient;
@@ -42,6 +42,9 @@ use crate::monitor::{
 use crate::opts::StorageOpts;
 use crate::StateStore;
 
+static FOYER_METRICS_REGISTRY: LazyLock<PrometheusMetricsRegistry> =
+    LazyLock::new(|| PrometheusMetricsRegistry::new(GLOBAL_METRICS_REGISTRY.clone()));
+
 mod opaque_type {
     use super::*;
 
@@ -613,15 +616,10 @@ impl StateStoreImpl {
     ) -> StorageResult<Self> {
         const MB: usize = 1 << 20;
 
-        if cfg!(not(madsim)) {
-            metrics_prometheus::Recorder::builder()
-                .with_registry(GLOBAL_METRICS_REGISTRY.deref().clone())
-                .build_and_install();
-        }
-
         let meta_cache = {
             let mut builder = HybridCacheBuilder::new()
                 .with_name("foyer.meta")
+                .with_metrics_registry(FOYER_METRICS_REGISTRY.clone())
                 .memory(opts.meta_cache_capacity_mb * MB)
                 .with_shards(opts.meta_cache_shard_num)
                 .with_eviction_config(opts.meta_cache_eviction_config.clone())
@@ -667,6 +665,7 @@ impl StateStoreImpl {
         let block_cache = {
             let mut builder = HybridCacheBuilder::new()
                 .with_name("foyer.data")
+                .with_metrics_registry(FOYER_METRICS_REGISTRY.clone())
                 .with_event_listener(Arc::new(BlockCacheEventListener::new(
                     state_store_metrics.clone(),
                 )))