From d334dd3b27c32fc3ca2a26e6f728445ba3ab909a Mon Sep 17 00:00:00 2001 From: CookiePieWw Date: Thu, 12 Dec 2024 14:53:14 +0800 Subject: [PATCH] refactor: use ReadableSize --- src/mito2/src/cache.rs | 8 ++++++-- src/mito2/src/cache/index.rs | 9 +++++---- src/mito2/src/sst/index/inverted_index/creator.rs | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/mito2/src/cache.rs b/src/mito2/src/cache.rs index a059c4ee7894..587b60560224 100644 --- a/src/mito2/src/cache.rs +++ b/src/mito2/src/cache.rs @@ -26,6 +26,7 @@ use std::mem; use std::sync::Arc; use bytes::Bytes; +use common_base::readable_size::ReadableSize; use datatypes::value::Value; use datatypes::vectors::VectorRef; use moka::notification::RemovalCause; @@ -355,8 +356,11 @@ impl CacheManagerBuilder { }) .build() }); - let inverted_index_cache = - InvertedIndexCache::new(self.index_metadata_size, self.index_content_size, PAGE_SIZE); + let inverted_index_cache = InvertedIndexCache::new( + self.index_metadata_size, + self.index_content_size, + ReadableSize::kb(PAGE_SIZE as u64), + ); let puffin_metadata_cache = PuffinMetadataCache::new(self.puffin_metadata_size, &CACHE_BYTES); let selector_result_cache = (self.selector_result_cache_size != 0).then(|| { diff --git a/src/mito2/src/cache/index.rs b/src/mito2/src/cache/index.rs index 65003d3c11c9..242564977be5 100644 --- a/src/mito2/src/cache/index.rs +++ b/src/mito2/src/cache/index.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use api::v1::index::InvertedIndexMetas; use async_trait::async_trait; +use common_base::readable_size::ReadableSize; use common_base::BitVec; use index::inverted_index::error::DecodeFstSnafu; use index::inverted_index::format::reader::InvertedIndexReader; @@ -214,7 +215,7 @@ pub struct InvertedIndexCache { impl InvertedIndexCache { /// Creates `InvertedIndexCache` with provided `index_metadata_cap` and `index_content_cap`. - pub fn new(index_metadata_cap: u64, index_content_cap: u64, page_size: usize) -> Self { + pub fn new(index_metadata_cap: u64, index_content_cap: u64, page_size: ReadableSize) -> Self { common_telemetry::debug!("Building InvertedIndexCache with metadata size: {index_metadata_cap}, content size: {index_content_cap}"); let index_metadata = moka::sync::CacheBuilder::new(index_metadata_cap) .name("inverted_index_metadata") @@ -239,7 +240,7 @@ impl InvertedIndexCache { Self { index_metadata, index: index_cache, - page_size, + page_size: page_size.as_bytes() as usize, } } } @@ -346,7 +347,7 @@ mod test { // Fuzz test for index data page key #[test] - fn fuzz_read_index() { + fn fuzz_index_calculation() { // randomly generate a large u8 array let mut rng = rand::thread_rng(); let mut data = vec![0u8; 1024 * 1024]; @@ -438,7 +439,7 @@ mod test { let mut cached_reader = CachedInvertedIndexBlobReader::new( FileId::random(), reader, - Arc::new(InvertedIndexCache::new(8192, 8192, 10)), + Arc::new(InvertedIndexCache::new(8192, 8192, ReadableSize(10))), ); let metadata = cached_reader.metadata().await.unwrap(); assert_eq!(metadata.total_row_count, 8); diff --git a/src/mito2/src/sst/index/inverted_index/creator.rs b/src/mito2/src/sst/index/inverted_index/creator.rs index 15cba55c4437..4d741a30a7f0 100644 --- a/src/mito2/src/sst/index/inverted_index/creator.rs +++ b/src/mito2/src/sst/index/inverted_index/creator.rs @@ -302,6 +302,7 @@ mod tests { use std::iter; use api::v1::SemanticType; + use common_base::readable_size::ReadableSize; use datafusion_expr::{binary_expr, col, lit, Expr as DfExpr, Operator}; use datatypes::data_type::ConcreteDataType; use datatypes::schema::ColumnSchema; @@ -448,7 +449,7 @@ mod tests { move |expr| { let _d = &d; - let cache = Arc::new(InvertedIndexCache::new(10, 10, 100)); + let cache = Arc::new(InvertedIndexCache::new(10, 10, ReadableSize(10))); let puffin_metadata_cache = Arc::new(PuffinMetadataCache::new(10, &CACHE_BYTES)); let applier = InvertedIndexApplierBuilder::new( region_dir.clone(),