From 03a28320d6f322000790eeb2b81ce7cb2cb9a972 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Tue, 10 Dec 2024 12:03:44 +0800 Subject: [PATCH] feat!: enable read cache and write cache when using remote object stores (#5093) * feat: enable read cache and write cache when using remote object stores * feat: make read cache be aware of remote store names * chore: docs * chore: apply review suggestions * chore: trim write cache path --------- Co-authored-by: Yingwen --- config/config.md | 16 ++++---- config/datanode.example.toml | 14 +++---- config/standalone.example.toml | 14 +++---- src/datanode/src/config.rs | 28 ++++++++++++- src/datanode/src/datanode.rs | 8 +++- src/datanode/src/lib.rs | 1 + src/datanode/src/store.rs | 63 +++++++++++++++++++++--------- src/mito2/src/config.rs | 10 +++-- src/mito2/src/error.rs | 8 ++++ src/mito2/src/worker.rs | 10 ++++- src/object-store/src/lib.rs | 2 + tests-integration/src/test_util.rs | 3 ++ tests-integration/tests/http.rs | 2 +- tests-integration/tests/main.rs | 3 +- 14 files changed, 132 insertions(+), 50 deletions(-) diff --git a/config/config.md b/config/config.md index 1f034d28731d..0f70a8cb126e 100644 --- a/config/config.md +++ b/config/config.md @@ -93,7 +93,7 @@ | `storage` | -- | -- | The data storage options. | | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. | | `storage.type` | String | `File` | The storage type used to store the data.
- `File`: the data is stored in the local file system.
- `S3`: the data is stored in the S3 object storage.
- `Gcs`: the data is stored in the Google Cloud Storage.
- `Azblob`: the data is stored in the Azure Blob Storage.
- `Oss`: the data is stored in the Aliyun OSS. | -| `storage.cache_path` | String | Unset | Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance.
The local file cache directory. | +| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. | | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. | | `storage.bucket` | String | Unset | The S3 bucket name.
**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. | | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. | @@ -131,9 +131,9 @@ | `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. | | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
If not set, it's default to 1/8 of OS memory. | | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. | -| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance. | -| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. | -| `region_engine.mito.experimental_write_cache_size` | String | `1GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. | +| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. | +| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. | +| `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. | | `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. | | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. | | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. | @@ -420,7 +420,7 @@ | `storage` | -- | -- | The data storage options. | | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. | | `storage.type` | String | `File` | The storage type used to store the data.
- `File`: the data is stored in the local file system.
- `S3`: the data is stored in the S3 object storage.
- `Gcs`: the data is stored in the Google Cloud Storage.
- `Azblob`: the data is stored in the Azure Blob Storage.
- `Oss`: the data is stored in the Aliyun OSS. | -| `storage.cache_path` | String | Unset | Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance.
The local file cache directory. | +| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. | | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. | | `storage.bucket` | String | Unset | The S3 bucket name.
**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. | | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. | @@ -458,9 +458,9 @@ | `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. | | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
If not set, it's default to 1/8 of OS memory. | | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. | -| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance. | -| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. | -| `region_engine.mito.experimental_write_cache_size` | String | `1GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. | +| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. | +| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. | +| `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. | | `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. | | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. | | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 11c2794e61df..8bfa8732cc41 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -294,14 +294,14 @@ data_home = "/tmp/greptimedb/" ## - `Oss`: the data is stored in the Aliyun OSS. type = "File" -## Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance. -## The local file cache directory. +## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance. +## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. ## @toml2docs:none-default -cache_path = "/path/local_cache" +#+ cache_path = "" ## The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. ## @toml2docs:none-default -cache_capacity = "1GiB" +cache_capacity = "5GiB" ## The S3 bucket name. ## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**. @@ -476,14 +476,14 @@ auto_flush_interval = "1h" ## @toml2docs:none-default="Auto" #+ selector_result_cache_size = "512MB" -## Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance. +## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. enable_experimental_write_cache = false -## File system path for write cache, defaults to `{data_home}/write_cache`. +## File system path for write cache, defaults to `{data_home}/object_cache/write`. experimental_write_cache_path = "" ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. -experimental_write_cache_size = "1GiB" +experimental_write_cache_size = "5GiB" ## TTL for write cache. ## @toml2docs:none-default diff --git a/config/standalone.example.toml b/config/standalone.example.toml index a69295af1644..56cbeaddb9d5 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -332,14 +332,14 @@ data_home = "/tmp/greptimedb/" ## - `Oss`: the data is stored in the Aliyun OSS. type = "File" -## Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance. -## The local file cache directory. +## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance. +## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. ## @toml2docs:none-default -cache_path = "/path/local_cache" +#+ cache_path = "" ## The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. ## @toml2docs:none-default -cache_capacity = "1GiB" +cache_capacity = "5GiB" ## The S3 bucket name. ## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**. @@ -514,14 +514,14 @@ auto_flush_interval = "1h" ## @toml2docs:none-default="Auto" #+ selector_result_cache_size = "512MB" -## Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance. +## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. enable_experimental_write_cache = false -## File system path for write cache, defaults to `{data_home}/write_cache`. +## File system path for write cache, defaults to `{data_home}/object_cache/write`. experimental_write_cache_path = "" ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. -experimental_write_cache_size = "1GiB" +experimental_write_cache_size = "5GiB" ## TTL for write cache. ## @toml2docs:none-default diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs index 4fedb9ea2cc0..4e1b4f3195a7 100644 --- a/src/datanode/src/config.rs +++ b/src/datanode/src/config.rs @@ -32,7 +32,7 @@ use servers::heartbeat_options::HeartbeatOptions; use servers::http::HttpOptions; use servers::Mode; -pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::gb(1); +pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::gb(5); /// Default data home in file storage const DEFAULT_DATA_HOME: &str = "/tmp/greptimedb"; @@ -60,6 +60,11 @@ impl ObjectStoreConfig { } } + /// Returns true when it's a remote object storage such as AWS s3 etc. + pub fn is_object_storage(&self) -> bool { + !matches!(self, Self::File(_)) + } + /// Returns the object storage configuration name, return the provider name if it's empty. pub fn config_name(&self) -> &str { let name = match self { @@ -91,6 +96,13 @@ pub struct StorageConfig { pub providers: Vec, } +impl StorageConfig { + /// Returns true when the default storage config is a remote object storage service such as AWS S3, etc. + pub fn is_object_storage(&self) -> bool { + self.store.is_object_storage() + } +} + impl Default for StorageConfig { fn default() -> Self { Self { @@ -452,6 +464,20 @@ mod tests { assert_eq!("S3", s3_config.provider_name()); } + #[test] + fn test_is_object_storage() { + let store = ObjectStoreConfig::default(); + assert!(!store.is_object_storage()); + let s3_config = ObjectStoreConfig::S3(S3Config::default()); + assert!(s3_config.is_object_storage()); + let oss_config = ObjectStoreConfig::Oss(OssConfig::default()); + assert!(oss_config.is_object_storage()); + let gcs_config = ObjectStoreConfig::Gcs(GcsConfig::default()); + assert!(gcs_config.is_object_storage()); + let azblob_config = ObjectStoreConfig::Azblob(AzblobConfig::default()); + assert!(azblob_config.is_object_storage()); + } + #[test] fn test_secstr() { let toml_str = r#" diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index c89c007082bf..53a0cf9fd78b 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -428,10 +428,16 @@ impl DatanodeBuilder { async fn build_mito_engine( opts: &DatanodeOptions, object_store_manager: ObjectStoreManagerRef, - config: MitoConfig, + mut config: MitoConfig, schema_metadata_manager: SchemaMetadataManagerRef, plugins: Plugins, ) -> Result { + if opts.storage.is_object_storage() { + // Enable the write cache when setting object storage + config.enable_experimental_write_cache = true; + info!("Configured 'enable_experimental_write_cache=true' for mito engine."); + } + let mito_engine = match &opts.wal { DatanodeWalConfig::RaftEngine(raft_engine_config) => MitoEngine::new( &opts.storage.data_home, diff --git a/src/datanode/src/lib.rs b/src/datanode/src/lib.rs index dae3eef76c79..6a7b1c596d2d 100644 --- a/src/datanode/src/lib.rs +++ b/src/datanode/src/lib.rs @@ -13,6 +13,7 @@ // limitations under the License. #![feature(assert_matches)] +#![feature(let_chains)] pub mod alive_keeper; pub mod config; diff --git a/src/datanode/src/store.rs b/src/datanode/src/store.rs index e8ede076741f..c78afe448e0c 100644 --- a/src/datanode/src/store.rs +++ b/src/datanode/src/store.rs @@ -19,21 +19,20 @@ mod fs; mod gcs; mod oss; mod s3; - +use std::path::Path; use std::sync::Arc; use std::time::Duration; use std::{env, path}; -use common_base::readable_size::ReadableSize; use common_telemetry::{info, warn}; use object_store::layers::{LruCacheLayer, RetryInterceptor, RetryLayer}; use object_store::services::Fs; use object_store::util::{join_dir, normalize_dir, with_instrument_layers}; -use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder}; +use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder, OBJECT_CACHE_DIR}; use snafu::prelude::*; use crate::config::{HttpClientConfig, ObjectStoreConfig, DEFAULT_OBJECT_STORE_CACHE_SIZE}; -use crate::error::{self, Result}; +use crate::error::{self, CreateDirSnafu, Result}; pub(crate) async fn new_raw_object_store( store: &ObjectStoreConfig, @@ -68,7 +67,7 @@ pub(crate) async fn new_object_store_without_cache( ) -> Result { let object_store = new_raw_object_store(store, data_home).await?; // Enable retry layer and cache layer for non-fs object storages - let object_store = if !matches!(store, ObjectStoreConfig::File(..)) { + let object_store = if store.is_object_storage() { // Adds retry layer with_retry_layers(object_store) } else { @@ -85,8 +84,8 @@ pub(crate) async fn new_object_store( ) -> Result { let object_store = new_raw_object_store(&store, data_home).await?; // Enable retry layer and cache layer for non-fs object storages - let object_store = if !matches!(store, ObjectStoreConfig::File(..)) { - let object_store = if let Some(cache_layer) = build_cache_layer(&store).await? { + let object_store = if store.is_object_storage() { + let object_store = if let Some(cache_layer) = build_cache_layer(&store, data_home).await? { // Adds cache layer object_store.layer(cache_layer) } else { @@ -105,44 +104,72 @@ pub(crate) async fn new_object_store( async fn build_cache_layer( store_config: &ObjectStoreConfig, + data_home: &str, ) -> Result>> { - let (cache_path, cache_capacity) = match store_config { + let (name, mut cache_path, cache_capacity) = match store_config { ObjectStoreConfig::S3(s3_config) => { - let path = s3_config.cache.cache_path.as_ref(); + let path = s3_config.cache.cache_path.clone(); + let name = &s3_config.name; let capacity = s3_config .cache .cache_capacity .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE); - (path, capacity) + (name, path, capacity) } ObjectStoreConfig::Oss(oss_config) => { - let path = oss_config.cache.cache_path.as_ref(); + let path = oss_config.cache.cache_path.clone(); + let name = &oss_config.name; let capacity = oss_config .cache .cache_capacity .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE); - (path, capacity) + (name, path, capacity) } ObjectStoreConfig::Azblob(azblob_config) => { - let path = azblob_config.cache.cache_path.as_ref(); + let path = azblob_config.cache.cache_path.clone(); + let name = &azblob_config.name; let capacity = azblob_config .cache .cache_capacity .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE); - (path, capacity) + (name, path, capacity) } ObjectStoreConfig::Gcs(gcs_config) => { - let path = gcs_config.cache.cache_path.as_ref(); + let path = gcs_config.cache.cache_path.clone(); + let name = &gcs_config.name; let capacity = gcs_config .cache .cache_capacity .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE); - (path, capacity) + (name, path, capacity) } - _ => (None, ReadableSize(0)), + _ => unreachable!("Already checked above"), }; - if let Some(path) = cache_path { + // Enable object cache by default + // Set the cache_path to be `${data_home}/object_cache/read/{name}` by default + // if it's not present + if cache_path.is_none() { + let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR); + let read_cache_path = join_dir(&object_cache_path, "read"); + let read_cache_path = join_dir(&read_cache_path, &name.to_lowercase()); + tokio::fs::create_dir_all(Path::new(&read_cache_path)) + .await + .context(CreateDirSnafu { + dir: &read_cache_path, + })?; + + info!( + "The object storage cache path is not set for '{}', using the default path: '{}'", + name, &read_cache_path + ); + + cache_path = Some(read_cache_path); + } + + if let Some(path) = cache_path.as_ref() + && !path.trim().is_empty() + { let atomic_temp_dir = join_dir(path, ".tmp/"); clean_temp_dir(&atomic_temp_dir)?; diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 067ab27938a2..9b113027a41b 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -21,6 +21,7 @@ use std::time::Duration; use common_base::readable_size::ReadableSize; use common_telemetry::warn; use object_store::util::join_dir; +use object_store::OBJECT_CACHE_DIR; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -96,7 +97,7 @@ pub struct MitoConfig { pub selector_result_cache_size: ReadableSize, /// Whether to enable the experimental write cache. pub enable_experimental_write_cache: bool, - /// File system path for write cache, defaults to `{data_home}/write_cache`. + /// File system path for write cache, defaults to `{data_home}/object_cache/write`. pub experimental_write_cache_path: String, /// Capacity for write cache. pub experimental_write_cache_size: ReadableSize, @@ -148,7 +149,7 @@ impl Default for MitoConfig { selector_result_cache_size: ReadableSize::mb(512), enable_experimental_write_cache: false, experimental_write_cache_path: String::new(), - experimental_write_cache_size: ReadableSize::gb(1), + experimental_write_cache_size: ReadableSize::gb(5), experimental_write_cache_ttl: None, sst_write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, parallel_scan_channel_size: DEFAULT_SCAN_CHANNEL_SIZE, @@ -232,8 +233,9 @@ impl MitoConfig { } // Sets write cache path if it is empty. - if self.experimental_write_cache_path.is_empty() { - self.experimental_write_cache_path = join_dir(data_home, "write_cache"); + if self.experimental_write_cache_path.trim().is_empty() { + let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR); + self.experimental_write_cache_path = join_dir(&object_cache_path, "write"); } self.index.sanitize(data_home, &self.inverted_index)?; diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 407c8c29e258..d5e47d213657 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -677,6 +677,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Failed to create directory {}", dir))] + CreateDir { + dir: String, + #[snafu(source)] + error: std::io::Error, + }, + #[snafu(display("Failed to filter record batch"))] FilterRecordBatch { source: common_recordbatch::error::Error, @@ -955,6 +962,7 @@ impl ErrorExt for Error { | ComputeVector { .. } | SerializeField { .. } | EncodeMemtable { .. } + | CreateDir { .. } | ReadDataPart { .. } | CorruptedEntry { .. } | BuildEntry { .. } => StatusCode::Internal, diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index e883f1833809..33d26c8196df 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -25,8 +25,8 @@ mod handle_manifest; mod handle_open; mod handle_truncate; mod handle_write; - use std::collections::HashMap; +use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -50,7 +50,7 @@ use crate::cache::write_cache::{WriteCache, WriteCacheRef}; use crate::cache::{CacheManager, CacheManagerRef}; use crate::compaction::CompactionScheduler; use crate::config::MitoConfig; -use crate::error::{JoinSnafu, Result, WorkerStoppedSnafu}; +use crate::error::{CreateDirSnafu, JoinSnafu, Result, WorkerStoppedSnafu}; use crate::flush::{FlushScheduler, WriteBufferManagerImpl, WriteBufferManagerRef}; use crate::memtable::MemtableBuilderProvider; use crate::metrics::{REGION_COUNT, WRITE_STALL_TOTAL}; @@ -373,6 +373,12 @@ async fn write_cache_from_config( // TODO(yingwen): Remove this and document the config once the write cache is ready. warn!("Write cache is an experimental feature"); + tokio::fs::create_dir_all(Path::new(&config.experimental_write_cache_path)) + .await + .context(CreateDirSnafu { + dir: &config.experimental_write_cache_path, + })?; + let cache = WriteCache::new_fs( &config.experimental_write_cache_path, object_store_manager, diff --git a/src/object-store/src/lib.rs b/src/object-store/src/lib.rs index 797e75f42d54..851484a0cffa 100644 --- a/src/object-store/src/lib.rs +++ b/src/object-store/src/lib.rs @@ -24,3 +24,5 @@ pub mod manager; mod metrics; pub mod test_util; pub mod util; +/// The default object cache directory name. +pub const OBJECT_CACHE_DIR: &str = "object_cache"; diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index b3a7269ae003..57ec7f6f86f9 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -233,6 +233,9 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te if *store_type == StorageType::S3WithCache { s3_config.cache.cache_path = Some("/tmp/greptimedb_cache".to_string()); + } else { + // An empty string means disabling. + s3_config.cache.cache_path = Some("".to_string()); } let mut builder = S3::default() diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 106c906372b2..083a9daa1a26 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -917,7 +917,7 @@ compress_manifest = false auto_flush_interval = "30m" enable_experimental_write_cache = false experimental_write_cache_path = "" -experimental_write_cache_size = "1GiB" +experimental_write_cache_size = "5GiB" sst_write_buffer_size = "8MiB" parallel_scan_channel_size = 32 allow_stale_entries = false diff --git a/tests-integration/tests/main.rs b/tests-integration/tests/main.rs index 4fc19f24b284..b30820517f9b 100644 --- a/tests-integration/tests/main.rs +++ b/tests-integration/tests/main.rs @@ -22,8 +22,9 @@ mod sql; mod region_migration; grpc_tests!(File, S3, S3WithCache, Oss, Azblob, Gcs); + http_tests!(File, S3, S3WithCache, Oss, Azblob, Gcs); -// region_failover_tests!(File, S3, S3WithCache, Oss, Azblob); + sql_tests!(File); region_migration_tests!(File);