Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: bump opendal version to 0.48 #4499

Merged
merged 1 commit into from
Aug 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions src/common/datasource/src/object_store/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ use snafu::ResultExt;
use crate::error::{BuildBackendSnafu, Result};

pub fn build_fs_backend(root: &str) -> Result<ObjectStore> {
let mut builder = Fs::default();
let _ = builder.root(root);
let object_store = ObjectStore::new(builder)
let builder = Fs::default();
let object_store = ObjectStore::new(builder.root(root))
.context(BuildBackendSnafu)?
.layer(
object_store::layers::LoggingLayer::default()
Expand Down
16 changes: 7 additions & 9 deletions src/common/datasource/src/object_store/s3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,28 +44,26 @@ pub fn build_s3_backend(
path: &str,
connection: &HashMap<String, String>,
) -> Result<ObjectStore> {
let mut builder = S3::default();

let _ = builder.root(path).bucket(host);
let mut builder = S3::default().root(path).bucket(host);

if let Some(endpoint) = connection.get(ENDPOINT) {
let _ = builder.endpoint(endpoint);
builder = builder.endpoint(endpoint);
}

if let Some(region) = connection.get(REGION) {
let _ = builder.region(region);
builder = builder.region(region);
}

if let Some(key_id) = connection.get(ACCESS_KEY_ID) {
let _ = builder.access_key_id(key_id);
builder = builder.access_key_id(key_id);
}

if let Some(key) = connection.get(SECRET_ACCESS_KEY) {
let _ = builder.secret_access_key(key);
builder = builder.secret_access_key(key);
}

if let Some(session_token) = connection.get(SESSION_TOKEN) {
let _ = builder.security_token(session_token);
builder = builder.session_token(session_token);
}

if let Some(enable_str) = connection.get(ENABLE_VIRTUAL_HOST_STYLE) {
Expand All @@ -79,7 +77,7 @@ pub fn build_s3_backend(
.build()
})?;
if enable {
let _ = builder.enable_virtual_host_style();
builder = builder.enable_virtual_host_style();
}
}

Expand Down
12 changes: 4 additions & 8 deletions src/common/datasource/src/test_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,15 @@ pub fn format_schema(schema: Schema) -> Vec<String> {
}

pub fn test_store(root: &str) -> ObjectStore {
let mut builder = Fs::default();
let _ = builder.root(root);

ObjectStore::new(builder).unwrap().finish()
let builder = Fs::default();
ObjectStore::new(builder.root(root)).unwrap().finish()
}

pub fn test_tmp_store(root: &str) -> (ObjectStore, TempDir) {
let dir = create_temp_dir(root);

let mut builder = Fs::default();
let _ = builder.root("/");

(ObjectStore::new(builder).unwrap().finish(), dir)
let builder = Fs::default();
(ObjectStore::new(builder.root("/")).unwrap().finish(), dir)
}

pub fn test_basic_schema() -> SchemaRef {
Expand Down
5 changes: 2 additions & 3 deletions src/common/procedure/src/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -680,9 +680,8 @@ pub(crate) mod test_util {

pub(crate) fn new_object_store(dir: &TempDir) -> ObjectStore {
let store_dir = dir.path().to_str().unwrap();
let mut builder = Builder::default();
let _ = builder.root(store_dir);
ObjectStore::new(builder).unwrap().finish()
let builder = Builder::default();
ObjectStore::new(builder.root(store_dir)).unwrap().finish()
}
}

Expand Down
3 changes: 1 addition & 2 deletions src/common/procedure/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -361,8 +361,7 @@ mod tests {

fn procedure_store_for_test(dir: &TempDir) -> ProcedureStore {
let store_dir = dir.path().to_str().unwrap();
let mut builder = Builder::default();
let _ = builder.root(store_dir);
let builder = Builder::default().root(store_dir);
let object_store = ObjectStore::new(builder).unwrap().finish();

ProcedureStore::from_object_store(object_store)
Expand Down
6 changes: 2 additions & 4 deletions src/common/procedure/src/store/state_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,8 +220,7 @@ mod tests {
async fn test_object_state_store() {
let dir = create_temp_dir("state_store");
let store_dir = dir.path().to_str().unwrap();
let mut builder = Builder::default();
let _ = builder.root(store_dir);
let builder = Builder::default().root(store_dir);

let object_store = ObjectStore::new(builder).unwrap().finish();
let state_store = ObjectStateStore::new(object_store);
Expand Down Expand Up @@ -291,8 +290,7 @@ mod tests {
async fn test_object_state_store_delete() {
let dir = create_temp_dir("state_store_list");
let store_dir = dir.path().to_str().unwrap();
let mut builder = Builder::default();
let _ = builder.root(store_dir);
let builder = Builder::default().root(store_dir);

let object_store = ObjectStore::new(builder).unwrap().finish();
let state_store = ObjectStateStore::new(object_store);
Expand Down
10 changes: 5 additions & 5 deletions src/datanode/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,11 @@ async fn create_object_store_with_cache(
let atomic_temp_dir = join_dir(path, ".tmp/");
clean_temp_dir(&atomic_temp_dir)?;

let cache_store = {
let mut builder = Fs::default();
builder.root(path).atomic_write_dir(&atomic_temp_dir);
builder.build().context(error::InitBackendSnafu)?
};
let cache_store = Fs::default()
.root(path)
.atomic_write_dir(&atomic_temp_dir)
.build()
.context(error::InitBackendSnafu)?;

let cache_layer = LruCacheLayer::new(Arc::new(cache_store), cache_capacity.0 as usize)
.await
Expand Down
7 changes: 3 additions & 4 deletions src/datanode/src/store/azblob.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ pub(crate) async fn new_azblob_object_store(azblob_config: &AzblobConfig) -> Res
azblob_config.container, &root
);

let mut builder = Azblob::default();
let _ = builder
let mut builder = Azblob::default()
.root(&root)
.container(&azblob_config.container)
.endpoint(&azblob_config.endpoint)
Expand All @@ -40,8 +39,8 @@ pub(crate) async fn new_azblob_object_store(azblob_config: &AzblobConfig) -> Res
.http_client(build_http_client()?);

if let Some(token) = &azblob_config.sas_token {
let _ = builder.sas_token(token);
}
builder = builder.sas_token(token);
};

Ok(ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
Expand Down
5 changes: 3 additions & 2 deletions src/datanode/src/store/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ pub(crate) async fn new_fs_object_store(
let atomic_write_dir = join_dir(data_home, ".tmp/");
store::clean_temp_dir(&atomic_write_dir)?;

let mut builder = Fs::default();
let _ = builder.root(data_home).atomic_write_dir(&atomic_write_dir);
let builder = Fs::default()
.root(data_home)
.atomic_write_dir(&atomic_write_dir);

let object_store = ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
Expand Down
3 changes: 1 addition & 2 deletions src/datanode/src/store/gcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ pub(crate) async fn new_gcs_object_store(gcs_config: &GcsConfig) -> Result<Objec
gcs_config.bucket, &root
);

let mut builder = Gcs::default();
builder
let builder = Gcs::default()
.root(&root)
.bucket(&gcs_config.bucket)
.scope(&gcs_config.scope)
Expand Down
3 changes: 1 addition & 2 deletions src/datanode/src/store/oss.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ pub(crate) async fn new_oss_object_store(oss_config: &OssConfig) -> Result<Objec
oss_config.bucket, &root
);

let mut builder = Oss::default();
let _ = builder
let builder = Oss::default()
.root(&root)
.bucket(&oss_config.bucket)
.endpoint(&oss_config.endpoint)
Expand Down
11 changes: 5 additions & 6 deletions src/datanode/src/store/s3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,19 @@ pub(crate) async fn new_s3_object_store(s3_config: &S3Config) -> Result<ObjectSt
s3_config.bucket, &root
);

let mut builder = S3::default();
let _ = builder
let mut builder = S3::default()
.root(&root)
.bucket(&s3_config.bucket)
.access_key_id(s3_config.access_key_id.expose_secret())
.secret_access_key(s3_config.secret_access_key.expose_secret())
.http_client(build_http_client()?);

if s3_config.endpoint.is_some() {
let _ = builder.endpoint(s3_config.endpoint.as_ref().unwrap());
}
builder = builder.endpoint(s3_config.endpoint.as_ref().unwrap());
};
if s3_config.region.is_some() {
let _ = builder.region(s3_config.region.as_ref().unwrap());
}
builder = builder.region(s3_config.region.as_ref().unwrap());
};

Ok(ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
Expand Down
3 changes: 1 addition & 2 deletions src/file-engine/src/test_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ use store_api::metadata::ColumnMetadata;
pub fn new_test_object_store(prefix: &str) -> (TempDir, ObjectStore) {
let dir = create_temp_dir(prefix);
let store_dir = dir.path().to_string_lossy();
let mut builder = Fs::default();
let _ = builder.root(&store_dir);
let builder = Fs::default().root(&store_dir);
(dir, ObjectStore::new(builder).unwrap().finish())
}

Expand Down
3 changes: 1 addition & 2 deletions src/metric-engine/src/test_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -307,8 +307,7 @@ mod test {
env.init_metric_region().await;
let region_id = to_metadata_region_id(env.default_physical_region_id());

let mut builder = Fs::default();
builder.root(&env.data_home());
let builder = Fs::default().root(&env.data_home());
let object_store = ObjectStore::new(builder).unwrap().finish();

let region_dir = "test_metric_region";
Expand Down
3 changes: 1 addition & 2 deletions src/mito2/src/access_layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,7 @@ pub(crate) async fn new_fs_cache_store(root: &str) -> Result<ObjectStore> {
let atomic_write_dir = join_dir(root, ".tmp/");
clean_dir(&atomic_write_dir).await?;

let mut builder = Fs::default();
builder.root(root).atomic_write_dir(&atomic_write_dir);
let builder = Fs::default().root(root).atomic_write_dir(&atomic_write_dir);
let store = ObjectStore::new(builder).context(OpenDalSnafu)?.finish();

Ok(with_instrument_layers(store, false))
Expand Down
3 changes: 1 addition & 2 deletions src/mito2/src/cache/file_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -382,8 +382,7 @@ mod tests {
use super::*;

fn new_fs_store(path: &str) -> ObjectStore {
let mut builder = Fs::default();
builder.root(path);
let builder = Fs::default().root(path);
ObjectStore::new(builder).unwrap().finish()
}

Expand Down
5 changes: 2 additions & 3 deletions src/mito2/src/cache/test_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ fn parquet_file_data() -> Vec<u8> {
}

pub(crate) fn new_fs_store(path: &str) -> ObjectStore {
let mut builder = Fs::default();
builder.root(path);
ObjectStore::new(builder).unwrap().finish()
let builder = Fs::default();
ObjectStore::new(builder.root(path)).unwrap().finish()
}
3 changes: 1 addition & 2 deletions src/mito2/src/manifest/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -642,8 +642,7 @@ mod tests {
fn new_test_manifest_store() -> ManifestObjectStore {
common_telemetry::init_default_ut_logging();
let tmp_dir = create_temp_dir("test_manifest_log_store");
let mut builder = Fs::default();
let _ = builder.root(&tmp_dir.path().to_string_lossy());
let builder = Fs::default().root(&tmp_dir.path().to_string_lossy());
let object_store = ObjectStore::new(builder).unwrap().finish();
ManifestObjectStore::new(
"/",
Expand Down
6 changes: 2 additions & 4 deletions src/mito2/src/sst/file_purger.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,7 @@ mod tests {

let dir = create_temp_dir("file-purge");
let dir_path = dir.path().display().to_string();
let mut builder = Fs::default();
builder.root(&dir_path);
let builder = Fs::default().root(&dir_path);
let sst_file_id = FileId::random();
let sst_dir = "table1";
let path = location::sst_file_path(sst_dir, sst_file_id);
Expand Down Expand Up @@ -171,8 +170,7 @@ mod tests {

let dir = create_temp_dir("file-purge");
let dir_path = dir.path().display().to_string();
let mut builder = Fs::default();
builder.root(&dir_path);
let builder = Fs::default().root(&dir_path);
let sst_file_id = FileId::random();
let sst_dir = "table1";

Expand Down
13 changes: 6 additions & 7 deletions src/mito2/src/test_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -365,8 +365,7 @@ impl TestEnv {
.display()
.to_string();
let mut builder = Fs::default();
builder.root(&data_path);
let object_store = ObjectStore::new(builder).unwrap().finish();
let object_store = ObjectStore::new(builder.root(&data_path)).unwrap().finish();
object_store_manager.add(storage_name, object_store);
}
let object_store_manager = Arc::new(object_store_manager);
Expand Down Expand Up @@ -553,8 +552,7 @@ impl TestEnv {
fn create_object_store_manager(&self) -> ObjectStoreManager {
let data_home = self.data_home.path();
let data_path = data_home.join("data").as_path().display().to_string();
let mut builder = Fs::default();
builder.root(&data_path);
let builder = Fs::default().root(&data_path);
let object_store = ObjectStore::new(builder).unwrap().finish();
ObjectStoreManager::new("default", object_store)
}
Expand All @@ -570,9 +568,10 @@ impl TestEnv {
let data_home = self.data_home.path();
let manifest_dir = data_home.join("manifest").as_path().display().to_string();

let mut builder = Fs::default();
builder.root(&manifest_dir);
let object_store = ObjectStore::new(builder).unwrap().finish();
let builder = Fs::default();
let object_store = ObjectStore::new(builder.root(&manifest_dir))
.unwrap()
.finish();

// The "manifest_dir" here should be the relative path from the `object_store`'s root.
// Otherwise the OpenDal's list operation would fail with "StripPrefixError". This is
Expand Down
Loading
Loading