From fe45558344753aa44b0484b0a1231abfd3fb4951 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Fri, 31 May 2024 23:48:04 +0800 Subject: [PATCH 01/21] move object_storage and dynamic_file_catalog_provider to core --- datafusion-cli/src/exec.rs | 3 +- datafusion-cli/src/lib.rs | 2 - datafusion-cli/src/main.rs | 2 +- .../core/src/catalog/dynamic_file_catalog.rs | 80 ++++++++++--------- datafusion/core/src/catalog/mod.rs | 1 + .../core/src/datasource/file_format/mod.rs | 1 + .../datasource/file_format}/object_storage.rs | 29 ++++--- 7 files changed, 63 insertions(+), 55 deletions(-) rename datafusion-cli/src/catalog.rs => datafusion/core/src/catalog/dynamic_file_catalog.rs (89%) rename {datafusion-cli/src => datafusion/core/src/datasource/file_format}/object_storage.rs (97%) diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index 855d6a7cbbc9..46b705f7d7a9 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -28,13 +28,12 @@ use crate::print_format::PrintFormat; use crate::{ command::{Command, OutputFormat}, helper::{unescape_input, CliHelper}, - object_storage::{get_object_store, register_options}, print_options::{MaxRows, PrintOptions}, }; use datafusion::common::instant::Instant; use datafusion::common::plan_datafusion_err; -use datafusion::datasource::listing::ListingTableUrl; +use datafusion::datasource::{listing::ListingTableUrl, file_format::object_storage::{get_object_store, register_options}}; use datafusion::error::{DataFusionError, Result}; use datafusion::logical_expr::{DdlStatement, LogicalPlan}; use datafusion::physical_plan::{collect, execute_stream, ExecutionPlanProperties}; diff --git a/datafusion-cli/src/lib.rs b/datafusion-cli/src/lib.rs index 139a60b8cf16..a9cb9cd91b0f 100644 --- a/datafusion-cli/src/lib.rs +++ b/datafusion-cli/src/lib.rs @@ -18,12 +18,10 @@ #![doc = include_str!("../README.md")] pub const DATAFUSION_CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); -pub mod catalog; pub mod command; pub mod exec; pub mod functions; pub mod helper; pub mod highlighter; -pub mod object_storage; pub mod print_format; pub mod print_options; diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index d81a73df20a4..a7517a1a2db0 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -27,7 +27,7 @@ use datafusion::execution::context::SessionConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool}; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; use datafusion::prelude::SessionContext; -use datafusion_cli::catalog::DynamicFileCatalog; +use datafusion::catalog::dynamic_file_catalog::DynamicFileCatalog; use datafusion_cli::functions::ParquetMetadataFunc; use datafusion_cli::{ exec, diff --git a/datafusion-cli/src/catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs similarity index 89% rename from datafusion-cli/src/catalog.rs rename to datafusion/core/src/catalog/dynamic_file_catalog.rs index faa657da6511..deec6f3ba8eb 100644 --- a/datafusion-cli/src/catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -15,24 +15,23 @@ // specific language governing permissions and limitations // under the License. -use std::any::Any; -use std::sync::{Arc, Weak}; - -use crate::object_storage::{get_object_store, AwsOptions, GcpOptions}; +//! Implementation of a dynamic file catalog that automatically creates table providers +//! for the file paths. -use datafusion::catalog::schema::SchemaProvider; -use datafusion::catalog::{CatalogProvider, CatalogProviderList}; -use datafusion::common::plan_datafusion_err; -use datafusion::datasource::listing::{ - ListingTable, ListingTableConfig, ListingTableUrl, +use crate::catalog::schema::SchemaProvider; +use crate::catalog::{CatalogProvider, CatalogProviderList}; +use crate::datasource::file_format::object_storage::{ + get_object_store, AwsOptions, GcpOptions, }; -use datafusion::datasource::TableProvider; -use datafusion::error::Result; -use datafusion::execution::context::SessionState; - +use crate::datasource::listing::{ListingTable, ListingTableConfig, ListingTableUrl}; +use crate::datasource::TableProvider; +use crate::execution::context::SessionState; use async_trait::async_trait; +use datafusion_common::plan_datafusion_err; use dirs::home_dir; use parking_lot::RwLock; +use std::any::Any; +use std::sync::{Arc, Weak}; /// Wraps another catalog, automatically creating table providers /// for local files if needed @@ -42,6 +41,7 @@ pub struct DynamicFileCatalog { } impl DynamicFileCatalog { + /// Create a new dynamic file catalog pub fn new( inner: Arc, state: Weak>, @@ -50,6 +50,21 @@ impl DynamicFileCatalog { } } +/// Wraps another catalog provider +struct DynamicFileCatalogProvider { + inner: Arc, + state: Weak>, +} + +impl DynamicFileCatalogProvider { + pub fn new( + inner: Arc, + state: Weak>, + ) -> Self { + Self { inner, state } + } +} + impl CatalogProviderList for DynamicFileCatalog { fn as_any(&self) -> &dyn Any { self @@ -75,21 +90,6 @@ impl CatalogProviderList for DynamicFileCatalog { } } -/// Wraps another catalog provider -struct DynamicFileCatalogProvider { - inner: Arc, - state: Weak>, -} - -impl DynamicFileCatalogProvider { - pub fn new( - inner: Arc, - state: Weak>, - ) -> Self { - Self { inner, state } - } -} - impl CatalogProvider for DynamicFileCatalogProvider { fn as_any(&self) -> &dyn Any { self @@ -110,7 +110,7 @@ impl CatalogProvider for DynamicFileCatalogProvider { &self, name: &str, schema: Arc, - ) -> Result>> { + ) -> datafusion_common::Result>> { self.inner.register_schema(name, schema) } } @@ -144,11 +144,14 @@ impl SchemaProvider for DynamicFileSchemaProvider { &self, name: String, table: Arc, - ) -> Result>> { + ) -> datafusion_common::Result>> { self.inner.register_table(name, table) } - async fn table(&self, name: &str) -> Result>> { + async fn table( + &self, + name: &str, + ) -> datafusion_common::Result>> { let inner_table = self.inner.table(name).await?; if inner_table.is_some() { return Ok(inner_table); @@ -207,7 +210,10 @@ impl SchemaProvider for DynamicFileSchemaProvider { Ok(Some(Arc::new(ListingTable::try_new(config)?))) } - fn deregister_table(&self, name: &str) -> Result>> { + fn deregister_table( + &self, + name: &str, + ) -> datafusion_common::Result>> { self.inner.deregister_table(name) } @@ -230,8 +236,8 @@ fn substitute_tilde(cur: String) -> String { mod tests { use super::*; - use datafusion::catalog::schema::SchemaProvider; - use datafusion::prelude::SessionContext; + use crate::catalog::schema::SchemaProvider; + use crate::prelude::SessionContext; fn setup_context() -> (SessionContext, Arc) { let mut ctx = SessionContext::new(); @@ -253,7 +259,7 @@ mod tests { } #[tokio::test] - async fn query_http_location_test() -> Result<()> { + async fn query_http_location_test() -> datafusion_common::Result<()> { // This is a unit test so not expecting a connection or a file to be // available let domain = "example.com"; @@ -280,7 +286,7 @@ mod tests { } #[tokio::test] - async fn query_s3_location_test() -> Result<()> { + async fn query_s3_location_test() -> datafusion_common::Result<()> { let bucket = "examples3bucket"; let location = format!("s3://{bucket}/file.parquet"); @@ -302,7 +308,7 @@ mod tests { } #[tokio::test] - async fn query_gs_location_test() -> Result<()> { + async fn query_gs_location_test() -> datafusion_common::Result<()> { let bucket = "examplegsbucket"; let location = format!("gs://{bucket}/file.parquet"); diff --git a/datafusion/core/src/catalog/mod.rs b/datafusion/core/src/catalog/mod.rs index 209d9b2af297..f60b196834b6 100644 --- a/datafusion/core/src/catalog/mod.rs +++ b/datafusion/core/src/catalog/mod.rs @@ -17,6 +17,7 @@ //! Interfaces and default implementations of catalogs and schemas. +pub mod dynamic_file_catalog; pub mod information_schema; pub mod listing_schema; pub mod schema; diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 7cc3421ebb48..9608cf8ea591 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -26,6 +26,7 @@ pub mod avro; pub mod csv; pub mod file_compression_type; pub mod json; +pub mod object_storage; pub mod options; #[cfg(feature = "parquet")] pub mod parquet; diff --git a/datafusion-cli/src/object_storage.rs b/datafusion/core/src/datasource/file_format/object_storage.rs similarity index 97% rename from datafusion-cli/src/object_storage.rs rename to datafusion/core/src/datasource/file_format/object_storage.rs index 85e0009bd267..2be703b67eb1 100644 --- a/datafusion-cli/src/object_storage.rs +++ b/datafusion/core/src/datasource/file_format/object_storage.rs @@ -15,17 +15,19 @@ // specific language governing permissions and limitations // under the License. +//! Implementations for object storage builders for different cloud storage schemes. + use std::any::Any; use std::fmt::{Debug, Display}; use std::sync::Arc; -use datafusion::common::config::{ +use crate::common::config::{ ConfigEntry, ConfigExtension, ConfigField, ExtensionOptions, TableOptions, Visit, }; -use datafusion::common::{config_err, exec_datafusion_err, exec_err}; -use datafusion::error::{DataFusionError, Result}; -use datafusion::execution::context::SessionState; -use datafusion::prelude::SessionContext; +use crate::common::{config_err, exec_datafusion_err, exec_err}; +use crate::error::{DataFusionError, Result}; +use crate::execution::context::SessionState; +use crate::prelude::SessionContext; use async_trait::async_trait; use aws_credential_types::provider::ProvideCredentials; @@ -35,7 +37,7 @@ use object_store::http::HttpBuilder; use object_store::{CredentialProvider, ObjectStore}; use url::Url; -pub async fn get_s3_object_store_builder( +pub(crate) async fn get_s3_object_store_builder( url: &Url, aws_options: &AwsOptions, ) -> Result { @@ -132,14 +134,14 @@ impl CredentialProvider for S3CredentialProvider { } } -pub fn get_oss_object_store_builder( +pub(crate) fn get_oss_object_store_builder( url: &Url, aws_options: &AwsOptions, ) -> Result { get_object_store_builder(url, aws_options, true) } -pub fn get_cos_object_store_builder( +pub(crate) fn get_cos_object_store_builder( url: &Url, aws_options: &AwsOptions, ) -> Result { @@ -173,7 +175,7 @@ fn get_object_store_builder( Ok(builder) } -pub fn get_gcs_object_store_builder( +pub(crate) fn get_gcs_object_store_builder( url: &Url, gs_options: &GcpOptions, ) -> Result { @@ -416,7 +418,7 @@ impl ConfigExtension for GcpOptions { /// Google Cloud Storage. /// /// NOTE: This function will not perform any action when given an unsupported scheme. -pub(crate) fn register_options(ctx: &SessionContext, scheme: &str) { +pub fn register_options(ctx: &SessionContext, scheme: &str) { // Match the provided scheme against supported cloud storage schemes: match scheme { // For Amazon S3 or Alibaba Cloud OSS @@ -434,7 +436,8 @@ pub(crate) fn register_options(ctx: &SessionContext, scheme: &str) { } } -pub(crate) async fn get_object_store( +/// Used to get an object store based on the given scheme and options. +pub async fn get_object_store( state: &SessionState, scheme: &str, url: &Url, @@ -500,8 +503,8 @@ pub(crate) async fn get_object_store( mod tests { use super::*; - use datafusion::common::plan_err; - use datafusion::{ + use crate::common::plan_err; + use crate::{ datasource::listing::ListingTableUrl, logical_expr::{DdlStatement, LogicalPlan}, prelude::SessionContext, From 30f4d71c57abe6e07c9495dfa377a4d714e5cbed Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:12:06 +0800 Subject: [PATCH 02/21] modified cargo toml and update lock --- Cargo.toml | 2 +- datafusion-cli/Cargo.lock | 5 +++-- datafusion-cli/Cargo.toml | 2 -- datafusion/core/Cargo.toml | 3 +++ 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 45504be3f1ba..5628acc6515e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ indexmap = "2.0.0" itertools = "0.12" log = "^0.4" num_cpus = "1.13.0" -object_store = { version = "0.9.1", default-features = false } +object_store = { version = "0.9.1", features = ["aws", "gcp", "http"] } parking_lot = "0.12" parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] } rand = "0.8" diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 6a1ba8aba005..d42bbcbe1731 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -1127,6 +1127,8 @@ dependencies = [ "arrow-schema", "async-compression", "async-trait", + "aws-config", + "aws-credential-types", "bytes", "bzip2", "chrono", @@ -1143,6 +1145,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-sql", + "dirs", "flate2", "futures", "glob", @@ -1175,8 +1178,6 @@ dependencies = [ "arrow", "assert_cmd", "async-trait", - "aws-config", - "aws-credential-types", "clap", "ctor", "datafusion", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 4e3d800cfe97..55bd32a60ec7 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -32,8 +32,6 @@ readme = "README.md" [dependencies] arrow = "51.0.0" async-trait = "0.1.41" -aws-config = "0.55" -aws-credential-types = "0.55" clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "38.0.0", features = [ "avro", diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 9f1f7484357b..f0db30c20017 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -92,6 +92,8 @@ async-compression = { version = "0.4.0", features = [ "tokio", ], optional = true } async-trait = { workspace = true } +aws-config = "0.55" +aws-credential-types = "0.55" bytes = { workspace = true } bzip2 = { version = "0.4.3", optional = true } chrono = { workspace = true } @@ -108,6 +110,7 @@ datafusion-physical-expr = { workspace = true } datafusion-physical-expr-common = { workspace = true } datafusion-physical-plan = { workspace = true } datafusion-sql = { workspace = true } +dirs = "4.0.0" flate2 = { version = "1.0.24", optional = true } futures = { workspace = true } glob = "0.3.0" From fc79e91d925b6e1e2969a741437ea7fc7fc1d4e4 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:29:59 +0800 Subject: [PATCH 03/21] rollback some unnecessary chaged --- .../core/src/catalog/dynamic_file_catalog.rs | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index deec6f3ba8eb..4c958ed0fb47 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -28,6 +28,7 @@ use crate::datasource::TableProvider; use crate::execution::context::SessionState; use async_trait::async_trait; use datafusion_common::plan_datafusion_err; +use datafusion_common::Result; use dirs::home_dir; use parking_lot::RwLock; use std::any::Any; @@ -50,21 +51,6 @@ impl DynamicFileCatalog { } } -/// Wraps another catalog provider -struct DynamicFileCatalogProvider { - inner: Arc, - state: Weak>, -} - -impl DynamicFileCatalogProvider { - pub fn new( - inner: Arc, - state: Weak>, - ) -> Self { - Self { inner, state } - } -} - impl CatalogProviderList for DynamicFileCatalog { fn as_any(&self) -> &dyn Any { self @@ -110,11 +96,26 @@ impl CatalogProvider for DynamicFileCatalogProvider { &self, name: &str, schema: Arc, - ) -> datafusion_common::Result>> { + ) -> Result>> { self.inner.register_schema(name, schema) } } +/// Wraps another catalog provider +struct DynamicFileCatalogProvider { + inner: Arc, + state: Weak>, +} + +impl DynamicFileCatalogProvider { + pub fn new( + inner: Arc, + state: Weak>, + ) -> Self { + Self { inner, state } + } +} + /// Wraps another schema provider struct DynamicFileSchemaProvider { inner: Arc, @@ -144,14 +145,14 @@ impl SchemaProvider for DynamicFileSchemaProvider { &self, name: String, table: Arc, - ) -> datafusion_common::Result>> { + ) -> Result>> { self.inner.register_table(name, table) } async fn table( &self, name: &str, - ) -> datafusion_common::Result>> { + ) -> Result>> { let inner_table = self.inner.table(name).await?; if inner_table.is_some() { return Ok(inner_table); @@ -213,7 +214,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { fn deregister_table( &self, name: &str, - ) -> datafusion_common::Result>> { + ) -> Result>> { self.inner.deregister_table(name) } @@ -259,7 +260,7 @@ mod tests { } #[tokio::test] - async fn query_http_location_test() -> datafusion_common::Result<()> { + async fn query_http_location_test() -> Result<()> { // This is a unit test so not expecting a connection or a file to be // available let domain = "example.com"; @@ -286,7 +287,7 @@ mod tests { } #[tokio::test] - async fn query_s3_location_test() -> datafusion_common::Result<()> { + async fn query_s3_location_test() -> Result<()> { let bucket = "examples3bucket"; let location = format!("s3://{bucket}/file.parquet"); @@ -308,7 +309,7 @@ mod tests { } #[tokio::test] - async fn query_gs_location_test() -> datafusion_common::Result<()> { + async fn query_gs_location_test() -> Result<()> { let bucket = "examplegsbucket"; let location = format!("gs://{bucket}/file.parquet"); From 236248c66111d42cadf05d603ed1aa99c10f8d94 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:30:35 +0800 Subject: [PATCH 04/21] cargo fmt --- datafusion/core/src/catalog/dynamic_file_catalog.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index 4c958ed0fb47..002f76e60df4 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -149,10 +149,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { self.inner.register_table(name, table) } - async fn table( - &self, - name: &str, - ) -> Result>> { + async fn table(&self, name: &str) -> Result>> { let inner_table = self.inner.table(name).await?; if inner_table.is_some() { return Ok(inner_table); @@ -211,10 +208,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { Ok(Some(Arc::new(ListingTable::try_new(config)?))) } - fn deregister_table( - &self, - name: &str, - ) -> Result>> { + fn deregister_table(&self, name: &str) -> Result>> { self.inner.deregister_table(name) } From dac401e134652d80c43d6326598648637c31c600 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:32:58 +0800 Subject: [PATCH 05/21] rollback the struct position --- .../core/src/catalog/dynamic_file_catalog.rs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index 002f76e60df4..88598c12a75d 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -76,6 +76,21 @@ impl CatalogProviderList for DynamicFileCatalog { } } +/// Wraps another schema provider +struct DynamicFileSchemaProvider { + inner: Arc, + state: Weak>, +} + +impl DynamicFileSchemaProvider { + pub fn new( + inner: Arc, + state: Weak>, + ) -> Self { + Self { inner, state } + } +} + impl CatalogProvider for DynamicFileCatalogProvider { fn as_any(&self) -> &dyn Any { self @@ -116,21 +131,6 @@ impl DynamicFileCatalogProvider { } } -/// Wraps another schema provider -struct DynamicFileSchemaProvider { - inner: Arc, - state: Weak>, -} - -impl DynamicFileSchemaProvider { - pub fn new( - inner: Arc, - state: Weak>, - ) -> Self { - Self { inner, state } - } -} - #[async_trait] impl SchemaProvider for DynamicFileSchemaProvider { fn as_any(&self) -> &dyn Any { From ebb777a287840d77e7c0128708c6c026327bfb76 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:35:13 +0800 Subject: [PATCH 06/21] switch the position --- .../core/src/catalog/dynamic_file_catalog.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index 88598c12a75d..50d28221569e 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -76,15 +76,15 @@ impl CatalogProviderList for DynamicFileCatalog { } } -/// Wraps another schema provider -struct DynamicFileSchemaProvider { - inner: Arc, +/// Wraps another catalog provider +struct DynamicFileCatalogProvider { + inner: Arc, state: Weak>, } -impl DynamicFileSchemaProvider { +impl DynamicFileCatalogProvider { pub fn new( - inner: Arc, + inner: Arc, state: Weak>, ) -> Self { Self { inner, state } @@ -116,15 +116,15 @@ impl CatalogProvider for DynamicFileCatalogProvider { } } -/// Wraps another catalog provider -struct DynamicFileCatalogProvider { - inner: Arc, +/// Wraps another schema provider +struct DynamicFileSchemaProvider { + inner: Arc, state: Weak>, } -impl DynamicFileCatalogProvider { +impl DynamicFileSchemaProvider { pub fn new( - inner: Arc, + inner: Arc, state: Weak>, ) -> Self { Self { inner, state } From 937d1b0777b06adbbac976d0491b26f03857b04c Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:42:30 +0800 Subject: [PATCH 07/21] fix cargo fmt check --- datafusion-cli/src/exec.rs | 5 ++++- datafusion-cli/src/main.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index 46b705f7d7a9..f19bf1fb316c 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -33,7 +33,10 @@ use crate::{ use datafusion::common::instant::Instant; use datafusion::common::plan_datafusion_err; -use datafusion::datasource::{listing::ListingTableUrl, file_format::object_storage::{get_object_store, register_options}}; +use datafusion::datasource::{ + file_format::object_storage::{get_object_store, register_options}, + listing::ListingTableUrl, +}; use datafusion::error::{DataFusionError, Result}; use datafusion::logical_expr::{DdlStatement, LogicalPlan}; use datafusion::physical_plan::{collect, execute_stream, ExecutionPlanProperties}; diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index a7517a1a2db0..8d1258cd5401 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -22,12 +22,12 @@ use std::process::ExitCode; use std::str::FromStr; use std::sync::{Arc, OnceLock}; +use datafusion::catalog::dynamic_file_catalog::DynamicFileCatalog; use datafusion::error::{DataFusionError, Result}; use datafusion::execution::context::SessionConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool}; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; use datafusion::prelude::SessionContext; -use datafusion::catalog::dynamic_file_catalog::DynamicFileCatalog; use datafusion_cli::functions::ParquetMetadataFunc; use datafusion_cli::{ exec, From 6c93f6f9c22a03df848ad6243c3e6de16aa89428 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 00:55:53 +0800 Subject: [PATCH 08/21] rollback the lock --- datafusion-cli/Cargo.lock | 159 +++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index d42bbcbe1731..f9d2d6d919ed 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -363,9 +363,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.11" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" dependencies = [ "bzip2", "flate2", @@ -387,7 +387,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -708,9 +708,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -869,9 +869,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.98" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" dependencies = [ "jobserver", "libc", @@ -1042,9 +1042,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1093,7 +1093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -1495,9 +1495,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.12.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "encoding_rs" @@ -1535,9 +1535,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1685,7 +1685,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -1747,9 +1747,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -1987,9 +1987,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.13" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", "js-sys", @@ -2114,9 +2114,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libflate" @@ -2150,9 +2150,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.38" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7bb23d733dfcc8af652a78b7bf232f0e967710d044732185e561e47c0336b6" +checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" dependencies = [ "cc", "libc", @@ -2170,9 +2170,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -2228,9 +2228,9 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mimalloc" -version = "0.1.42" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9186d86b79b52f4a77af65604b51225e8db1d6ee7e3f41aec1e40829c71a176" +checksum = "9f41a2280ded0da56c8cf898babb86e8f10651a34adcfff190ae9a1159c6908d" dependencies = [ "libmimalloc-sys", ] @@ -2243,9 +2243,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -2289,9 +2289,9 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" [[package]] name = "num" -version = "0.4.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" dependencies = [ "num-bigint", "num-complex", @@ -2313,9 +2313,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] @@ -2348,10 +2348,11 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ + "autocfg", "num-bigint", "num-integer", "num-traits", @@ -2379,9 +2380,9 @@ dependencies = [ [[package]] name = "object" -version = "0.35.0" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -2452,9 +2453,9 @@ checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", "parking_lot_core", @@ -2531,9 +2532,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -2594,7 +2595,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -2683,9 +2684,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.84" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" dependencies = [ "unicode-ident", ] @@ -3000,9 +3001,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "51f344d206c5e1b010eec27349b815a4805f70a778895959d70b74b9b529b30a" [[package]] name = "rustls-webpki" @@ -3016,9 +3017,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" [[package]] name = "rustyline" @@ -3120,29 +3121,29 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ "itoa", "ryu", @@ -3270,7 +3271,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3316,7 +3317,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3329,7 +3330,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3351,9 +3352,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" dependencies = [ "proc-macro2", "quote", @@ -3422,22 +3423,22 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3507,9 +3508,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -3526,13 +3527,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3628,7 +3629,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3673,7 +3674,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] @@ -3827,7 +3828,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", "wasm-bindgen-shared", ] @@ -3861,7 +3862,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4126,14 +4127,14 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.61", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" [[package]] name = "zstd" From f64f2574037f76567d6b887625d6744fa1f0cae1 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 01:23:34 +0800 Subject: [PATCH 09/21] fix cargo audit --- datafusion-cli/Cargo.lock | 548 ++++++++++++++----------------------- datafusion/core/Cargo.toml | 4 +- 2 files changed, 215 insertions(+), 337 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index f9d2d6d919ed..e05d0774c6f6 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -363,9 +363,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "bzip2", "flate2", @@ -387,7 +387,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -418,153 +418,162 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" +checksum = "84f9625b71b3ee4adbfbca369c6680d156e316ed86d2c7199a2a134563917414" dependencies = [ "aws-credential-types", "aws-http", + "aws-runtime", "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand 1.9.0", + "fastrand", "hex", "http", "hyper", - "ring 0.16.20", + "ring", "time", "tokio", - "tower", "tracing", "zeroize", ] [[package]] name = "aws-credential-types" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" +checksum = "5924466398ac76ffd411d297b9d516dcebb0577f7344c0c15fd8e8e04d9c7895" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand 1.9.0", - "tokio", - "tracing", "zeroize", ] [[package]] -name = "aws-endpoint" -version = "0.55.3" +name = "aws-http" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" +checksum = "bb9a3aa335a105a00975c971f1dad403c3175f2a210d98f39345c6af53923912" dependencies = [ - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes", "http", - "regex", + "http-body", + "pin-project-lite", "tracing", ] [[package]] -name = "aws-http" -version = "0.55.3" +name = "aws-runtime" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" +checksum = "b75844ecbdf3dc5e0f5ac5fd1088fb1623849990ea9445d2826258ce63be4de5" dependencies = [ "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes", + "fastrand", "http", - "http-body", - "lazy_static", "percent-encoding", - "pin-project-lite", "tracing", + "uuid", ] [[package]] name = "aws-sdk-sso" -version = "0.28.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" +checksum = "c870aa95e1e85f837f74af2cc937b3f8e72e2315a89e524265875843655b4d47" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "http", "regex", - "tokio-stream", - "tower", "tracing", ] [[package]] -name = "aws-sdk-sts" -version = "0.28.0" +name = "aws-sdk-ssooidc" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" +checksum = "107ee812e46f9120e68d48bf985d2f2a538315bd8be8a3e54db619250cc4c95e" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", - "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", - "aws-smithy-xml", "aws-types", "bytes", "http", "regex", - "tower", "tracing", ] [[package]] -name = "aws-sig-auth" -version = "0.55.3" +name = "aws-sdk-sts" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" +checksum = "a4e3958c43a78f6c3822e62009a35802af5cc7c120fbe8e60b98565604569aae" dependencies = [ "aws-credential-types", - "aws-sigv4", + "aws-http", + "aws-runtime", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", "aws-types", "http", + "regex", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +checksum = "06130e3686db3c5ae2fc44b3516fffe6b4d4eccebe09bd8ccc4067f3c9c183fb" dependencies = [ + "aws-credential-types", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", "form_urlencoded", "hex", "hmac", @@ -579,53 +588,28 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" +checksum = "d787b7e07925b450bed90d9d29ac8e57006c9c2ac907151d175ac0e376bfee0e" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-client" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 1.9.0", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", - "lazy_static", - "pin-project-lite", - "rustls 0.20.9", - "tokio", - "tower", - "tracing", ] [[package]] name = "aws-smithy-http" -version = "0.55.3" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +checksum = "96daaad925331c72449423574fdc72b54af780d5a23ace3c0a6ad0ccbf378715" dependencies = [ + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http", "http-body", - "hyper", "once_cell", "percent-encoding", "pin-project-lite", @@ -634,72 +618,103 @@ dependencies = [ ] [[package]] -name = "aws-smithy-http-tower" -version = "0.55.3" +name = "aws-smithy-json" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" +checksum = "0ff985bee3fe21046dc501fadc1d04a1161977c55a0cbbccd9b111c18206aa64" dependencies = [ - "aws-smithy-http", "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", ] [[package]] -name = "aws-smithy-json" -version = "0.55.3" +name = "aws-smithy-query" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +checksum = "cb4006503693766d34717efc5f58325062845fce26a683a71b70f23156d72e67" dependencies = [ "aws-smithy-types", + "urlencoding", ] [[package]] -name = "aws-smithy-query" -version = "0.55.3" +name = "aws-smithy-runtime" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +checksum = "d28af854558601b4202a4273b9720aebe43d73e472143e6056f16e3bd90bc837" dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "urlencoding", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c68e17e754b86da350b43add38294189121a880e9c3fb454f83ff7044f5257" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +checksum = "d97b978d8a351ea5744206ecc643a1d3806628680e9f151b4d6b7a76fec1596f" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", + "serde", "time", ] [[package]] name = "aws-smithy-xml" -version = "0.55.3" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" +checksum = "97500a0d0884b9576e65076075f81d899cfbb84f7db5ca1dd317f0582204e528" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" +checksum = "61065f0c6070cb0f9aaddfa614605fb1049908481da71ba5b39b2ffca12f57e4" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "http", "rustc_version", @@ -708,9 +723,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -869,9 +884,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" dependencies = [ "jobserver", "libc", @@ -1042,9 +1057,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1093,7 +1108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -1495,9 +1510,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" [[package]] name = "encoding_rs" @@ -1535,9 +1550,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1553,15 +1568,6 @@ dependencies = [ "str-buf", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.1.0" @@ -1685,7 +1691,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -1747,9 +1753,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -1903,21 +1909,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.9", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1927,9 +1918,11 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.12", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", ] [[package]] @@ -1987,9 +1980,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", "js-sys", @@ -2114,9 +2107,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libflate" @@ -2150,9 +2143,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" +checksum = "0e7bb23d733dfcc8af652a78b7bf232f0e967710d044732185e561e47c0336b6" dependencies = [ "cc", "libc", @@ -2170,9 +2163,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" @@ -2228,9 +2221,9 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mimalloc" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f41a2280ded0da56c8cf898babb86e8f10651a34adcfff190ae9a1159c6908d" +checksum = "e9186d86b79b52f4a77af65604b51225e8db1d6ee7e3f41aec1e40829c71a176" dependencies = [ "libmimalloc-sys", ] @@ -2243,9 +2236,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -2289,9 +2282,9 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" [[package]] name = "num" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -2313,9 +2306,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -2348,11 +2341,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -2380,9 +2372,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -2407,7 +2399,7 @@ dependencies = [ "quick-xml", "rand", "reqwest", - "ring 0.17.8", + "ring", "rustls-pemfile 2.1.2", "serde", "serde_json", @@ -2453,9 +2445,9 @@ checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -2532,9 +2524,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -2578,26 +2570,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.61", -] - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2684,9 +2656,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" dependencies = [ "unicode-ident", ] @@ -2826,7 +2798,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.2", + "hyper-rustls", "ipnet", "js-sys", "log", @@ -2834,7 +2806,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.12", + "rustls", "rustls-native-certs", "rustls-pemfile 1.0.4", "serde", @@ -2843,7 +2815,7 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -2854,21 +2826,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.8" @@ -2879,8 +2836,8 @@ dependencies = [ "cfg-if", "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "spin", + "untrusted", "windows-sys 0.52.0", ] @@ -2944,18 +2901,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" -dependencies = [ - "log", - "ring 0.16.20", - "sct", - "webpki", -] - [[package]] name = "rustls" version = "0.21.12" @@ -2963,7 +2908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.8", + "ring", "rustls-webpki", "sct", ] @@ -3001,9 +2946,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f344d206c5e1b010eec27349b815a4805f70a778895959d70b74b9b529b30a" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" @@ -3011,15 +2956,15 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] name = "rustversion" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rustyline" @@ -3080,8 +3025,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3121,29 +3066,29 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.200" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.200" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -3241,12 +3186,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -3271,7 +3210,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -3317,7 +3256,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -3330,7 +3269,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -3352,9 +3291,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.61" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -3395,7 +3334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand", "rustix", "windows-sys 0.52.0", ] @@ -3423,22 +3362,22 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -3508,9 +3447,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -3527,24 +3466,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", -] - -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.9", - "tokio", - "webpki", + "syn 2.0.66", ] [[package]] @@ -3553,18 +3481,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.12", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" -dependencies = [ - "futures-core", - "pin-project-lite", + "rustls", "tokio", ] @@ -3581,28 +3498,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -3615,7 +3510,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3629,7 +3523,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -3674,7 +3568,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -3716,12 +3610,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -3828,7 +3716,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -3862,7 +3750,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3896,16 +3784,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - [[package]] name = "winapi" version = "0.3.9" @@ -4127,14 +4005,14 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index f0db30c20017..5bb4187779ca 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -92,8 +92,8 @@ async-compression = { version = "0.4.0", features = [ "tokio", ], optional = true } async-trait = { workspace = true } -aws-config = "0.55" -aws-credential-types = "0.55" +aws-config = "0.101.0" +aws-credential-types = "0.101.0" bytes = { workspace = true } bzip2 = { version = "0.4.3", optional = true } chrono = { workspace = true } From ea05807b88f9fc48ea30fec191a199a63d98d815 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 01:39:50 +0800 Subject: [PATCH 10/21] fix warning: remove deprecated method --- datafusion/core/src/datasource/file_format/object_storage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/datasource/file_format/object_storage.rs b/datafusion/core/src/datasource/file_format/object_storage.rs index 2be703b67eb1..f2c19bc0583c 100644 --- a/datafusion/core/src/datasource/file_format/object_storage.rs +++ b/datafusion/core/src/datasource/file_format/object_storage.rs @@ -30,6 +30,7 @@ use crate::execution::context::SessionState; use crate::prelude::SessionContext; use async_trait::async_trait; +use aws_config::BehaviorVersion; use aws_credential_types::provider::ProvideCredentials; use object_store::aws::{AmazonS3Builder, AwsCredential}; use object_store::gcp::GoogleCloudStorageBuilder; @@ -64,7 +65,7 @@ pub(crate) async fn get_s3_object_store_builder( builder = builder.with_token(session_token); } } else { - let config = aws_config::from_env().load().await; + let config = aws_config::load_defaults(BehaviorVersion::latest()).await; if let Some(region) = config.region() { builder = builder.with_region(region.to_string()); } From a3fb3902efc3040e3da22262e6eec991bf8e44ed Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sat, 1 Jun 2024 11:57:06 +0800 Subject: [PATCH 11/21] fix test --- datafusion/core/src/datasource/file_format/object_storage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/datasource/file_format/object_storage.rs b/datafusion/core/src/datasource/file_format/object_storage.rs index f2c19bc0583c..8633a779e142 100644 --- a/datafusion/core/src/datasource/file_format/object_storage.rs +++ b/datafusion/core/src/datasource/file_format/object_storage.rs @@ -591,7 +591,8 @@ mod tests { .await .unwrap_err(); - assert_eq!(err.to_string(), "Invalid or Unsupported Configuration: Invalid endpoint: http://endpoint33. HTTP is not allowed for S3 endpoints. To allow HTTP, set 'aws.allow_http' to true"); + // There are other backstraces in the error message, so we just check for containing the message + assert_eq!(err.to_string().contains("Invalid or Unsupported Configuration: Invalid endpoint: http://endpoint33. HTTP is not allowed for S3 endpoints. To allow HTTP, set 'aws.allow_http' to true"), true); } else { return plan_err!("LogicalPlan is not a CreateExternalTable"); } From c2ce0568be866dd84d623fe175373d64d189980c Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 19:14:08 +0800 Subject: [PATCH 12/21] exclude aws, gcp, http feature for wasm --- Cargo.toml | 2 +- datafusion-cli/Cargo.lock | 419 ++++++++++++------ datafusion/core/Cargo.toml | 10 +- .../core/src/catalog/dynamic_file_catalog.rs | 50 ++- .../core/src/datasource/file_format/mod.rs | 1 + .../datasource/file_format/object_storage.rs | 5 +- 6 files changed, 315 insertions(+), 172 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5628acc6515e..f954f6c8e818 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ indexmap = "2.0.0" itertools = "0.12" log = "^0.4" num_cpus = "1.13.0" -object_store = { version = "0.9.1", features = ["aws", "gcp", "http"] } +object_store = { version = "0.9.1", dafault-features = false } parking_lot = "0.12" parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] } rand = "0.8" diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index e05d0774c6f6..b8d9b982bd98 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -418,162 +418,153 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "0.101.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f9625b71b3ee4adbfbca369c6680d156e316ed86d2c7199a2a134563917414" +checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" dependencies = [ "aws-credential-types", "aws-http", - "aws-runtime", "aws-sdk-sso", - "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", + "aws-smithy-client", "aws-smithy-http", + "aws-smithy-http-tower", "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 1.9.0", "hex", "http", "hyper", - "ring", + "ring 0.16.20", "time", "tokio", + "tower", "tracing", "zeroize", ] [[package]] name = "aws-credential-types" -version = "0.101.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5924466398ac76ffd411d297b9d516dcebb0577f7344c0c15fd8e8e04d9c7895" +checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" dependencies = [ "aws-smithy-async", - "aws-smithy-runtime-api", "aws-smithy-types", + "fastrand 1.9.0", + "tokio", + "tracing", "zeroize", ] [[package]] -name = "aws-http" -version = "0.59.0" +name = "aws-endpoint" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb9a3aa335a105a00975c971f1dad403c3175f2a210d98f39345c6af53923912" +checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" dependencies = [ - "aws-smithy-runtime-api", + "aws-smithy-http", "aws-smithy-types", "aws-types", - "bytes", "http", - "http-body", - "pin-project-lite", + "regex", "tracing", ] [[package]] -name = "aws-runtime" -version = "0.101.0" +name = "aws-http" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75844ecbdf3dc5e0f5ac5fd1088fb1623849990ea9445d2826258ce63be4de5" +checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" dependencies = [ "aws-credential-types", - "aws-http", - "aws-sigv4", - "aws-smithy-async", "aws-smithy-http", - "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "fastrand", + "bytes", "http", + "http-body", + "lazy_static", "percent-encoding", + "pin-project-lite", "tracing", - "uuid", ] [[package]] name = "aws-sdk-sso" -version = "0.38.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c870aa95e1e85f837f74af2cc937b3f8e72e2315a89e524265875843655b4d47" +checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" dependencies = [ "aws-credential-types", + "aws-endpoint", "aws-http", - "aws-runtime", + "aws-sig-auth", "aws-smithy-async", + "aws-smithy-client", "aws-smithy-http", + "aws-smithy-http-tower", "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "http", "regex", + "tokio-stream", + "tower", "tracing", ] [[package]] -name = "aws-sdk-ssooidc" -version = "0.38.0" +name = "aws-sdk-sts" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "107ee812e46f9120e68d48bf985d2f2a538315bd8be8a3e54db619250cc4c95e" +checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" dependencies = [ "aws-credential-types", + "aws-endpoint", "aws-http", - "aws-runtime", + "aws-sig-auth", "aws-smithy-async", + "aws-smithy-client", "aws-smithy-http", + "aws-smithy-http-tower", "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", + "aws-smithy-query", "aws-smithy-types", + "aws-smithy-xml", "aws-types", "bytes", "http", "regex", + "tower", "tracing", ] [[package]] -name = "aws-sdk-sts" -version = "0.38.0" +name = "aws-sig-auth" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e3958c43a78f6c3822e62009a35802af5cc7c120fbe8e60b98565604569aae" +checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" dependencies = [ "aws-credential-types", - "aws-http", - "aws-runtime", - "aws-smithy-async", + "aws-sigv4", "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", "aws-types", "http", - "regex", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.101.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06130e3686db3c5ae2fc44b3516fffe6b4d4eccebe09bd8ccc4067f3c9c183fb" +checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" dependencies = [ - "aws-credential-types", "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", "form_urlencoded", "hex", "hmac", @@ -588,133 +579,127 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.101.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d787b7e07925b450bed90d9d29ac8e57006c9c2ac907151d175ac0e376bfee0e" +checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" dependencies = [ "futures-util", "pin-project-lite", "tokio", + "tokio-stream", ] [[package]] -name = "aws-smithy-http" -version = "0.59.0" +name = "aws-smithy-client" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96daaad925331c72449423574fdc72b54af780d5a23ace3c0a6ad0ccbf378715" +checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" dependencies = [ - "aws-smithy-runtime-api", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", "aws-smithy-types", "bytes", - "bytes-utils", - "futures-core", + "fastrand 1.9.0", "http", "http-body", - "once_cell", - "percent-encoding", + "hyper", + "hyper-rustls 0.23.2", + "lazy_static", "pin-project-lite", - "pin-utils", + "rustls 0.20.9", + "tokio", + "tower", "tracing", ] [[package]] -name = "aws-smithy-json" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff985bee3fe21046dc501fadc1d04a1161977c55a0cbbccd9b111c18206aa64" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-query" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4006503693766d34717efc5f58325062845fce26a683a71b70f23156d72e67" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "0.101.0" +name = "aws-smithy-http" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d28af854558601b4202a4273b9720aebe43d73e472143e6056f16e3bd90bc837" +checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand", + "bytes-utils", + "futures-core", "http", "http-body", "hyper", - "hyper-rustls", "once_cell", + "percent-encoding", "pin-project-lite", "pin-utils", - "rustls", - "tokio", "tracing", ] [[package]] -name = "aws-smithy-runtime-api" -version = "0.101.0" +name = "aws-smithy-http-tower" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c68e17e754b86da350b43add38294189121a880e9c3fb454f83ff7044f5257" +checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" dependencies = [ - "aws-smithy-async", + "aws-smithy-http", "aws-smithy-types", "bytes", "http", + "http-body", "pin-project-lite", - "tokio", + "tower", "tracing", - "zeroize", +] + +[[package]] +name = "aws-smithy-json" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +dependencies = [ + "aws-smithy-types", + "urlencoding", ] [[package]] name = "aws-smithy-types" -version = "0.101.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97b978d8a351ea5744206ecc643a1d3806628680e9f151b4d6b7a76fec1596f" +checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" dependencies = [ "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http", - "http-body", "itoa", "num-integer", - "pin-project-lite", - "pin-utils", "ryu", - "serde", "time", ] [[package]] name = "aws-smithy-xml" -version = "0.59.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97500a0d0884b9576e65076075f81d899cfbb84f7db5ca1dd317f0582204e528" +checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.101.0" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61065f0c6070cb0f9aaddfa614605fb1049908481da71ba5b39b2ffca12f57e4" +checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-runtime-api", + "aws-smithy-client", + "aws-smithy-http", "aws-smithy-types", "http", "rustc_version", @@ -957,7 +942,7 @@ version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", @@ -991,7 +976,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ "strum 0.26.2", - "strum_macros 0.26.2", + "strum_macros 0.26.3", "unicode-width", ] @@ -1271,7 +1256,7 @@ dependencies = [ "serde_json", "sqlparser", "strum 0.26.2", - "strum_macros 0.26.2", + "strum_macros 0.26.3", ] [[package]] @@ -1568,6 +1553,15 @@ dependencies = [ "str-buf", ] +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -1815,6 +1809,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1909,6 +1909,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "log", + "rustls 0.20.9", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.23.4", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1918,11 +1933,9 @@ dependencies = [ "futures-util", "http", "hyper", - "log", - "rustls", - "rustls-native-certs", + "rustls 0.21.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] @@ -2399,7 +2412,7 @@ dependencies = [ "quick-xml", "rand", "reqwest", - "ring", + "ring 0.17.8", "rustls-pemfile 2.1.2", "serde", "serde_json", @@ -2570,6 +2583,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2656,9 +2689,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -2798,7 +2831,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -2806,7 +2839,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.12", "rustls-native-certs", "rustls-pemfile 1.0.4", "serde", @@ -2815,7 +2848,7 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tokio-util", "tower-service", "url", @@ -2826,6 +2859,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + [[package]] name = "ring" version = "0.17.8" @@ -2836,8 +2884,8 @@ dependencies = [ "cfg-if", "getrandom", "libc", - "spin", - "untrusted", + "spin 0.9.8", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -2901,6 +2949,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring 0.16.20", + "sct", + "webpki", +] + [[package]] name = "rustls" version = "0.21.12" @@ -2908,7 +2968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -2956,8 +3016,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -3025,8 +3085,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -3164,7 +3224,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -3186,6 +3246,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -3243,7 +3309,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ - "strum_macros 0.26.2", + "strum_macros 0.26.3", ] [[package]] @@ -3252,7 +3318,7 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", @@ -3261,11 +3327,11 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +checksum = "f7993a8e3a9e88a00351486baae9522c91b123a088f76469e5bd5cc17198ea87" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -3334,7 +3400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.1.0", "rustix", "windows-sys 0.52.0", ] @@ -3475,13 +3541,35 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls 0.20.9", + "tokio", + "webpki", +] + [[package]] name = "tokio-rustls" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", "tokio", ] @@ -3498,6 +3586,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -3510,6 +3620,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3610,6 +3721,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -3784,6 +3901,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 5bb4187779ca..c9d46aaf2813 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -92,8 +92,8 @@ async-compression = { version = "0.4.0", features = [ "tokio", ], optional = true } async-trait = { workspace = true } -aws-config = "0.101.0" -aws-credential-types = "0.101.0" +aws-config = "0.55" +aws-credential-types = "0.55" bytes = { workspace = true } bzip2 = { version = "0.4.3", optional = true } chrono = { workspace = true } @@ -161,6 +161,12 @@ tokio-postgres = "0.7.7" [target.'cfg(not(target_os = "windows"))'.dev-dependencies] nix = { version = "0.29.0", features = ["fs"] } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +object_store = { version = "0.9.1", features = ["aws", "gcp", "http"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +object_store = { version = "0.9.1", default-features = false } + [[bench]] harness = false name = "aggregate_query_sql" diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index 50d28221569e..6feee6d35871 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -20,6 +20,7 @@ use crate::catalog::schema::SchemaProvider; use crate::catalog::{CatalogProvider, CatalogProviderList}; +#[cfg(not(target_arch = "wasm32"))] use crate::datasource::file_format::object_storage::{ get_object_store, AwsOptions, GcpOptions, }; @@ -33,6 +34,7 @@ use dirs::home_dir; use parking_lot::RwLock; use std::any::Any; use std::sync::{Arc, Weak}; +use url::Url; /// Wraps another catalog, automatically creating table providers /// for local files if needed @@ -157,7 +159,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { // if the inner schema provider didn't have a table by // that name, try to treat it as a listing table - let mut state = self + let state = self .state .upgrade() .ok_or_else(|| plan_datafusion_err!("locking error"))? @@ -175,25 +177,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { match state.runtime_env().object_store_registry.get_store(url) { Ok(_) => { /*Nothing to do here, store for this URL is already registered*/ } Err(_) => { - // Register the store for this URL. Here we don't have access - // to any command options so the only choice is to use an empty collection - match scheme { - "s3" | "oss" | "cos" => { - state = state.add_table_options_extension(AwsOptions::default()); - } - "gs" | "gcs" => { - state = state.add_table_options_extension(GcpOptions::default()) - } - _ => {} - }; - let store = get_object_store( - &state, - table_url.scheme(), - url, - &state.default_table_options(), - ) - .await?; - state.runtime_env().register_object_store(url, store); + let _ = Box::pin(handle_table_error(scheme, state.clone(), url)).await; } } @@ -216,6 +200,32 @@ impl SchemaProvider for DynamicFileSchemaProvider { self.inner.table_exist(name) } } + +#[cfg(not(target_arch = "wasm32"))] +async fn handle_table_error( + scheme: &str, + mut state: SessionState, + url: &Url, +) -> Result<()> { + // Register the store for this URL. Here we don't have access + // to any command options so the only choice is to use an empty collection + match scheme { + "s3" | "oss" | "cos" => { + state = state.add_table_options_extension(AwsOptions::default()); + } + "gs" | "gcs" => state = state.add_table_options_extension(GcpOptions::default()), + _ => {} + }; + let store = + get_object_store(&state, scheme, url, &state.default_table_options()).await?; + state.runtime_env().register_object_store(url, store); + Ok(()) +} + +#[cfg(target_arch = "wasm32")] +async fn handle_table_error(_: &str, _: SessionState, _: &Url) -> Result<()> { + unreachable!("Object storage is not supported in WASM") +} fn substitute_tilde(cur: String) -> String { if let Some(usr_dir_path) = home_dir() { if let Some(usr_dir) = usr_dir_path.to_str() { diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 9608cf8ea591..e62c9d6cc4d0 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -26,6 +26,7 @@ pub mod avro; pub mod csv; pub mod file_compression_type; pub mod json; +#[cfg(not(target_arch = "wasm32"))] pub mod object_storage; pub mod options; #[cfg(feature = "parquet")] diff --git a/datafusion/core/src/datasource/file_format/object_storage.rs b/datafusion/core/src/datasource/file_format/object_storage.rs index 8633a779e142..600f923acefa 100644 --- a/datafusion/core/src/datasource/file_format/object_storage.rs +++ b/datafusion/core/src/datasource/file_format/object_storage.rs @@ -30,7 +30,6 @@ use crate::execution::context::SessionState; use crate::prelude::SessionContext; use async_trait::async_trait; -use aws_config::BehaviorVersion; use aws_credential_types::provider::ProvideCredentials; use object_store::aws::{AmazonS3Builder, AwsCredential}; use object_store::gcp::GoogleCloudStorageBuilder; @@ -65,7 +64,7 @@ pub(crate) async fn get_s3_object_store_builder( builder = builder.with_token(session_token); } } else { - let config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let config = aws_config::from_env().load().await; if let Some(region) = config.region() { builder = builder.with_region(region.to_string()); } @@ -592,7 +591,7 @@ mod tests { .unwrap_err(); // There are other backstraces in the error message, so we just check for containing the message - assert_eq!(err.to_string().contains("Invalid or Unsupported Configuration: Invalid endpoint: http://endpoint33. HTTP is not allowed for S3 endpoints. To allow HTTP, set 'aws.allow_http' to true"), true); + assert!(err.to_string().contains("Invalid or Unsupported Configuration: Invalid endpoint: http://endpoint33. HTTP is not allowed for S3 endpoints. To allow HTTP, set 'aws.allow_http' to true")); } else { return plan_err!("LogicalPlan is not a CreateExternalTable"); } From 16b2c9ee286fecbbd8c4eb7613c478935cd1bc3b Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 19:20:01 +0800 Subject: [PATCH 13/21] fix audit --- datafusion/core/Cargo.toml | 4 ++-- datafusion/core/src/datasource/file_format/object_storage.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index c9d46aaf2813..c2135c465728 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -92,8 +92,8 @@ async-compression = { version = "0.4.0", features = [ "tokio", ], optional = true } async-trait = { workspace = true } -aws-config = "0.55" -aws-credential-types = "0.55" +aws-config = "0.101.0" +aws-credential-types = "0.101.0" bytes = { workspace = true } bzip2 = { version = "0.4.3", optional = true } chrono = { workspace = true } diff --git a/datafusion/core/src/datasource/file_format/object_storage.rs b/datafusion/core/src/datasource/file_format/object_storage.rs index 600f923acefa..6d0e12c21c9e 100644 --- a/datafusion/core/src/datasource/file_format/object_storage.rs +++ b/datafusion/core/src/datasource/file_format/object_storage.rs @@ -30,6 +30,7 @@ use crate::execution::context::SessionState; use crate::prelude::SessionContext; use async_trait::async_trait; +use aws_config::BehaviorVersion; use aws_credential_types::provider::ProvideCredentials; use object_store::aws::{AmazonS3Builder, AwsCredential}; use object_store::gcp::GoogleCloudStorageBuilder; @@ -64,7 +65,7 @@ pub(crate) async fn get_s3_object_store_builder( builder = builder.with_token(session_token); } } else { - let config = aws_config::from_env().load().await; + let config = aws_config::load_defaults(BehaviorVersion::latest()).await; if let Some(region) = config.region() { builder = builder.with_region(region.to_string()); } From d67f8fe6b130e75cb7af8b9963b9a0a329fcc9ff Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 19:25:31 +0800 Subject: [PATCH 14/21] fix wasm-build --- datafusion/core/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index c2135c465728..9677b2accbd4 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -92,8 +92,6 @@ async-compression = { version = "0.4.0", features = [ "tokio", ], optional = true } async-trait = { workspace = true } -aws-config = "0.101.0" -aws-credential-types = "0.101.0" bytes = { workspace = true } bzip2 = { version = "0.4.3", optional = true } chrono = { workspace = true } @@ -162,6 +160,8 @@ tokio-postgres = "0.7.7" nix = { version = "0.29.0", features = ["fs"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +aws-config = "0.101.0" +aws-credential-types = "0.101.0" object_store = { version = "0.9.1", features = ["aws", "gcp", "http"] } [target.'cfg(target_arch = "wasm32")'.dependencies] From 256adbd40be03c594e93123e969ac49c11b41696 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 19:30:28 +0800 Subject: [PATCH 15/21] update cargo lock --- datafusion-cli/Cargo.lock | 391 +++++++++++++------------------------- 1 file changed, 135 insertions(+), 256 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index b8d9b982bd98..c50f86164ffe 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -418,153 +418,162 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" +checksum = "84f9625b71b3ee4adbfbca369c6680d156e316ed86d2c7199a2a134563917414" dependencies = [ "aws-credential-types", "aws-http", + "aws-runtime", "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand 1.9.0", + "fastrand", "hex", "http", "hyper", - "ring 0.16.20", + "ring", "time", "tokio", - "tower", "tracing", "zeroize", ] [[package]] name = "aws-credential-types" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" +checksum = "5924466398ac76ffd411d297b9d516dcebb0577f7344c0c15fd8e8e04d9c7895" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand 1.9.0", - "tokio", - "tracing", "zeroize", ] [[package]] -name = "aws-endpoint" -version = "0.55.3" +name = "aws-http" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" +checksum = "bb9a3aa335a105a00975c971f1dad403c3175f2a210d98f39345c6af53923912" dependencies = [ - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes", "http", - "regex", + "http-body", + "pin-project-lite", "tracing", ] [[package]] -name = "aws-http" -version = "0.55.3" +name = "aws-runtime" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" +checksum = "b75844ecbdf3dc5e0f5ac5fd1088fb1623849990ea9445d2826258ce63be4de5" dependencies = [ "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes", + "fastrand", "http", - "http-body", - "lazy_static", "percent-encoding", - "pin-project-lite", "tracing", + "uuid", ] [[package]] name = "aws-sdk-sso" -version = "0.28.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" +checksum = "c870aa95e1e85f837f74af2cc937b3f8e72e2315a89e524265875843655b4d47" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "http", "regex", - "tokio-stream", - "tower", "tracing", ] [[package]] -name = "aws-sdk-sts" -version = "0.28.0" +name = "aws-sdk-ssooidc" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" +checksum = "107ee812e46f9120e68d48bf985d2f2a538315bd8be8a3e54db619250cc4c95e" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", - "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", - "aws-smithy-xml", "aws-types", "bytes", "http", "regex", - "tower", "tracing", ] [[package]] -name = "aws-sig-auth" -version = "0.55.3" +name = "aws-sdk-sts" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" +checksum = "a4e3958c43a78f6c3822e62009a35802af5cc7c120fbe8e60b98565604569aae" dependencies = [ "aws-credential-types", - "aws-sigv4", + "aws-http", + "aws-runtime", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", "aws-types", "http", + "regex", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +checksum = "06130e3686db3c5ae2fc44b3516fffe6b4d4eccebe09bd8ccc4067f3c9c183fb" dependencies = [ + "aws-credential-types", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", "form_urlencoded", "hex", "hmac", @@ -579,53 +588,28 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" +checksum = "d787b7e07925b450bed90d9d29ac8e57006c9c2ac907151d175ac0e376bfee0e" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-client" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 1.9.0", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", - "lazy_static", - "pin-project-lite", - "rustls 0.20.9", - "tokio", - "tower", - "tracing", ] [[package]] name = "aws-smithy-http" -version = "0.55.3" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +checksum = "96daaad925331c72449423574fdc72b54af780d5a23ace3c0a6ad0ccbf378715" dependencies = [ + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http", "http-body", - "hyper", "once_cell", "percent-encoding", "pin-project-lite", @@ -634,72 +618,103 @@ dependencies = [ ] [[package]] -name = "aws-smithy-http-tower" -version = "0.55.3" +name = "aws-smithy-json" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" +checksum = "0ff985bee3fe21046dc501fadc1d04a1161977c55a0cbbccd9b111c18206aa64" dependencies = [ - "aws-smithy-http", "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", ] [[package]] -name = "aws-smithy-json" -version = "0.55.3" +name = "aws-smithy-query" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +checksum = "cb4006503693766d34717efc5f58325062845fce26a683a71b70f23156d72e67" dependencies = [ "aws-smithy-types", + "urlencoding", ] [[package]] -name = "aws-smithy-query" -version = "0.55.3" +name = "aws-smithy-runtime" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +checksum = "d28af854558601b4202a4273b9720aebe43d73e472143e6056f16e3bd90bc837" dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "urlencoding", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c68e17e754b86da350b43add38294189121a880e9c3fb454f83ff7044f5257" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +checksum = "d97b978d8a351ea5744206ecc643a1d3806628680e9f151b4d6b7a76fec1596f" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", + "serde", "time", ] [[package]] name = "aws-smithy-xml" -version = "0.55.3" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" +checksum = "97500a0d0884b9576e65076075f81d899cfbb84f7db5ca1dd317f0582204e528" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.3" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" +checksum = "61065f0c6070cb0f9aaddfa614605fb1049908481da71ba5b39b2ffca12f57e4" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "http", "rustc_version", @@ -1553,15 +1568,6 @@ dependencies = [ "str-buf", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.1.0" @@ -1909,21 +1915,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.9", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1933,9 +1924,11 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.12", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", ] [[package]] @@ -2412,7 +2405,7 @@ dependencies = [ "quick-xml", "rand", "reqwest", - "ring 0.17.8", + "ring", "rustls-pemfile 2.1.2", "serde", "serde_json", @@ -2583,26 +2576,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2831,7 +2804,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.2", + "hyper-rustls", "ipnet", "js-sys", "log", @@ -2839,7 +2812,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.12", + "rustls", "rustls-native-certs", "rustls-pemfile 1.0.4", "serde", @@ -2848,7 +2821,7 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -2859,21 +2832,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.8" @@ -2884,8 +2842,8 @@ dependencies = [ "cfg-if", "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "spin", + "untrusted", "windows-sys 0.52.0", ] @@ -2949,18 +2907,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" -dependencies = [ - "log", - "ring 0.16.20", - "sct", - "webpki", -] - [[package]] name = "rustls" version = "0.21.12" @@ -2968,7 +2914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.8", + "ring", "rustls-webpki", "sct", ] @@ -3016,8 +2962,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3085,8 +3031,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3246,12 +3192,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -3400,7 +3340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand", "rustix", "windows-sys 0.52.0", ] @@ -3541,35 +3481,13 @@ dependencies = [ "syn 2.0.66", ] -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.9", - "tokio", - "webpki", -] - [[package]] name = "tokio-rustls" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.12", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" -dependencies = [ - "futures-core", - "pin-project-lite", + "rustls", "tokio", ] @@ -3586,28 +3504,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -3620,7 +3516,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3721,12 +3616,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -3901,16 +3790,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - [[package]] name = "winapi" version = "0.3.9" From c32ccd07b86481110aa7b15d75a096c0458fe102 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 20:42:15 +0800 Subject: [PATCH 16/21] fix test --- .../core/src/catalog/dynamic_file_catalog.rs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index 6feee6d35871..39f5c6dc90a2 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -34,7 +34,6 @@ use dirs::home_dir; use parking_lot::RwLock; use std::any::Any; use std::sync::{Arc, Weak}; -use url::Url; /// Wraps another catalog, automatically creating table providers /// for local files if needed @@ -159,7 +158,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { // if the inner schema provider didn't have a table by // that name, try to treat it as a listing table - let state = self + let mut state = self .state .upgrade() .ok_or_else(|| plan_datafusion_err!("locking error"))? @@ -177,7 +176,17 @@ impl SchemaProvider for DynamicFileSchemaProvider { match state.runtime_env().object_store_registry.get_store(url) { Ok(_) => { /*Nothing to do here, store for this URL is already registered*/ } Err(_) => { - let _ = Box::pin(handle_table_error(scheme, state.clone(), url)).await; + // Register the store for this URL. Here we don't have access + // to any command options so the only choice is to use an empty collection + state = add_extension_option(state, scheme); + let store = get_object_store( + &state, + table_url.scheme(), + url, + &state.default_table_options(), + ) + .await?; + state.runtime_env().register_object_store(url, store); } } @@ -202,13 +211,7 @@ impl SchemaProvider for DynamicFileSchemaProvider { } #[cfg(not(target_arch = "wasm32"))] -async fn handle_table_error( - scheme: &str, - mut state: SessionState, - url: &Url, -) -> Result<()> { - // Register the store for this URL. Here we don't have access - // to any command options so the only choice is to use an empty collection +fn add_extension_option(mut state: SessionState, scheme: &str) -> SessionState { match scheme { "s3" | "oss" | "cos" => { state = state.add_table_options_extension(AwsOptions::default()); @@ -216,16 +219,14 @@ async fn handle_table_error( "gs" | "gcs" => state = state.add_table_options_extension(GcpOptions::default()), _ => {} }; - let store = - get_object_store(&state, scheme, url, &state.default_table_options()).await?; - state.runtime_env().register_object_store(url, store); - Ok(()) + state } #[cfg(target_arch = "wasm32")] -async fn handle_table_error(_: &str, _: SessionState, _: &Url) -> Result<()> { - unreachable!("Object storage is not supported in WASM") +fn add_extension_option(state: SessionState, _: &str) -> SessionState { + state } + fn substitute_tilde(cur: String) -> String { if let Some(usr_dir_path) = home_dir() { if let Some(usr_dir) = usr_dir_path.to_str() { From 46543972f0b1b9df4e5224e2f12a3084b9a411a1 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 21:06:22 +0800 Subject: [PATCH 17/21] fix wasm build --- .../core/src/catalog/dynamic_file_catalog.rs | 5 ++-- .../core/src/datasource/file_format/mod.rs | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/datafusion/core/src/catalog/dynamic_file_catalog.rs b/datafusion/core/src/catalog/dynamic_file_catalog.rs index 39f5c6dc90a2..3b49490f5d68 100644 --- a/datafusion/core/src/catalog/dynamic_file_catalog.rs +++ b/datafusion/core/src/catalog/dynamic_file_catalog.rs @@ -20,10 +20,9 @@ use crate::catalog::schema::SchemaProvider; use crate::catalog::{CatalogProvider, CatalogProviderList}; +use crate::datasource::file_format::get_object_store; #[cfg(not(target_arch = "wasm32"))] -use crate::datasource::file_format::object_storage::{ - get_object_store, AwsOptions, GcpOptions, -}; +use crate::datasource::file_format::object_storage::{AwsOptions, GcpOptions}; use crate::datasource::listing::{ListingTable, ListingTableConfig, ListingTableUrl}; use crate::datasource::TableProvider; use crate::execution::context::SessionState; diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index e62c9d6cc4d0..42b3be407f36 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -43,11 +43,13 @@ use crate::error::Result; use crate::execution::context::SessionState; use crate::physical_plan::{ExecutionPlan, Statistics}; -use datafusion_common::not_impl_err; +use datafusion_common::{not_impl_err, DataFusionError}; use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement}; use async_trait::async_trait; +use datafusion_common::config::TableOptions; use object_store::{ObjectMeta, ObjectStore}; +use url::Url; /// This trait abstracts all the file format specific implementations /// from the [`TableProvider`]. This helps code re-utilization across @@ -108,6 +110,28 @@ pub trait FileFormat: Send + Sync + fmt::Debug { } } +/// Get the object store for the given scheme and url. Only available when not targeting wasm32. +#[cfg(not(target_arch = "wasm32"))] +pub async fn get_object_store( + state: &SessionState, + scheme: &str, + url: &Url, + table_options: &TableOptions, +) -> Result, DataFusionError> { + object_storage::get_object_store(state, scheme, url, table_options).await +} + +/// Get the object store for the given scheme and url. Only available when targeting wasm32. +#[cfg(target_arch = "wasm32")] +pub async fn get_object_store( + state: &SessionState, + scheme: &str, + url: &Url, + table_options: &TableOptions, +) -> Result, DataFusionError> { + unimplemented!("Object storage is not supported in WASM") +} + #[cfg(test)] pub(crate) mod test_util { use std::ops::Range; From d1a9368c2981b7f4b0171ba06a46dfb44096ff13 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Sun, 2 Jun 2024 23:23:26 +0800 Subject: [PATCH 18/21] fix typo --- datafusion/core/src/datasource/file_format/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 42b3be407f36..a81043bab43b 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -121,7 +121,7 @@ pub async fn get_object_store( object_storage::get_object_store(state, scheme, url, table_options).await } -/// Get the object store for the given scheme and url. Only available when targeting wasm32. +/// Get the object store for the given scheme and url. Only available when not targeting wasm32. #[cfg(target_arch = "wasm32")] pub async fn get_object_store( state: &SessionState, From efa9bac18981019a0b9dd103963601bab575b071 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Mon, 3 Jun 2024 00:00:17 +0800 Subject: [PATCH 19/21] fix typo --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f954f6c8e818..45504be3f1ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ indexmap = "2.0.0" itertools = "0.12" log = "^0.4" num_cpus = "1.13.0" -object_store = { version = "0.9.1", dafault-features = false } +object_store = { version = "0.9.1", default-features = false } parking_lot = "0.12" parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] } rand = "0.8" From 6146d087f101386086f049d3d58dbb1808855fb6 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Mon, 3 Jun 2024 23:37:52 +0800 Subject: [PATCH 20/21] try to disable default-feature --- datafusion/core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 9677b2accbd4..bccadba60ee7 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -162,7 +162,7 @@ nix = { version = "0.29.0", features = ["fs"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] aws-config = "0.101.0" aws-credential-types = "0.101.0" -object_store = { version = "0.9.1", features = ["aws", "gcp", "http"] } +object_store = { version = "0.9.1", default-features = false , features = ["aws", "gcp", "http"] } [target.'cfg(target_arch = "wasm32")'.dependencies] object_store = { version = "0.9.1", default-features = false } From 679935b0ac1f6bef4507abd5009601260716c548 Mon Sep 17 00:00:00 2001 From: Jia-Xuan Liu Date: Mon, 3 Jun 2024 23:56:50 +0800 Subject: [PATCH 21/21] format toml file and add comments --- datafusion/core/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index bccadba60ee7..822307e7b581 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -162,9 +162,11 @@ nix = { version = "0.29.0", features = ["fs"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] aws-config = "0.101.0" aws-credential-types = "0.101.0" -object_store = { version = "0.9.1", default-features = false , features = ["aws", "gcp", "http"] } +# the default features will cause libunwind error when testing on macos paltform. +object_store = { version = "0.9.1", default-features = false, features = ["aws", "gcp", "http"] } [target.'cfg(target_arch = "wasm32")'.dependencies] +# the default features will cause libunwind error when testing on macos paltform. object_store = { version = "0.9.1", default-features = false } [[bench]]