From 1cda23aaabf3acce783445d243df64dc4f0ece90 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Thu, 5 Dec 2024 00:33:44 -0800 Subject: [PATCH] Retract IndexSet, IndexMap type alias (#13655) * Retract IndexSet, IndexMap type alias * Fix linter errors --- datafusion-cli/Cargo.lock | 4 ++++ datafusion/common/src/cse.rs | 2 +- datafusion/common/src/lib.rs | 4 ---- datafusion/expr/Cargo.toml | 1 + datafusion/expr/src/logical_plan/builder.rs | 4 +++- datafusion/expr/src/logical_plan/plan.rs | 3 ++- datafusion/expr/src/utils.rs | 3 ++- datafusion/optimizer/src/eliminate_duplicated_expr.rs | 5 ++++- datafusion/optimizer/src/push_down_filter.rs | 3 ++- .../optimizer/src/simplify_expressions/expr_simplifier.rs | 5 ++--- datafusion/physical-expr/Cargo.toml | 1 + datafusion/physical-expr/src/equivalence/class.rs | 4 +++- datafusion/physical-expr/src/equivalence/properties.rs | 5 ++--- datafusion/physical-expr/src/window/window_expr.rs | 4 +++- datafusion/physical-plan/Cargo.toml | 1 + .../physical-plan/src/windows/bounded_window_agg_exec.rs | 4 +++- datafusion/sql/Cargo.toml | 1 + datafusion/sql/src/select.rs | 4 +++- datafusion/sql/src/unparser/utils.rs | 4 +++- datafusion/sql/src/utils.rs | 7 +++++-- 20 files changed, 46 insertions(+), 23 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 6c0f1e993a9d9..1fa99185a5a08 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -1361,6 +1361,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", + "indexmap", "paste", "recursive", "serde_json", @@ -1545,6 +1546,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", + "indexmap", "itertools", "log", "paste", @@ -1601,6 +1603,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", + "indexmap", "itertools", "log", "once_cell", @@ -1619,6 +1622,7 @@ dependencies = [ "arrow-schema", "datafusion-common", "datafusion-expr", + "indexmap", "log", "recursive", "regex", diff --git a/datafusion/common/src/cse.rs b/datafusion/common/src/cse.rs index bb9abd5399081..ab02915858cd2 100644 --- a/datafusion/common/src/cse.rs +++ b/datafusion/common/src/cse.rs @@ -24,8 +24,8 @@ use crate::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter, TreeNodeVisitor, }; -use crate::IndexMap; use crate::Result; +use indexmap::IndexMap; use std::collections::HashMap; use std::hash::{BuildHasher, Hash, Hasher, RandomState}; use std::marker::PhantomData; diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index ad3b6ab0d1715..77e8cd60ede24 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -73,7 +73,6 @@ pub use param_value::ParamValues; pub use scalar::{ScalarType, ScalarValue}; pub use schema_reference::SchemaReference; pub use stats::{ColumnStatistics, Statistics}; -use std::hash::RandomState; pub use table_reference::{ResolvedTableReference, TableReference}; pub use unnest::{RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; @@ -94,9 +93,6 @@ pub use error::{ pub type HashMap = hashbrown::HashMap; pub type HashSet = hashbrown::HashSet; -pub type IndexMap = indexmap::IndexMap; -pub type IndexSet = indexmap::IndexSet; - /// Downcast an Arrow Array to a concrete type, return an `DataFusionError::Internal` if the cast is /// not possible. In normal usage of DataFusion the downcast should always succeed. /// diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 772a57ac93d3f..438662e0642b0 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -49,6 +49,7 @@ datafusion-expr-common = { workspace = true } datafusion-functions-aggregate-common = { workspace = true } datafusion-functions-window-common = { workspace = true } datafusion-physical-expr-common = { workspace = true } +indexmap = { workspace = true } paste = "^1.0" recursive = { workspace = true } serde_json = { workspace = true } diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index a44219be4385d..e15c0a36a0e94 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -54,11 +54,13 @@ use datafusion_common::file_options::file_type::FileType; use datafusion_common::{ exec_err, get_target_functional_dependencies, internal_err, not_impl_err, plan_datafusion_err, plan_err, Column, DFSchema, DFSchemaRef, DataFusionError, - FunctionalDependencies, IndexSet, Result, ScalarValue, TableReference, ToDFSchema, + FunctionalDependencies, Result, ScalarValue, TableReference, ToDFSchema, UnnestOptions, }; use datafusion_expr_common::type_coercion::binary::type_union_resolution; +use indexmap::IndexSet; + /// Default table name for unnamed table pub const UNNAMED_TABLE: &str = "?table?"; diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index a1875012eea7f..fe725e7d96ded 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -51,9 +51,10 @@ use datafusion_common::tree_node::{ use datafusion_common::{ aggregate_functional_dependencies, internal_err, plan_err, Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency, FunctionalDependence, - FunctionalDependencies, IndexSet, ParamValues, Result, ScalarValue, TableReference, + FunctionalDependencies, ParamValues, Result, ScalarValue, TableReference, UnnestOptions, }; +use indexmap::IndexSet; // backwards compatibility use crate::display::PgJsonVisitor; diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index b99b9e7f4c1c3..6f7c5d379260e 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -36,9 +36,10 @@ use datafusion_common::tree_node::{ use datafusion_common::utils::get_at_indices; use datafusion_common::{ internal_err, plan_datafusion_err, plan_err, Column, DFSchema, DFSchemaRef, - DataFusionError, HashMap, IndexSet, Result, TableReference, + DataFusionError, HashMap, Result, TableReference, }; +use indexmap::IndexSet; use sqlparser::ast::{ExceptSelectItem, ExcludeSelectItem}; pub use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity; diff --git a/datafusion/optimizer/src/eliminate_duplicated_expr.rs b/datafusion/optimizer/src/eliminate_duplicated_expr.rs index 54acaa335f5d5..4669500920956 100644 --- a/datafusion/optimizer/src/eliminate_duplicated_expr.rs +++ b/datafusion/optimizer/src/eliminate_duplicated_expr.rs @@ -20,10 +20,13 @@ use crate::optimizer::ApplyOrder; use crate::{OptimizerConfig, OptimizerRule}; use datafusion_common::tree_node::Transformed; -use datafusion_common::{IndexSet, Result}; +use datafusion_common::Result; use datafusion_expr::logical_plan::LogicalPlan; use datafusion_expr::{Aggregate, Expr, Sort, SortExpr}; use std::hash::{Hash, Hasher}; + +use indexmap::IndexSet; + /// Optimization rule that eliminate duplicated expr. #[derive(Default, Debug)] pub struct EliminateDuplicatedExpr; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 955da36cfa014..0b328ad39f558 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -20,13 +20,14 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use indexmap::IndexSet; use itertools::Itertools; use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, }; use datafusion_common::{ - internal_err, plan_err, qualified_name, Column, DFSchema, IndexSet, Result, + internal_err, plan_err, qualified_name, Column, DFSchema, Result, }; use datafusion_expr::expr_rewriter::replace_col; use datafusion_expr::logical_plan::{Join, JoinType, LogicalPlan, TableScan, Union}; diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index bb67dea008a69..e3bcb6da8e533 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -31,9 +31,7 @@ use datafusion_common::{ cast::{as_large_list_array, as_list_array}, tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter}, }; -use datafusion_common::{ - internal_err, DFSchema, DataFusionError, IndexSet, Result, ScalarValue, -}; +use datafusion_common::{internal_err, DFSchema, DataFusionError, Result, ScalarValue}; use datafusion_expr::simplify::ExprSimplifyResult; use datafusion_expr::{ and, lit, or, BinaryExpr, Case, ColumnarValue, Expr, Like, Operator, Volatility, @@ -50,6 +48,7 @@ use crate::analyzer::type_coercion::TypeCoercionRewriter; use crate::simplify_expressions::guarantees::GuaranteeRewriter; use crate::simplify_expressions::regex::simplify_regex_expr; use crate::simplify_expressions::SimplifyInfo; +use indexmap::IndexSet; use regex::Regex; use super::inlist_simplifier::ShortenInListSimplifier; diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index 3d7f57aebe232..522eaea8f67c7 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -48,6 +48,7 @@ datafusion-functions-aggregate-common = { workspace = true } datafusion-physical-expr-common = { workspace = true } half = { workspace = true } hashbrown = { workspace = true } +indexmap = { workspace = true } itertools = { workspace = true, features = ["use_std"] } log = { workspace = true } paste = "^1.0" diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index add6871d1ac44..1812844d98c93 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -24,9 +24,11 @@ use std::fmt::Display; use std::sync::Arc; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; -use datafusion_common::{IndexSet, JoinType}; +use datafusion_common::JoinType; use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; +use indexmap::IndexSet; + /// A structure representing a expression known to be constant in a physical execution plan. /// /// The `ConstExpr` struct encapsulates an expression that is constant during the execution diff --git a/datafusion/physical-expr/src/equivalence/properties.rs b/datafusion/physical-expr/src/equivalence/properties.rs index 6c049196c4d76..fe866450b2b26 100644 --- a/datafusion/physical-expr/src/equivalence/properties.rs +++ b/datafusion/physical-expr/src/equivalence/properties.rs @@ -36,13 +36,12 @@ use crate::{ use arrow_schema::{SchemaRef, SortOptions}; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; -use datafusion_common::{ - internal_err, plan_err, IndexMap, IndexSet, JoinSide, JoinType, Result, -}; +use datafusion_common::{internal_err, plan_err, JoinSide, JoinType, Result}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_physical_expr_common::utils::ExprPropertiesNode; +use indexmap::{IndexMap, IndexSet}; use itertools::Itertools; /// A `EquivalenceProperties` object stores information known about the output diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index 3070eb2b3f3d1..8b130506cdea7 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -28,12 +28,14 @@ use arrow::compute::SortOptions; use arrow::datatypes::Field; use arrow::record_batch::RecordBatch; use datafusion_common::utils::compare_rows; -use datafusion_common::{internal_err, DataFusionError, IndexMap, Result, ScalarValue}; +use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; use datafusion_expr::window_state::{ PartitionBatchState, WindowAggState, WindowFrameContext, WindowFrameStateGroups, }; use datafusion_expr::{Accumulator, PartitionEvaluator, WindowFrame, WindowFrameBound}; +use indexmap::IndexMap; + /// Common trait for [window function] implementations /// /// # Aggregate Window Expressions diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 7dd69e69eb392..bb0e21fdfd158 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -57,6 +57,7 @@ datafusion-physical-expr-common = { workspace = true } futures = { workspace = true } half = { workspace = true } hashbrown = { workspace = true } +indexmap = { workspace = true } itertools = { workspace = true, features = ["use_std"] } log = { workspace = true } once_cell = "1.18.0" diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs index 287ef6d767ef0..07d99c8e7129c 100644 --- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs @@ -52,7 +52,7 @@ use datafusion_common::utils::{ evaluate_partition_ranges, get_at_indices, get_row_at_idx, }; use datafusion_common::{ - arrow_datafusion_err, exec_err, DataFusionError, HashMap, IndexMap, Result, + arrow_datafusion_err, exec_err, DataFusionError, HashMap, Result, }; use datafusion_execution::TaskContext; use datafusion_expr::window_state::{PartitionBatchState, WindowAggState}; @@ -62,9 +62,11 @@ use datafusion_physical_expr::window::{ }; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement}; + use futures::stream::Stream; use futures::{ready, StreamExt}; use hashbrown::raw::RawTable; +use indexmap::IndexMap; use log::debug; /// Window execution plan diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index b3a28fc9d5613..01bf920438162 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -46,6 +46,7 @@ arrow-array = { workspace = true } arrow-schema = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } +indexmap = { workspace = true } log = { workspace = true } recursive = { workspace = true } regex = { workspace = true } diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 002185adafd63..c2a9bac24e660 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -25,7 +25,7 @@ use crate::utils::{ }; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; -use datafusion_common::{not_impl_err, plan_err, IndexMap, Result}; +use datafusion_common::{not_impl_err, plan_err, Result}; use datafusion_common::{RecursionUnnestOption, UnnestOptions}; use datafusion_expr::expr::{Alias, PlannedReplaceSelectItem, WildcardOptions}; use datafusion_expr::expr_rewriter::{ @@ -38,6 +38,8 @@ use datafusion_expr::{ qualified_wildcard_with_options, wildcard_with_options, Aggregate, Expr, Filter, GroupingSet, LogicalPlan, LogicalPlanBuilder, Partitioning, }; + +use indexmap::IndexMap; use sqlparser::ast::{ Distinct, Expr as SQLExpr, GroupByExpr, NamedWindowExpr, OrderByExpr, WildcardAdditionalOptions, WindowType, diff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rs index c5aed2929e504..518781106c3b1 100644 --- a/datafusion/sql/src/unparser/utils.rs +++ b/datafusion/sql/src/unparser/utils.rs @@ -20,12 +20,14 @@ use std::{cmp::Ordering, sync::Arc, vec}; use datafusion_common::{ internal_err, tree_node::{Transformed, TransformedResult, TreeNode}, - Column, DataFusionError, IndexSet, Result, ScalarValue, + Column, DataFusionError, Result, ScalarValue, }; use datafusion_expr::{ expr, utils::grouping_set_to_exprlist, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, Projection, SortExpr, Unnest, Window, }; + +use indexmap::IndexSet; use sqlparser::ast; use super::{ diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index c220ab5c10cc7..69e3953341efa 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -27,7 +27,7 @@ use datafusion_common::tree_node::{ }; use datafusion_common::{ exec_err, internal_err, plan_err, Column, DFSchemaRef, DataFusionError, HashMap, - IndexMap, Result, ScalarValue, + Result, ScalarValue, }; use datafusion_expr::builder::get_struct_unnested_columns; use datafusion_expr::expr::{Alias, GroupingSet, Unnest, WindowFunction}; @@ -35,6 +35,8 @@ use datafusion_expr::utils::{expr_as_column_expr, find_column_exprs}; use datafusion_expr::{ col, expr_vec_fmt, ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, }; + +use indexmap::IndexMap; use sqlparser::ast::{Ident, Value}; /// Make a best-effort attempt at resolving all columns in the expression tree @@ -617,7 +619,7 @@ mod tests { use arrow::datatypes::{DataType as ArrowDataType, Field, Schema}; use arrow_schema::Fields; - use datafusion_common::{Column, DFSchema, IndexMap, Result}; + use datafusion_common::{Column, DFSchema, Result}; use datafusion_expr::{ col, lit, unnest, ColumnUnnestList, EmptyRelation, LogicalPlan, }; @@ -625,6 +627,7 @@ mod tests { use datafusion_functions_aggregate::expr_fn::count; use crate::utils::{resolve_positions_to_exprs, rewrite_recursive_unnest_bottom_up}; + use indexmap::IndexMap; fn column_unnests_eq( l: Vec<&str>,