Skip to content

Commit

Permalink
merge with apache main
Browse files Browse the repository at this point in the history
add pushes_global_limit_into_multiple_fetch_plans test case
change limit_pushdown.rs as manual top down operator and simplify algorithm by supporting most parent node remove and other pushdown cases
  • Loading branch information
mertak-synnada committed Aug 14, 2024
1 parent ebda00a commit 27342ff
Show file tree
Hide file tree
Showing 20 changed files with 464 additions and 1,013 deletions.
2 changes: 1 addition & 1 deletion datafusion/common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

mod column;
mod dfschema;
mod error;
mod functional_dependencies;
mod join_type;
mod param_value;
Expand All @@ -33,6 +32,7 @@ pub mod alias;
pub mod cast;
pub mod config;
pub mod display;
pub mod error;
pub mod file_options;
pub mod format;
pub mod hash_utils;
Expand Down
5 changes: 2 additions & 3 deletions datafusion/core/src/physical_optimizer/enforce_sorting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,12 @@ use crate::physical_plan::{Distribution, ExecutionPlan, InputOrderMode};
use datafusion_common::plan_err;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_physical_expr::{Partitioning, PhysicalSortExpr, PhysicalSortRequirement};
use datafusion_physical_plan::limit::LocalLimitExec;
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::partial_sort::PartialSortExec;
use datafusion_physical_plan::ExecutionPlanProperties;

use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use itertools::izip;

/// This rule inspects [`SortExec`]'s in the given physical plan and removes the
Expand Down Expand Up @@ -1136,7 +1135,7 @@ mod tests {
" MemoryExec: partitions=1, partition_sizes=[0]",
];
let expected_optimized = [
"LocalLimitExec: fetch=2",
"GlobalLimitExec: skip=0, fetch=2",
" SortExec: expr=[non_nullable_col@1 ASC,nullable_col@0 ASC], preserve_partitioning=[false]",
" MemoryExec: partitions=1, partition_sizes=[0]",
];
Expand Down
Loading

0 comments on commit 27342ff

Please sign in to comment.