Skip to content

Commit

Permalink
Remove unused AggregateOptions struct and scalar_update_factor co…
Browse files Browse the repository at this point in the history
…nfig setting (#12241)

* Remove scalar_update_factor from AggregateOptions

Keeping the AggregateOptions struct in case we need it later.

#11998

* Remove AggregateOption; update configs.md

* Run prettier

* Fix cargo doc
  • Loading branch information
jc4x4 authored Sep 2, 2024
1 parent d86b7f9 commit dd32089
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 127 deletions.
24 changes: 0 additions & 24 deletions datafusion/common/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,9 +261,6 @@ config_namespace! {
/// Parquet options
pub parquet: ParquetOptions, default = Default::default()

/// Aggregate options
pub aggregate: AggregateOptions, default = Default::default()

/// Fan-out during initial physical planning.
///
/// This is mostly use to plan `UNION` children in parallel.
Expand Down Expand Up @@ -493,27 +490,6 @@ config_namespace! {
}
}

config_namespace! {
/// Options related to aggregate execution
///
/// See also: [`SessionConfig`]
///
/// [`SessionConfig`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionConfig.html
pub struct AggregateOptions {
/// Specifies the threshold for using `ScalarValue`s to update
/// accumulators during high-cardinality aggregations for each input batch.
///
/// The aggregation is considered high-cardinality if the number of affected groups
/// is greater than or equal to `batch_size / scalar_update_factor`. In such cases,
/// `ScalarValue`s are utilized for updating accumulators, rather than the default
/// batch-slice approach. This can lead to performance improvements.
///
/// By adjusting the `scalar_update_factor`, you can balance the trade-off between
/// more efficient accumulator updates and the number of groups affected.
pub scalar_update_factor: usize, default = 10
}
}

config_namespace! {
/// Options related to query optimization
///
Expand Down
14 changes: 0 additions & 14 deletions datafusion/execution/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ use datafusion_common::{
/// | --------- | ------------- |
/// | `datafusion.catalog` | [CatalogOptions][datafusion_common::config::CatalogOptions] |
/// | `datafusion.execution` | [ExecutionOptions][datafusion_common::config::ExecutionOptions] |
/// | `datafusion.execution.aggregate` | [AggregateOptions][datafusion_common::config::AggregateOptions] |
/// | `datafusion.execution.parquet` | [ParquetOptions][datafusion_common::config::ParquetOptions] |
/// | `datafusion.optimizer` | [OptimizerOptions][datafusion_common::config::OptimizerOptions] |
/// | `datafusion.sql_parser` | [SqlParserOptions][datafusion_common::config::SqlParserOptions] |
Expand Down Expand Up @@ -383,19 +382,6 @@ impl SessionConfig {
self.options.execution.batch_size
}

/// Get the currently configured scalar_update_factor for aggregate
pub fn agg_scalar_update_factor(&self) -> usize {
self.options.execution.aggregate.scalar_update_factor
}

/// Customize scalar_update_factor for aggregate
pub fn with_agg_scalar_update_factor(mut self, n: usize) -> Self {
// scalar update factor must be greater than zero
assert!(n > 0);
self.options.execution.aggregate.scalar_update_factor = n;
self
}

/// Enables or disables the coalescence of small batches into larger batches
pub fn with_coalesce_batches(mut self, enabled: bool) -> Self {
self.options.execution.coalesce_batches = enabled;
Expand Down
2 changes: 0 additions & 2 deletions datafusion/sqllogictest/test_files/information_schema.slt
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ datafusion.catalog.has_header true
datafusion.catalog.information_schema true
datafusion.catalog.location NULL
datafusion.catalog.newlines_in_values false
datafusion.execution.aggregate.scalar_update_factor 10
datafusion.execution.batch_size 8192
datafusion.execution.coalesce_batches true
datafusion.execution.collect_statistics false
Expand Down Expand Up @@ -259,7 +258,6 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR
datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information
datafusion.catalog.location NULL Location scanned to load tables for `default` schema
datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance.
datafusion.execution.aggregate.scalar_update_factor 10 Specifies the threshold for using `ScalarValue`s to update accumulators during high-cardinality aggregations for each input batch. The aggregation is considered high-cardinality if the number of affected groups is greater than or equal to `batch_size / scalar_update_factor`. In such cases, `ScalarValue`s are utilized for updating accumulators, rather than the default batch-slice approach. This can lead to performance improvements. By adjusting the `scalar_update_factor`, you can balance the trade-off between more efficient accumulator updates and the number of groups affected.
datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption
datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting
datafusion.execution.collect_statistics false Should DataFusion collect statistics after listing files
Expand Down
Loading

0 comments on commit dd32089

Please sign in to comment.