Skip to content

Commit

Permalink
Reduce impact of datafusion features on the dependency tree (#1235)
Browse files Browse the repository at this point in the history
In service of my one true passion (removing crate features), an attemtp
at reducing datafusion to use the smallest possible subset of features,
preferring to use smaller crates when possible.
  • Loading branch information
AdamGS authored Nov 6, 2024
1 parent 0195c02 commit 7307b4a
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 136 deletions.
130 changes: 3 additions & 127 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ allocator-api2 = "0.2.16"
anyhow = "1.0"
arbitrary = "1.3.2"
arrayref = "0.3.7"
arrow = { version = "53.0.0", features = ["prettyprint"] }
arrow = { version = "53.0.0" }
arrow-arith = "53.0.0"
arrow-array = "53.0.0"
arrow-buffer = "53.0.0"
Expand All @@ -69,7 +69,7 @@ clap = "4.5.13"
criterion = { version = "0.5.1", features = ["html_reports"] }
croaring = "2.1.0"
csv = "1.3.0"
datafusion = "42.0.0"
datafusion = { version = "42.0.0", default-features = false }
datafusion-common = "42.0.0"
datafusion-execution = "42.0.0"
datafusion-expr = "42.0.0"
Expand Down Expand Up @@ -208,9 +208,9 @@ use_debug = "deny"

[profile.release]
codegen-units = 1
lto = "thin" # attempts to perform optimizations across all crates within the dependency graph
lto = "thin" # attempts to perform optimizations across all crates within the dependency graph

[profile.bench]
codegen-units = 16 # default for "release", which "bench" inherits
lto = false # default
lto = false # default
debug = true
10 changes: 8 additions & 2 deletions bench-vortex/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,13 @@ bytes = { workspace = true }
bzip2 = { workspace = true }
clap = { workspace = true, features = ["derive"] }
csv = { workspace = true }
datafusion = { workspace = true }
datafusion = { workspace = true, features = [
"parquet",
"datetime_expressions",
] }
datafusion-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
datafusion-execution = { workspace = true }
enum-iterator = { workspace = true }
flexbuffers = { workspace = true }
futures = { workspace = true, features = ["executor"] }
Expand All @@ -41,7 +47,7 @@ itertools = { workspace = true }
log = { workspace = true }
mimalloc = { workspace = true }
object_store = { workspace = true, features = ["aws"] }
parquet = { workspace = true, features = [] }
parquet = { workspace = true, features = ["async"] }
prettytable-rs = { workspace = true }
rand = { workspace = true }
rayon = { workspace = true }
Expand Down
2 changes: 1 addition & 1 deletion bench-vortex/benches/datafusion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ use arrow_array::RecordBatch;
use arrow_schema::{DataType, Field, Schema};
use criterion::measurement::Measurement;
use criterion::{black_box, criterion_group, criterion_main, BenchmarkGroup, Criterion};
use datafusion::common::Result as DFResult;
use datafusion::datasource::{MemTable, TableProvider};
use datafusion::execution::memory_pool::human_readable_size;
use datafusion::functions_aggregate::count::count_distinct;
use datafusion::logical_expr::lit;
use datafusion::prelude::{col, DataFrame, SessionContext};
use datafusion_common::Result as DFResult;
use vortex::aliases::hash_set::HashSet;
use vortex::compress::CompressionStrategy;
use vortex::dict::DictEncoding;
Expand Down
4 changes: 2 additions & 2 deletions bench-vortex/src/tpch/execute.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use arrow_array::RecordBatch;
use datafusion::common::Result;
use datafusion::physical_plan::collect;
use datafusion::prelude::SessionContext;
use datafusion_common::Result;
use datafusion_physical_plan::collect;

use crate::tpch::Format;

Expand Down

0 comments on commit 7307b4a

Please sign in to comment.