Skip to content

Commit

Permalink
Hive style partitions (#20)
Browse files Browse the repository at this point in the history
* most things set up for implementing partitions, just a few steps left

* hive style partitions are working, just need to clean up a bit I think

* linting

* more linting

* addressed PR comments

* small linter fix
  • Loading branch information
maximedion2 authored Jun 28, 2024
1 parent f852355 commit dbb3e42
Show file tree
Hide file tree
Showing 6 changed files with 712 additions and 48 deletions.
8 changes: 3 additions & 5 deletions src/datafusion/file_opener.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,17 @@ impl FileOpener for ZarrFileOpener {
let zarr_path = ZarrPath::new(config.object_store, file_meta.object_meta.location);
let rng = file_meta.range.map(|r| (r.start as usize, r.end as usize));
let projection = ZarrProjection::from(config.projection.as_ref());

let mut batch_reader_builder =
ZarrRecordBatchStreamBuilder::new(zarr_path.clone()).with_projection(projection);
if let Some(filters) = filters_to_pushdown {
let schema = zarr_path
let file_schema = zarr_path
.get_zarr_metadata()
.await
.map_err(|e| DataFusionError::External(Box::new(e)))?
.arrow_schema()
.map_err(|e| DataFusionError::External(Box::new(e)))?;
let filters = build_row_filter(&filters, &schema)?;
let filters = build_row_filter(&filters, &file_schema)?;

if let Some(filters) = filters {
batch_reader_builder = batch_reader_builder.with_filter(filters);
}
Expand All @@ -71,9 +71,7 @@ impl FileOpener for ZarrFileOpener {
.build_partial_reader(rng)
.await
.map_err(|e| DataFusionError::External(Box::new(e)))?;

let stream = batch_reader.map_err(|e| ArrowError::from_external_error(Box::new(e)));

Ok(stream.boxed())
}))
}
Expand Down
Loading

0 comments on commit dbb3e42

Please sign in to comment.