v0.2 (#262)

.github/workflows/test.yml

@@ -161,34 +161,34 @@ jobs:
       - name: Run Deltalake tests
         run: |
           cargo test --features=deltalake extension_cases::deltalake
-  test-hudi:
-    name: Hudi
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        arch: [amd64]
-        rust: [stable]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Cache Cargo
-        uses: actions/cache@v2
-        with:
-          path: /home/runner/.cargo
-          key: cargo-dft-cache-
-      - name: Cache Rust dependencies
-        uses: actions/cache@v2
-        with:
-          path: /home/runner/target
-          key: target-dft-cache-
-      - name: Setup Rust toolchain
-        run: |
-          rustup toolchain install ${{ matrix.rust }}
-          rustup default ${{ matrix.rust }}
-      - name: Run Hudi tests
-        run: |
-          cargo test --features=hudi extension_cases::hudi
+  # test-hudi:
+  #   name: Hudi
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       arch: [amd64]
+  #       rust: [stable]
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #       with:
+  #         submodules: true
+  #     - name: Cache Cargo
+  #       uses: actions/cache@v2
+  #       with:
+  #         path: /home/runner/.cargo
+  #         key: cargo-dft-cache-
+  #     - name: Cache Rust dependencies
+  #       uses: actions/cache@v2
+  #       with:
+  #         path: /home/runner/target
+  #         key: target-dft-cache-
+  #     - name: Setup Rust toolchain
+  #       run: |
+  #         rustup toolchain install ${{ matrix.rust }}
+  #         rustup default ${{ matrix.rust }}
+  #     - name: Run Hudi tests
+  #       run: |
+  #         cargo test --features=hudi extension_cases::hudi
   test-huggingface:
     name: HuggingFace
     runs-on: ubuntu-latest

Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "dft"
-description = "Terminal based, extensible, interactive data analysis tool using SQL "
-homepage = "https://github.com/datafusion-contrib/datafusion-tui"
-repository = "https://github.com/datafusion-contrib/datafusion-tui"
+description = "An opinionated and batteries included DataFusion implementation"
+homepage = "https://github.com/datafusion-contrib/datafusion-dft"
+repository = "https://github.com/datafusion-contrib/datafusion-dft"
 readme = "README.md"
 authors = ["Matthew Turner <[email protected]>"]
 license = "Apache-2.0"
 keywords = ["arrow", "query", "sql", "datafusion"]
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 default-run = "dft"
 
@@ -29,9 +29,9 @@ env_logger = "0.11.5"
 futures = "0.3.30"
 http = "1"
 http-body = "1"
-hudi = { features = ["datafusion"], optional = true, git = "https://github.com/apache/hudi-rs", rev = "64b1dc11cb9f1ec9be472025ecc9a43cae49d6cb" }
-iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "16f9411dd3897134a401ece97d73cd33d6790bff", optional = true}
-iceberg-datafusion = { git = "https://github.com/apache/iceberg-rust", rev = "16f9411dd3897134a401ece97d73cd33d6790bff", optional = true }
+# hudi = { features = ["datafusion"], optional = true, git = "https://github.com/apache/hudi-rs", rev = "64b1dc11cb9f1ec9be472025ecc9a43cae49d6cb" }
+iceberg-catalog-rest = { version = "0.4", optional = true}
+iceberg-datafusion = { version = "0.4", optional = true }
 itertools = "0.13.0"
 lazy_static = "1.4.0"
 log = "0.4.22"
@@ -70,7 +70,7 @@ url = "2.5.2"
 [features]
 default = ["functions-parquet"]
 deltalake = ["dep:deltalake"]
-hudi = ["dep:hudi"]
+# hudi = ["dep:hudi"]
 iceberg = ["dep:iceberg-datafusion", "dep:iceberg-catalog-rest"]
 flightsql = ["dep:arrow-flight", "dep:tonic"]
 experimental-flightsql-server = ["flightsql"]

crates/datafusion-functions-parquet/Cargo.toml

@@ -1,7 +1,14 @@
 [package]
 name = "datafusion-functions-parquet"
+description = "DataFusion helper functions for working with Parquet files"
+homepage = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-functions-parquet"
+repository = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-functions-parquet"
+readme = "README.md"
+authors = ["Matthew Turner <[email protected]>"]
+license = "Apache-2.0"
 version = "0.1.0"
 edition = "2021"
+keywords = ["parquet", "query", "sql", "datafusion"]
 
 [dependencies]
 arrow = { version = "53.0.0" }

crates/datafusion-functions-parquet/README.md

@@ -0,0 +1 @@
+# DataFusion Functions Parquet

src/extensions/mod.rs

@@ -25,8 +25,8 @@ use std::{fmt::Debug, sync::Arc};
 mod builder;
 #[cfg(feature = "deltalake")]
 mod deltalake;
-#[cfg(feature = "hudi")]
-mod hudi;
+// #[cfg(feature = "hudi")]
+// mod hudi;
 #[cfg(feature = "huggingface")]
 mod huggingface;
 #[cfg(feature = "iceberg")]
@@ -60,8 +60,8 @@ pub fn enabled_extensions() -> Vec<Arc<dyn Extension>> {
         Arc::new(s3::AwsS3Extension::new()),
         #[cfg(feature = "deltalake")]
         Arc::new(deltalake::DeltaLakeExtension::new()),
-        #[cfg(feature = "hudi")]
-        Arc::new(hudi::HudiExtension::new()),
+        // #[cfg(feature = "hudi")]
+        // Arc::new(hudi::HudiExtension::new()),
         #[cfg(feature = "iceberg")]
         Arc::new(iceberg::IcebergExtension::new()),
         #[cfg(feature = "huggingface")]