Merge remote-tracking branch 'apache/main' into issue_14466

Lordworms · Feb 9, 2025 · 09b1923 · 09b1923
2 parents a6a7d70 + 3550758
commit 09b1923
Show file tree

Hide file tree

Showing 189 changed files with 3,547 additions and 1,031 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -28,24 +28,9 @@ updates:
       # arrow is bumped manually
       - dependency-name: "arrow*"
         update-types: ["version-update:semver-major"]
-  - package-ecosystem: cargo
-    directory: "datafusion-cli/"
-    schedule:
-      interval: daily
-    open-pull-requests-limit: 10
-    target-branch: main
-    labels: [auto-dependencies]
-    ignore:
-      # arrow is bumped manually
-      - dependency-name: "arrow*"
-        update-types: ["version-update:semver-major"]
-      # datafusion is bumped manually
-      - dependency-name: "datafusion*"
-        update-types: ["version-update:semver-major"]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
-
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
@@ -25,9 +25,11 @@ on:
   push:
     paths:
       - "**/Cargo.toml"
+      - "**/Cargo.lock"
   pull_request:
     paths:
       - "**/Cargo.toml"
+      - "**/Cargo.lock"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
@@ -50,4 +52,4 @@ jobs:
       - name: Check dependencies
         run: |
           cd dev/depcheck
-          cargo run
+          cargo run
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml
@@ -23,9 +23,11 @@ concurrency:
 
 # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
 #
-# this job is intended to only run only on the main branch as it is time consuming
-# and should not fail often. However, it is important coverage to ensure correctness
-# in the (very rare) event of a hash failure or sqlite query failure.
+# These jobs only run on the `main` branch as they are time consuming
+# and should not fail often.
+#
+# We still run them as they provide important coverage to ensure correctness
+# in the (very rare) event of a hash failure or sqlite library query failure.
 on:
   # Run on all commits to main
   push:

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -60,7 +60,11 @@ jobs:
         with:
           rust-version: stable
       - name: Prepare cargo build
-        run: cargo check --profile ci --all-targets --features integration-tests
+        run: |
+          # Adding `--locked` here to assert that the `Cargo.lock` file is up to
+          # date with the manifest. When this fails, please make sure to commit
+          # the changes to `Cargo.lock` after building with the updated manifest.
+          cargo check --profile ci --workspace --all-targets --features integration-tests --locked
 
   # cargo check common, functions and substrait with no default features
   linux-cargo-check-no-default-features:
@@ -95,12 +99,6 @@ jobs:
       - name: Check workspace with additional features
         run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests
 
-      - name: Check Cargo.lock for datafusion-cli
-        run: |
-          # If this test fails, try running `cargo update` in the `datafusion-cli` directory
-          # and check in the updated Cargo.lock file.
-          cargo check --profile ci --manifest-path datafusion-cli/Cargo.toml --locked
-
   # cargo check datafusion to ensure that the datafusion crate can be built with only a
   # subset of the function packages enabled.
   linux-cargo-check-datafusion:
@@ -189,28 +187,6 @@ jobs:
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
-  linux-test-datafusion-cli:
-    name: cargo test datafusion-cli (amd64)
-    needs: linux-build-lib
-    runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
-      - name: Run tests (excluding doctests)
-        run: |
-          cd datafusion-cli
-          cargo test --profile ci --lib --tests --bins --all-features
-      - name: Verify Working Directory Clean
-        run: git diff --exit-code
-
   linux-test-example:
     name: cargo examples (amd64)
     needs: linux-build-lib
@@ -252,10 +228,7 @@ jobs:
         with:
           rust-version: stable
       - name: Run doctests
-        run: |
-          cargo test --profile ci --doc --features avro,json
-          cd datafusion-cli
-          cargo test --profile ci --doc --all-features
+        run: cargo test --profile ci --doc --features avro,json
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -364,45 +337,40 @@ jobs:
           POSTGRES_HOST: postgres
           POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
 
-#  Temporarily commenting out the Windows flow, the reason is enormously slow running build
-#  Waiting for new Windows 2025 github runner
-#  Details: https://github.com/apache/datafusion/issues/13726
-#
-#  windows:
-#    name: cargo test (win64)
-#    runs-on: windows-latest
-#    steps:
-#      - uses: actions/checkout@v4
-#        with:
-#          submodules: true
-#      - name: Setup Rust toolchain
-#        uses: ./.github/actions/setup-windows-builder
-#      - name: Run tests (excluding doctests)
-#        shell: bash
-#        run: |
-#          export PATH=$PATH:$HOME/d/protoc/bin
-#          cargo test --lib --tests --bins --features avro,json,backtrace
-#          cd datafusion-cli
-#          cargo test --lib --tests --bins --all-features
-
-# Commenting out intel mac build as so few users would ever use it
-# Details: https://github.com/apache/datafusion/issues/13846
-#  macos:
-#    name: cargo test (macos)
-#    runs-on: macos-latest
-#    steps:
-#      - uses: actions/checkout@v4
-#        with:
-#          submodules: true
-#          fetch-depth: 1
-#      - name: Setup Rust toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#      - name: Run tests (excluding doctests)
-#        shell: bash
-#        run: |
-#          cargo test run --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace
-#          cd datafusion-cli
-#          cargo test run --profile ci --lib --tests --bins --all-features
+  #  Temporarily commenting out the Windows flow, the reason is enormously slow running build
+  #  Waiting for new Windows 2025 github runner
+  #  Details: https://github.com/apache/datafusion/issues/13726
+  #
+  #  windows:
+  #    name: cargo test (win64)
+  #    runs-on: windows-latest
+  #    steps:
+  #      - uses: actions/checkout@v4
+  #        with:
+  #          submodules: true
+  #      - name: Setup Rust toolchain
+  #        uses: ./.github/actions/setup-windows-builder
+  #      - name: Run tests (excluding doctests)
+  #        shell: bash
+  #        run: |
+  #          export PATH=$PATH:$HOME/d/protoc/bin
+  #          cargo test --lib --tests --bins --features avro,json,backtrace
+
+  # Commenting out intel mac build as so few users would ever use it
+  # Details: https://github.com/apache/datafusion/issues/13846
+  #  macos:
+  #    name: cargo test (macos)
+  #    runs-on: macos-latest
+  #    steps:
+  #      - uses: actions/checkout@v4
+  #        with:
+  #          submodules: true
+  #          fetch-depth: 1
+  #      - name: Setup Rust toolchain
+  #        uses: ./.github/actions/setup-macos-builder
+  #      - name: Run tests (excluding doctests)
+  #        shell: bash
+  #        run: cargo test run --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace
 
   macos-aarch64:
     name: cargo test (macos-aarch64)
@@ -416,10 +384,7 @@ jobs:
         uses: ./.github/actions/setup-macos-aarch64-builder
       - name: Run tests (excluding doctests)
         shell: bash
-        run: |
-          cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests
-          cd datafusion-cli
-          cargo test --profile ci --lib --tests --bins --all-features
+        run: cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests
 
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)
@@ -615,19 +580,19 @@ jobs:
           # (Min Supported Rust Version) than the one specified in the
           # `rust-version` key of `Cargo.toml`.
           #
-          # To reproduce: 
-          # 1. Install the version of Rust that is failing. Example: 
+          # To reproduce:
+          # 1. Install the version of Rust that is failing. Example:
           #    rustup install 1.80.1
           # 2. Run the command that failed with that version. Example:
           #    cargo +1.80.1 check -p datafusion
-          # 
+          #
           # To resolve, either:
-          # 1. Change your code to use older Rust features, 
+          # 1. Change your code to use older Rust features,
           # 2. Revert dependency update
           # 3. Update the MSRV version in `Cargo.toml`
           #
           # Please see the DataFusion Rust Version Compatibility Policy before
-          # updating Cargo.toml. You may have to update the code instead. 
+          # updating Cargo.toml. You may have to update the code instead.
           # https://github.com/apache/datafusion/blob/main/README.md#rust-version-compatibility-policy
           cargo msrv --output-format json --log-target stdout verify
       - name: Check datafusion-substrait
@@ -636,6 +601,3 @@ jobs:
       - name: Check datafusion-proto
         working-directory: datafusion/proto
         run: cargo msrv --output-format json --log-target stdout verify
-      - name: Check datafusion-cli
-        working-directory: datafusion-cli
-        run: cargo msrv --output-format json --log-target stdout verify
diff --git a/.gitignore b/.gitignore
@@ -42,8 +42,6 @@ venv/*
 
 # Rust
 target
-Cargo.lock
-!datafusion-cli/Cargo.lock
 
 rusty-tags.vi
 .history