Skip to content

Commit

Permalink
Merge remote-tracking branch 'apache/main' into alamb/specialized_gro…
Browse files Browse the repository at this point in the history
…up_keys
  • Loading branch information
alamb committed Jan 27, 2024
2 parents 886666b + ed24539 commit 9224a38
Show file tree
Hide file tree
Showing 289 changed files with 13,193 additions and 7,138 deletions.
15 changes: 2 additions & 13 deletions .github/actions/setup-builder/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,8 @@ runs:
rustup toolchain install ${{ inputs.rust-version }}
rustup default ${{ inputs.rust-version }}
rustup component add rustfmt
- name: Disable debuginfo generation
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
shell: bash
run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
- name: Disable incremental compilation
# Disable incremental compilation to save diskspace (the CI doesn't recompile modified files)
# https://github.com/apache/arrow-datafusion/issues/6676
shell: bash
run: echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
- name: Enable backtraces
shell: bash
run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
- name: Configure rust runtime env
uses: ./.github/actions/setup-rust-runtime
- name: Fixup git permissions
# https://github.com/actions/checkout/issues/766
shell: bash
Expand Down
47 changes: 47 additions & 0 deletions .github/actions/setup-macos-builder/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Prepare Rust Builder for MacOS
description: 'Prepare Rust Build Environment for MacOS'
inputs:
rust-version:
description: 'version of rust to install (e.g. stable)'
required: true
default: 'stable'
runs:
using: "composite"
steps:
- name: Install protobuf compiler
shell: bash
run: |
mkdir -p $HOME/d/protoc
cd $HOME/d/protoc
export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
unzip $PROTO_ZIP
echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
export PATH=$PATH:$HOME/d/protoc/bin
protoc --version
- name: Setup Rust toolchain
shell: bash
run: |
rustup update stable
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
- name: Configure rust runtime env
uses: ./.github/actions/setup-rust-runtime
41 changes: 41 additions & 0 deletions .github/actions/setup-rust-runtime/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Setup Rust Runtime
description: 'Setup Rust Runtime Environment'
runs:
using: "composite"
steps:
- name: Run sccache-cache
uses: mozilla-actions/[email protected]
- name: Configure runtime env
shell: bash
# do not produce debug symbols to keep memory usage down
# hardcoding other profile params to avoid profile override values
# More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
#
# Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
# See for more details: https://github.com/rust-lang/rust/issues/119560
#
# set RUST_MIN_STACK to avoid rust stack overflows on tpc-ds tests
run: |
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
46 changes: 46 additions & 0 deletions .github/actions/setup-windows-builder/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Prepare Rust Builder for Windows
description: 'Prepare Rust Build Environment for Windows'
inputs:
rust-version:
description: 'version of rust to install (e.g. stable)'
required: true
default: 'stable'
runs:
using: "composite"
steps:
- name: Install protobuf compiler
shell: bash
run: |
mkdir -p $HOME/d/protoc
cd $HOME/d/protoc
export PROTO_ZIP="protoc-21.4-win64.zip"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
unzip $PROTO_ZIP
export PATH=$PATH:$HOME/d/protoc/bin
protoc.exe --version
- name: Setup Rust toolchain
shell: bash
run: |
rustup update stable
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
- name: Configure rust runtime env
uses: ./.github/actions/setup-rust-runtime
84 changes: 17 additions & 67 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ jobs:
rust-version: stable

- name: Cache Cargo
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
Expand Down Expand Up @@ -96,17 +96,9 @@ jobs:
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: stable
rust-version: stable
- name: Run tests (excluding doctests)
run: cargo test --lib --tests --bins --features avro,json,backtrace
env:
# do not produce debug symbols to keep memory usage down
# hardcoding other profile params to avoid profile override values
# More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256"
RUST_BACKTRACE: "1"
# avoid rust stack overflows on tpc-ds tests
RUST_MIN_STACK: "3000000"
- name: Verify Working Directory Clean
run: git diff --exit-code

Expand Down Expand Up @@ -284,79 +276,31 @@ jobs:
- uses: actions/checkout@v4
with:
submodules: true
- name: Install protobuf compiler
shell: bash
run: |
mkdir -p $HOME/d/protoc
cd $HOME/d/protoc
export PROTO_ZIP="protoc-21.4-win64.zip"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
unzip $PROTO_ZIP
export PATH=$PATH:$HOME/d/protoc/bin
protoc.exe --version
# TODO: this won't cache anything, which is expensive. Setup this action
# with a OS-dependent path.
- name: Setup Rust toolchain
run: |
rustup update stable
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
uses: ./.github/actions/setup-windows-builder
- name: Run tests (excluding doctests)
shell: bash
run: |
export PATH=$PATH:$HOME/d/protoc/bin
cargo test --lib --tests --bins --features avro,json,backtrace
cd datafusion-cli
cargo test --lib --tests --bins --all-features
env:
# Minimize producing debug symbols to keep memory usage down
# Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
# See for more details: https://github.com/rust-lang/rust/issues/119560
RUSTFLAGS: "-C debuginfo=line-tables-only"
RUST_BACKTRACE: "1"
# avoid rust stack overflows on tpc-ds tests
RUST_MIN_STACK: "3000000"
macos:
name: cargo test (mac)
name: cargo test (macos)
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Install protobuf compiler
shell: bash
run: |
mkdir -p $HOME/d/protoc
cd $HOME/d/protoc
export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
unzip $PROTO_ZIP
echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
export PATH=$PATH:$HOME/d/protoc/bin
protoc --version
# TODO: this won't cache anything, which is expensive. Setup this action
# with a OS-dependent path.
submodules: true
- name: Setup Rust toolchain
run: |
rustup update stable
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
uses: ./.github/actions/setup-macos-builder
- name: Run tests (excluding doctests)
shell: bash
run: |
cargo test --lib --tests --bins --features avro,json,backtrace
cd datafusion-cli
cargo test --lib --tests --bins --all-features
env:
# do not produce debug symbols to keep memory usage down
# hardcoding other profile params to avoid profile override values
# More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256"
RUST_BACKTRACE: "1"
# avoid rust stack overflows on tpc-ds tests
RUST_MIN_STACK: "3000000"
cargo test --lib --tests --bins --all-features
test-datafusion-pyarrow:
name: cargo test pyarrow (amd64)
Expand Down Expand Up @@ -439,7 +383,7 @@ jobs:
# rustup default stable
# rustup component add rustfmt clippy
# - name: Cache Cargo
# uses: actions/cache@v3
# uses: actions/cache@v4
# with:
# path: /home/runner/.cargo
# # this key is not equal because the user is different than on a container (runner vs github)
Expand Down Expand Up @@ -544,7 +488,7 @@ jobs:
# Verify MSRV for the crates which are directly used by other projects.
msrv:
name: Verify MSRV
name: Verify MSRV (Min Supported Rust Version)
runs-on: ubuntu-latest
container:
image: amd64/rust
Expand All @@ -556,7 +500,13 @@ jobs:
run: cargo install cargo-msrv
- name: Check datafusion
working-directory: datafusion/core
run: cargo msrv verify
run: |
# If you encounter an error with any of the commands below
# it means some crate in your dependency tree has a higher
# MSRV (Min Supported Rust Version) than the one specified
# in the `rust-version` key of `Cargo.toml`. Check your
# dependencies or update the version in `Cargo.toml`
cargo msrv verify
- name: Check datafusion-substrait
working-directory: datafusion/substrait
run: cargo msrv verify
Expand Down
46 changes: 23 additions & 23 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,33 +29,33 @@ license = "Apache-2.0"
readme = "README.md"
repository = "https://github.com/apache/arrow-datafusion"
rust-version = "1.70"
version = "34.0.0"
version = "35.0.0"

[workspace.dependencies]
arrow = { version = "49.0.0", features = ["prettyprint"] }
arrow-array = { version = "49.0.0", default-features = false, features = ["chrono-tz"] }
arrow-buffer = { version = "49.0.0", default-features = false }
arrow-flight = { version = "49.0.0", features = ["flight-sql-experimental"] }
arrow-ipc = { version = "49.0.0", default-features = false, features = ["lz4"] }
arrow-ord = { version = "49.0.0", default-features = false }
arrow-schema = { version = "49.0.0", default-features = false }
arrow = { version = "50.0.0", features = ["prettyprint"] }
arrow-array = { version = "50.0.0", default-features = false, features = ["chrono-tz"] }
arrow-buffer = { version = "50.0.0", default-features = false }
arrow-flight = { version = "50.0.0", features = ["flight-sql-experimental"] }
arrow-ipc = { version = "50.0.0", default-features = false, features = ["lz4"] }
arrow-ord = { version = "50.0.0", default-features = false }
arrow-schema = { version = "50.0.0", default-features = false }
async-trait = "0.1.73"
bigdecimal = "0.4.1"
bytes = "1.4"
chrono = { version = "0.4.31", default-features = false }
ctor = "0.2.0"
dashmap = "5.4.0"
datafusion = { path = "datafusion/core", version = "34.0.0" }
datafusion-common = { path = "datafusion/common", version = "34.0.0" }
datafusion-execution = { path = "datafusion/execution", version = "34.0.0" }
datafusion-expr = { path = "datafusion/expr", version = "34.0.0" }
datafusion-optimizer = { path = "datafusion/optimizer", version = "34.0.0" }
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "34.0.0" }
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "34.0.0" }
datafusion-proto = { path = "datafusion/proto", version = "34.0.0" }
datafusion-sql = { path = "datafusion/sql", version = "34.0.0" }
datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "34.0.0" }
datafusion-substrait = { path = "datafusion/substrait", version = "34.0.0" }
datafusion = { path = "datafusion/core", version = "35.0.0" }
datafusion-common = { path = "datafusion/common", version = "35.0.0" }
datafusion-execution = { path = "datafusion/execution", version = "35.0.0" }
datafusion-expr = { path = "datafusion/expr", version = "35.0.0" }
datafusion-optimizer = { path = "datafusion/optimizer", version = "35.0.0" }
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "35.0.0" }
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "35.0.0" }
datafusion-proto = { path = "datafusion/proto", version = "35.0.0" }
datafusion-sql = { path = "datafusion/sql", version = "35.0.0" }
datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "35.0.0" }
datafusion-substrait = { path = "datafusion/substrait", version = "35.0.0" }
doc-comment = "0.3"
env_logger = "0.10"
futures = "0.3"
Expand All @@ -64,13 +64,13 @@ indexmap = "2.0.0"
itertools = "0.12"
log = "^0.4"
num_cpus = "1.13.0"
object_store = { version = "0.8.0", default-features = false }
object_store = { version = "0.9.0", default-features = false }
parking_lot = "0.12"
parquet = { version = "49.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
parquet = { version = "50.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
rand = "0.8"
rstest = "0.18.0"
serde_json = "1"
sqlparser = { version = "0.41.0", features = ["visitor"] }
sqlparser = { version = "0.43.0", features = ["visitor"] }
tempfile = "3"
thiserror = "1.0.44"
url = "2.2"
Expand All @@ -91,4 +91,4 @@ lto = false
opt-level = 3
overflow-checks = false
panic = 'unwind'
rpath = false
rpath = false
Loading

0 comments on commit 9224a38

Please sign in to comment.