Add CI (#2)

.cargo/config

@@ -0,0 +1 @@
+../config.toml

.github/actions/setup-python/action.yml

@@ -0,0 +1,47 @@
+name: 'Setup Rust'
+description: 'Toolchain setup and Initial compilation'
+inputs:
+  rye-version:
+    description: 'Rye version to use'
+    required: true
+    default: '0.16.0'
+runs:
+  using: "composite"
+  steps:
+    - name: Rye Cache
+      id: rye-cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.rye
+        key: "rye-${{ runner.os }}-${{ inputs.rye-version }}"
+
+    - name: Rye Install
+      shell: bash
+      run: curl -sSf https://rye-up.com/get | bash
+      if: steps.rye-cache.outputs.cache-hit != 'true'
+      env:
+        RYE_VERSION: "${{ inputs.rye-version }}"
+        RYE_INSTALL_OPTION: "--yes"
+
+    - name: Rye Shims
+      shell: bash
+      run: echo "~/.rye/shims" >> $GITHUB_PATH
+
+    - name: Venv Cache
+      id: venv-cache
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: "venv-${{ runner.os }}-${{ hashFiles('requirements**.lock') }}"
+
+    - name: Rye Sync
+      shell: bash
+      # --no-lock prevents resolution of the lock file. The locks are still respected.
+      # We always run `rye sync` even if the cache fetch was successful since it builds our Rust extensions for us.
+      run: rye sync --no-lock
+      env:
+        MATURIN_PEP517_ARGS: "--profile dev"
+
+    - name: Export Path
+      shell: bash
+      run: echo "PATH=$PATH" >> $GITHUB_ENV

.github/actions/setup-rust/action.yml

@@ -0,0 +1,44 @@
+name: 'Setup Rust'
+description: 'Toolchain setup and Initial compilation'
+inputs:
+  rust-toolchain: # id of input
+    description: 'Rust toolchain version to use'
+    required: true
+    default: stable
+runs:
+  using: "composite"
+  steps:
+    - name: Rust Toolchain Cache
+      id: rustup-cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.rustup
+        key: "rustup-${{ runner.os }}-${{ inputs.rust-toolchain }}"
+
+    - name: Rust Toolchain
+      uses: dtolnay/rust-toolchain@stable
+      if: steps.rustup-cache.outputs.cache-hit != 'true'
+      with:
+        toolchain: "${{ inputs.rust-toolchain }}"
+        components: clippy, rustfmt
+    - name: Rust Dependency Cache
+      uses: Swatinem/rust-cache@v2
+      with:
+        shared-key: "shared" # To allow reuse across jobs
+
+    - name: Rust Compile Cache
+      uses: mozilla-actions/[email protected]
+    - name: Rust Compile Cache Config
+      shell: bash
+      run: |
+        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
+
+    - name: Rust Build
+      shell: bash
+      run: cargo build --all-targets
+
+    - name: Export Path
+      shell: bash
+      run: echo "PATH=$PATH" >> $GITHUB_ENV

.github/actions/setup-zig/action.yml

@@ -0,0 +1,23 @@
+name: 'Setup Zig'
+description: 'Toolchain setup and Initial compilation'
+runs:
+  using: "composite"
+  steps:
+    - name: Zig Version
+      id: zig-version
+      shell: bash
+      run: echo "version=$(cat .zig-version | tr -d '\n ')" >> $GITHUB_OUTPUT
+
+    - name: Zig Setup
+      uses: goto-bus-stop/setup-zig@v2
+      with:
+        version: "${{ steps.zig-version.outputs.version }}"
+
+    # TODO(ngates): should we cache zig-cache? Or zig-out?
+    - name: Zig Build
+      shell: bash
+      run: zig build
+
+    - name: Export Path
+      shell: bash
+      run: echo "PATH=$PATH" >> $GITHUB_ENV

.github/workflows/bench-pr.yml

@@ -0,0 +1,42 @@
+name: PR Benchmarks
+
+on:
+  pull_request:
+    types: [ labeled, synchronize ]
+    branches: [ "develop" ]
+  workflow_dispatch: { }
+
+permissions:
+  actions: write
+  contents: read
+  pull-requests: write
+
+jobs:
+  bench:
+    runs-on: ubuntu-latest-large
+    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
+    steps:
+      # We remove the benchmark label first so that the workflow can be re-triggered.
+      - uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: benchmark
+
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/setup-zig
+      - uses: ./.github/actions/setup-rust
+
+      - name: Bench - Vortex
+        run: cargo bench | tee bench.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/[email protected]
+        with:
+          name: Vortex Benchmarks
+          tool: cargo
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          output-file-path: bench.txt
+          summary-always: true
+          auto-push: true
+          fail-on-alert: false
+

.github/workflows/bench.yml

@@ -0,0 +1,35 @@
+name: Benchmarks
+
+on:
+  push:
+    branches: [ "develop" ]
+  workflow_dispatch: { }
+
+permissions:
+  actions: read
+  contents: write
+  deployments: write
+
+jobs:
+  bench:
+    runs-on: ubuntu-latest-large
+    if: ${{ github.event_name == 'workflow_dispatch' || (contains(github.event.head_commit.message, '[benchmark]') && github.ref_name == 'develop') }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/setup-zig
+      - uses: ./.github/actions/setup-rust
+
+      - name: Bench - Vortex
+        run: cargo bench | tee bench.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/[email protected]
+        with:
+          name: Vortex Benchmarks
+          tool: cargo
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          output-file-path: bench.txt
+          summary-always: true
+          auto-push: true
+          fail-on-alert: false

.github/workflows/ci.yml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    branches: [ "develop" ]
+  workflow_dispatch: { }
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  build:
+    name: 'build'
+    runs-on: ubuntu-latest-medium
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-zig
+      - uses: ./.github/actions/setup-rust
+      - uses: ./.github/actions/setup-python
+
+      - name: Python Lint - Format
+        run: rye run ruff format --check .
+      - name: Python Lint - Ruff
+        run: rye run ruff .
+
+      - name: Rust Lint - Format
+        run: cargo fmt --all --check
+      - name: Rust Lint - Clippy
+        run: cargo clippy --all-targets
+
+      - name: Zig Lint - Fmt
+        run: zig fmt --check zig/
+
+      - name: Rust Test
+        run: cargo test --all
+
+      - name: Zig Test
+        run: zig build test
+
+      - name: Pytest - PyEnc
+        run: rye run pytest --benchmark-disable test/
+        working-directory: pyenc/
+

codecz-sys/build.rs

@@ -26,11 +26,7 @@ fn main() {
     for entry in WalkDir::new(root_dir.join("zig"))
         .into_iter()
         .filter_map(|e| e.ok())
-        .filter(|e| {
-            !e.path()
-                .components()
-                .any(|c| c.as_os_str() == "zig-cache")
-        })
+        .filter(|e| !e.path().components().any(|c| c.as_os_str() == "zig-cache"))
         .filter(|e| {
             e.path()
                 .extension()

codecz/src/math.rs

@@ -17,7 +17,6 @@ use codecz_sys::{
     RunLengthStats_t,
 };
 use num_traits::Num;
-use paste;
 
 #[allow(dead_code)]
 pub struct RunLengthStats {

enc-ree/src/compress.rs

@@ -95,7 +95,7 @@ mod test {
     use arrow::buffer::BooleanBuffer;
 
     use enc::array::bool::BoolArray;
-    use enc::array::primitive::PrimitiveArray;
+    use enc::array::downcast::DowncastArrayBuiltin;
     use enc::array::Array;
 
     use crate::compress::ree_decode;
@@ -117,14 +117,8 @@ mod test {
         );
 
         let decoded = ree_decode(
-            arr.ends()
-                .as_any()
-                .downcast_ref::<PrimitiveArray>()
-                .unwrap(),
-            arr.values()
-                .as_any()
-                .downcast_ref::<PrimitiveArray>()
-                .unwrap(),
+            arr.ends().as_primitive(),
+            arr.values().as_primitive(),
             arr.validity().cloned(),
         );
 
@@ -133,13 +127,7 @@ mod test {
             vec![1i32, 1, 2, 2, 2, 3, 3, 3, 3, 3].as_slice()
         );
         assert_eq!(
-            decoded
-                .validity()
-                .unwrap()
-                .as_any()
-                .downcast_ref::<BoolArray>()
-                .unwrap()
-                .buffer(),
+            decoded.validity().unwrap().as_bool().buffer(),
             &BooleanBuffer::from(vec![
                 true, true, false, true, true, true, true, false, true, true,
             ])

enc-ree/src/ree.rs

@@ -312,15 +312,13 @@ fn run_ends_logical_length<T: AsRef<dyn Array>>(ends: &T) -> usize {
 
 #[cfg(test)]
 mod test {
-    use std::ops::Deref;
-
     use arrow::array::cast::AsArray;
     use arrow::array::types::Int32Type;
+    use enc::array::Array;
     use itertools::Itertools;
 
-    use enc::dtype::{IntWidth, Nullability, Signedness};
-
-    use super::*;
+    use crate::REEArray;
+    use enc::dtype::{DType, IntWidth, Nullability, Signedness};
 
     #[test]
     fn new() {
@@ -354,7 +352,7 @@ mod test {
         arr.iter_arrow()
             .zip_eq([vec![2, 2, 3, 3, 3]])
             .for_each(|(from_iter, orig)| {
-                assert_eq!(from_iter.as_primitive::<Int32Type>().values().deref(), orig);
+                assert_eq!(*from_iter.as_primitive::<Int32Type>().values(), orig);
             });
     }
 
@@ -364,7 +362,7 @@ mod test {
         arr.iter_arrow()
             .zip_eq([vec![1, 1, 2, 2, 2, 3, 3, 3, 3, 3]])
             .for_each(|(from_iter, orig)| {
-                assert_eq!(from_iter.as_primitive::<Int32Type>().values().deref(), orig);
+                assert_eq!(*from_iter.as_primitive::<Int32Type>().values(), orig);
             });
     }
 }

enc-roaring/src/boolean/serde.rs

@@ -32,6 +32,7 @@ impl EncodingSerde for RoaringBoolEncoding {
 
 #[cfg(test)]
 mod test {
+    use crate::downcast::DowncastRoaring;
     use croaring::Bitmap;
 
     use crate::serde_tests::test::roundtrip_array;
@@ -42,10 +43,7 @@ mod test {
         let arr = RoaringBoolArray::new(Bitmap::from_range(245..63000), 65536);
         let read_arr = roundtrip_array(arr.as_ref()).unwrap();
 
-        let read_roaring = read_arr
-            .as_any()
-            .downcast_ref::<RoaringBoolArray>()
-            .unwrap();
+        let read_roaring = read_arr.as_roaring_bool();
         assert_eq!(arr.bitmap(), read_roaring.bitmap());
     }
 }

enc-roaring/src/downcast.rs

@@ -6,6 +6,7 @@ mod private {
     pub trait Sealed {}
 }
 
+#[allow(dead_code)]
 pub trait DowncastRoaring: private::Sealed {
     fn maybe_roaring_int(&self) -> Option<&RoaringIntArray>;
 

enc-zigzag/src/compress.rs

@@ -19,9 +19,7 @@ impl EncodingCompression for ZigZagEncoding {
         _config: &CompressConfig,
     ) -> Option<&'static Compressor> {
         // Only support primitive arrays
-        let Some(parray) = array.maybe_primitive() else {
-            return None;
-        };
+        let parray = array.maybe_primitive()?;
 
         // Only supports signed integers
         if !parray.ptype().is_signed_int() {