Skip to content

Proof of concept: caching part of the emit_dist computation #1081

Proof of concept: caching part of the emit_dist computation

Proof of concept: caching part of the emit_dist computation #1081

Workflow file for this run

name: checks
permissions:
contents: read
on:
push:
branches:
- main
- 'release/**'
pull_request:
branches-ignore:
- 'release/**'
schedule:
- cron: '0 4 * * *'
merge_group:
branches:
- main
jobs:
build:
name: Build and test
runs-on: "${{ matrix.os }}"
strategy:
matrix:
include:
- rust: stable
os: ubuntu-latest
features: ""
target: "x86_64-unknown-linux-gnu"
- rust: msrv
os: ubuntu-latest
features: ""
target: "x86_64-unknown-linux-gnu"
- rust: beta
os: ubuntu-latest
features: ""
target: "x86_64-unknown-linux-gnu"
- rust: "stable"
os: macos-latest
features: ""
target: "x86_64-apple-darwin"
- rust: "stable"
os: macos-14
features: ""
target: "aarch64-apple-darwin"
- rust: stable-x86_64-gnu
os: windows-2022
features: ""
target: "x86_64-pc-windows-gnu"
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Set target rust version
run: echo "TARGET_RUST_VERSION=$(if [ "${{matrix.rust}}" = "msrv" ]; then grep rust-version Cargo.toml | grep MSRV | cut -d'"' -f2; else echo "${{matrix.rust}}"; fi)" >> $GITHUB_ENV
if: matrix.os != 'windows-2022'
- name: Install toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: "${TARGET_RUST_VERSION}"
targets: "${{ matrix.target }}"
if: matrix.os != 'windows-2022'
- name: Install toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: "${{matrix.rust}}"
targets: "${{ matrix.target }}"
if: matrix.os == 'windows-2022'
- name: target
run: "rustc -vV | sed -n 's|host: ||p'"
if: matrix.os != 'windows-2022'
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-llvm-cov
- name: Install cargo-nextest
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-nextest
- name: cargo build
run: cargo build --target ${{matrix.target}} ${{ matrix.features }}
- name: cargo nextest # reports segfaults in a helpful way
run: cargo nextest run --target ${{matrix.target}} ${{ matrix.features }} --no-fail-fast
env:
RUST_BACKTRACE: 1
- name: cargo test with llvm-cov
run: cargo llvm-cov --target ${{matrix.target}} ${{ matrix.features }} --lcov --output-path lcov.info
env:
RUST_BACKTRACE: 1
if: matrix.os != 'windows-2022'
- name: Upload coverage to Codecov
uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
if: matrix.os != 'windows-2022'
with:
files: lcov.info
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
slug: codecov/codecov-demo
build-qemu:
name: QEMU build & test
runs-on: "${{ matrix.os }}"
strategy:
matrix:
include:
- rust: stable
os: ubuntu-latest
features: ""
target: "s390x-unknown-linux-gnu"
gcc: "s390x-linux-gnu-gcc"
- rust: stable
os: ubuntu-latest
features: ""
target: "i686-unknown-linux-gnu"
gcc: "i686-linux-gnu-gcc"
steps:
- uses: actions/checkout@v2
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: qemu-user qemu-user-static qemu-system-s390x gcc-s390x-linux-gnu gcc-i686-linux-gnu g++-s390x-linux-gnu
version: 1.0
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Copy QEMU Cargo Config
run: |
mkdir -p .cargo
cp qemu-cargo-config.toml .cargo/config.toml
- name: Install toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: "stable"
targets: "${{ matrix.target }}"
- name: target
run: "rustc -vV | sed -n 's|host: ||p'"
- name: Install cargo-nextest
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-nextest
- name: cargo build
run: cargo build --target ${{matrix.target}} ${{ matrix.features }}
- name: cargo nextest # reports segfaults in a helpful way
run: cargo nextest run --target ${{matrix.target}} ${{ matrix.features }}
env:
RUST_BACKTRACE: 1
CC: ${{matrix.gcc}}
clippy:
name: Clippy
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
features:
- ""
- '--no-default-features --features="c-allocator"'
- '--no-default-features --features="rust-allocator"'
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Install rust toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: stable
components: clippy
targets: ${{matrix.target}}
- name: Rust cache
uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8
with:
shared-key: "stable-${{matrix.target}}"
- name: Run clippy
run: cargo clippy --target ${{matrix.target}} --workspace --all-targets -- -D warnings
- name: Run clippy (fuzzers)
run: cargo clippy --target ${{matrix.target}} --manifest-path ./fuzz/Cargo.toml --all-targets -- -D warnings
if: ${{matrix.fuzzer}}
fuzz:
name: Smoke-test fuzzing targets
runs-on: ubuntu-20.04
strategy:
matrix:
features:
- 'default'
- 'c-allocator'
- 'rust-allocator'
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Install nightly toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: nightly
- name: Install cargo fuzz
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-fuzz
- name: Smoke-test fuzz targets
run: |
cargo fuzz build --no-default-features --features="${{ matrix.features }}"
for target in $(cargo fuzz list); do
if [ "$target" = "uncompress2" ]; then
features="${{ matrix.features }} disable-checksum"
else
features="${{ matrix.features }}"
fi
RUST_BACKTRACE=1 cargo fuzz run --no-default-features --features="$features" $target -- -max_total_time=10
done
fuzz-aarch64:
name: Smoke-test fuzzing targets
runs-on: macos-14
strategy:
matrix:
include:
- rust: "stable"
os: macos-14
features: ""
target: "aarch64-apple-darwin"
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Install nightly toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: nightly
- name: Install cargo fuzz
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-fuzz
- name: Smoke-test fuzz targets
run: |
cargo fuzz build --no-default-features --features="${{ matrix.features }}"
for target in $(cargo fuzz list); do
if [ "$target" = "uncompress2" ]; then
features="${{ matrix.features }} disable-checksum"
else
features="${{ matrix.features }}"
fi
RUST_BACKTRACE=1 cargo fuzz run --no-default-features --features="$features" $target -- -max_total_time=10
done
link-c-dynamic-library:
name: vanilla dynamic library
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
features:
- ''
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Install rust toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: stable
targets: ${{matrix.target}}
- name: "cdylib: default settings"
working-directory: libz-rs-sys-cdylib
env:
LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps"
run: |
cargo build --release --target ${{matrix.target}}
cc -o zpipe zpipe.c target/${{matrix.target}}/release/deps/libz_rs.so
./zpipe < Cargo.toml | ./zpipe -d > out.txt
cmp -s Cargo.toml out.txt
- name: "cdylib: rust-allocator"
env:
LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps"
working-directory: libz-rs-sys-cdylib
run: |
cargo build --release --target ${{matrix.target}} --no-default-features --features="rust-allocator"
cc -o zpipe zpipe.c target/${{matrix.target}}/release/deps/libz_rs.so
./zpipe < Cargo.toml | ./zpipe -d > out.txt
cmp -s Cargo.toml out.txt
- name: "cdylib: no_std"
env:
LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps"
working-directory: libz-rs-sys-cdylib
run: |
cargo build --release --target ${{matrix.target}} --no-default-features
cc -DNO_STD -o zpipe zpipe.c target/${{matrix.target}}/release/deps/libz_rs.so
./zpipe < Cargo.toml | ./zpipe -d > out.txt
cmp -s Cargo.toml out.txt
- name: "cdylib: custom-prefix"
working-directory: libz-rs-sys-cdylib
env:
LIBZ_RS_SYS_PREFIX: "MY_CUSTOM_PREFIX_"
run: |
cargo build --release --target ${{matrix.target}} --features=custom-prefix
objdump -tT target/${{matrix.target}}/release/deps/libz_rs.so | grep -q "MY_CUSTOM_PREFIX_uncompress" || (echo "symbol not found!" && exit 1)
cargo-c-dynamic-library:
name: cargo-c dynamic library
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
features:
- ''
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Install rust toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: stable
targets: ${{matrix.target}}
- name: Install cargo-c
env:
LINK: https://github.com/lu-zero/cargo-c/releases/download/v0.10.5
run: |
curl -L "$LINK/cargo-c-x86_64-unknown-linux-musl.tar.gz" |
tar xz -C $HOME/.cargo/bin
- name: build with and test the result of cargo-c
working-directory: libz-rs-sys-cdylib
run: |
# build using cargo-c this time
cargo cinstall --offline --release --destdir=/tmp/cargo-cbuild-libz-rs
tree /tmp/cargo-cbuild-libz-rs
# verify that the SONAME is set and includes a version
objdump -p target/x86_64-unknown-linux-gnu/release/libz_rs.so | awk '/SONAME/{print $2}' | grep -E 'libz_rs\.so\.1'
# build zpipe with our library
cc -o zpipe zpipe.c -L/tmp/cargo-cbuild-libz-rs/usr/local/lib/x86_64-linux-gnu -lz_rs
export LD_LIBRARY_PATH=/tmp/cargo-cbuild-libz-rs/usr/local/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
./zpipe < Cargo.toml | ./zpipe -d > out.txt
cmp -s Cargo.toml out.txt
wasm32:
name: "wasm32"
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: Install wasmtime
run: curl https://wasmtime.dev/install.sh -sSf | bash
- name: Copy .cargo/config.toml
run: |
mkdir -p .cargo
cp qemu-cargo-config.toml .cargo/config.toml
- name: Install toolchain
uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
with:
toolchain: "stable"
targets: "wasm32-wasip1"
- name: target
run: "rustc -vV | sed -n 's|host: ||p'"
- name: Install cargo-nextest
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-nextest
- name: Download wasi-sdk
run: |
curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-linux.tar.gz | tar xz
- name: cargo nextest (without SIMD)
run: |
export CMAKE_TOOLCHAIN_FILE="$(pwd)/wasi-sdk-24.0-x86_64-linux/share/cmake/wasi-sdk.cmake"
cargo nextest run -p zlib-rs -p test-libz-rs-sys --target wasm32-wasip1
env:
RUST_BACKTRACE: 1
RUSTFLAGS: ""
- name: cargo nextest (with SIMD)
run: |
export CMAKE_TOOLCHAIN_FILE="$(pwd)/wasi-sdk-24.0-x86_64-linux/share/cmake/wasi-sdk.cmake"
cargo nextest run -p zlib-rs -p test-libz-rs-sys --target wasm32-wasip1
env:
RUST_BACKTRACE: 1
RUSTFLAGS: "-Ctarget-feature=+simd128"
miri:
name: "Miri"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install Miri
run: |
rustup toolchain install nightly --component miri
cargo +nightly miri setup
- name: Install cargo-nextest
uses: taiki-e/install-action@56ab7930c591507f833cbaed864d201386d518a8
with:
tool: cargo-nextest
- name: Test public C api with NULL arguments
run: "cargo +nightly miri nextest run -j4 -p test-libz-rs-sys null::"
env:
RUSTFLAGS: "-Ctarget-feature=+avx2"
- name: Test allocator with miri
run: "cargo +nightly miri nextest run -j4 -p zlib-rs allocate::"
run-flate2-test-suite:
name: run flate2 test suite
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
features:
- ''
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
persist-credentials: false
- name: run
run: |
cd ..
git clone https://github.com/rust-lang/flate2-rs.git
cd flate2-rs
sed -i '/^libz-rs-sys =/c\libz-rs-sys = { path = "../zlib-rs/libz-rs-sys", optional = true, default-features = false, features = ["std", "rust-allocator"] }' Cargo.toml
cargo test --no-default-features --features="zlib-rs"