From e1c62d7430a047b85f7b37c61a6f0ede34c49cce Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 21 Mar 2024 16:40:44 +0100 Subject: [PATCH] ci: initial addition of upstream testing (#1024) This will give a great speed up. Caveats: - No more binary stripping in PR ci. - Test and build are separated so we upload the binaries even if the tests fail. - Using a reusable workflow for the testing to keep that out of the workflow file. - This now uses `sccache` with the github cache, (Not the bucket like prefix.dev) --- .../{release.yml => post_release.yml} | 0 .github/workflows/rust.yml | 182 +++++++++--------- .github/workflows/test_downstream.yml | 114 +++++++++++ Cargo.lock | 47 ----- Cargo.toml | 5 - 5 files changed, 209 insertions(+), 139 deletions(-) rename .github/workflows/{release.yml => post_release.yml} (100%) create mode 100644 .github/workflows/test_downstream.yml diff --git a/.github/workflows/release.yml b/.github/workflows/post_release.yml similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/post_release.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ee2ebf9c2..bc6ce55ce 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -27,8 +27,9 @@ env: RUSTFLAGS: "-D warnings" CARGO_TERM_COLOR: always CICD_INTERMEDIATES_DIR: "_cicd-intermediates" - TEST_FEATURES: "slow_integration_tests" XDG_CACHE_HOME: ${{ github.workspace }}/.cache + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" jobs: check-rustdoc-links: @@ -36,14 +37,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - uses: actions-rust-lang/setup-rust-toolchain@v1 - run: | for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | .name'); do cargo rustdoc -p "$package" --all-features -- -D warnings -W unreachable-pub done - format_and_lint: - name: Format and Lint + format: + name: Cargo Format runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -51,9 +56,24 @@ jobs: submodules: recursive - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - components: clippy, rustfmt + components: rustfmt - name: Run rustfmt uses: actions-rust-lang/rustfmt@v1 + + lint: + name: Cargo Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: clippy + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: Run clippy run: cargo clippy @@ -75,40 +95,84 @@ jobs: maintainer: ${{ steps.crate_metadata.outputs.maintainer }} homepage: ${{ steps.crate_metadata.outputs.homepage }} + cargo-test: + name: Cargo Test | ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - { name: "Linux", target: x86_64-unknown-linux-musl, os: ubuntu-20.04 } + - { name: "macOS", target: x86_64-apple-darwin, os: macOS-latest } + - { name: "macOS-arm", target: aarch64-apple-darwin, os: macOS-14 } + - { name: "Windows", target: x86_64-pc-windows-msvc, os: windows-latest } + steps: + - uses: actions/checkout@v4 + - uses: taiki-e/setup-cross-toolchain-action@v1 + with: + target: ${{ matrix.target }} +# - uses: Swatinem/rust-cache@v2 + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Show version information (Rust, cargo, GCC) + shell: bash + run: | + gcc --version || true + rustup -V + rustup toolchain list + cargo -V + rustc -V + + - name: "Install cargo nextest" + uses: taiki-e/install-action@v2 + with: + tool: cargo-nextest + + - if: ${{ matrix.os != 'windows-latest' }} + uses: rui314/setup-mold@v1 + + - name: "Cargo nextest" + run: | + cargo nextest run --status-level skip --failure-output immediate-final --no-fail-fast --final-status-level slow --features slow_integration_tests + build: - name: ${{ matrix.name }} + name: Build Binary | ${{ matrix.name }} runs-on: ${{ matrix.os }} - needs: [ crate_metadata, format_and_lint ] + needs: [ crate_metadata ] strategy: fail-fast: false matrix: include: - - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-20.04 } - - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } + - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-20.04 } + - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest } - - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } + - { name: "macOS-x86", target: x86_64-apple-darwin, os: macOS-latest } + - { name: "macOS-arm", target: aarch64-apple-darwin, os: macOS-14 } # macOS-14 is the ARM chipset - - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } + - { name: "Windows", target: x86_64-pc-windows-msvc, os: windows-latest } env: # # These are some environment variables that configure the build so that the binary size is reduced. # Inspiration was taken from this blog: https://arusahni.net/blog/2020/03/optimizing-rust-binary-size.html + # They're only enable it on main and releases. # # Enable Link Time Optimization (LTO) for our release builds. This increases link time but drastically reduces # binary size. - CARGO_PROFILE_RELEASE_LTO: true + CARGO_PROFILE_RELEASE_LTO: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') && 'true' || 'false' }} # Use a single code gen unit, this effectively disables parallel linking but ensures that everything is linked # together in a single unit which reduces the file-size at the cost of link time. - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1 + # Default for a release build is 16 + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') && 1 || 16 }} # Strip the binaries. This reduces the filesize of the final release. - CARGO_PROFILE_RELEASE_STRIP: "symbols" + CARGO_PROFILE_RELEASE_STRIP: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') && 'symbols' || 'false' }} # Optimize the binary for size. This reduces the filesize at the cost of a slower binary. - CARGO_PROFILE_OPT_LEVEL: s + CARGO_PROFILE_OPT_LEVEL: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') && 's' || '0' }} steps: - name: Checkout source code uses: actions/checkout@v4 @@ -128,7 +192,9 @@ jobs: run: echo "RUSTFLAGS=${RUSTFLAGS} -C target-feature=+crt-static" >> "${GITHUB_ENV}" if: endsWith(matrix.target, 'windows-msvc') - - uses: Swatinem/rust-cache@v2 +# - uses: Swatinem/rust-cache@v2 + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 - name: Setup | Install cargo-wix [Windows] continue-on-error: true @@ -193,26 +259,6 @@ jobs: echo "BIN_NAME=${BIN_NAME}" >> $GITHUB_OUTPUT echo "EXE_SUFFIX=${EXE_SUFFIX}" >> $GITHUB_OUTPUT - - name: Set testing options - id: test-options - if: ${{ !matrix.skip-tests }} - shell: bash - run: | - # test only library unit tests and binary for arm-type targets - unset CARGO_TEST_OPTIONS - unset CARGO_TEST_OPTIONS ; case ${{ matrix.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ needs.crate_metadata.outputs.name }}" ;; esac; - CARGO_TEST_OPTIONS="${CARGO_TEST_OPTIONS} --features ${{ env.TEST_FEATURES }}" - echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT - - - name: Run tests - if: ${{ !matrix.skip-tests }} - run: > - cargo test - --locked - --release - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} - ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} - - name: Build Installer continue-on-error: true if: matrix.os == 'windows-latest' && matrix.target != 'aarch64-pc-windows-msvc' @@ -342,60 +388,22 @@ jobs: target/wix/pixi-${{ matrix.target }}.msi env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - test-examples: - name: Integration test ${{ matrix.name }} - runs-on: ${{ matrix.os }} - needs: build + + downstream_tests: + needs: + - build strategy: fail-fast: false matrix: include: - - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-20.04 } -# - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest } Doesn't work yet + - { target: x86_64-unknown-linux-musl, os: ubuntu-20.04 } - - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-14 } # macOS-14 is the ARM chipset + - { target: x86_64-apple-darwin, os: macOS-latest } + - { target: aarch64-apple-darwin, os: macOS-14 } # macOS-14 is the ARM chipset - - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest , extension: .exe} - steps: - - name: checkout repo - uses: actions/checkout@v4 - - name: Download binary from build - uses: actions/download-artifact@v4 - with: - name: pixi-${{ matrix.target }}${{ matrix.extension }} - path: pixi_bin - - name: Debug - run: | - pwd - - name: Setup unix binary, add to github path - if: matrix.os != 'windows-latest' - run: | - mv pixi_bin/pixi-${{ matrix.target }} pixi_bin/pixi - chmod a+x pixi_bin/pixi - echo "$(pwd)/pixi_bin" >> $GITHUB_PATH - - name: Create Directory and Move Executable - if: matrix.os == 'windows-latest' && matrix.target == 'x86_64-pc-windows-msvc' - run: | - New-Item -ItemType Directory -Force -Path "D:\.pixi" - Move-Item -Path "pixi_bin/pixi-${{ matrix.target }}${{ matrix.extension }}" -Destination "D:\.pixi\pixi.exe" - shell: pwsh - - name: Add to PATH - if: matrix.os == 'windows-latest' && matrix.target == 'x86_64-pc-windows-msvc' - run: echo "D:\.pixi" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH - shell: pwsh - - name: Verify and Use Executable - if: matrix.os == 'windows-latest' && matrix.target == 'x86_64-pc-windows-msvc' - run: | - echo "Current PATH: $env:PATH" - pixi --version - shell: pwsh - - name: Help - run: pixi --help - - name: Info - run: pixi info - - name: Install pixi - run: pixi install -v - - name: Test examples - shell: bash - run: bash tests/test_examples.sh + - { target: x86_64-pc-windows-msvc, os: windows-latest , extension: .exe } + uses: ./.github/workflows/test_downstream.yml + with: + os: ${{ matrix.os }} + target: ${{ matrix.target }} + extension: ${{ matrix.extension }} diff --git a/.github/workflows/test_downstream.yml b/.github/workflows/test_downstream.yml new file mode 100644 index 000000000..3ba0a1d0a --- /dev/null +++ b/.github/workflows/test_downstream.yml @@ -0,0 +1,114 @@ +on: + workflow_call: + inputs: + os: + description: "Operating system to test" + required: true + type: string + target: + description: "Target to test" + required: true + type: string + extension: + description: "Extension of the binary" + required: false + type: string + + +jobs: + test-downstream: + name: Downstream test ${{ matrix.repository }} on ${{ matrix.os }} + if: ${{ startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' }} + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + repository: [ "Deltares/Ribasim", "quantco/polarify" ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + repository: ${{ matrix.repository }} + # Setup pixi binary + - name: Download binary from build + uses: actions/download-artifact@v4 + with: + name: pixi-${{ inputs.target }}${{ inputs.extension }} + path: pixi_bin + - name: Setup unix binary, add to github path + if: inputs.os != 'windows-latest' + run: | + mv pixi_bin/pixi-${{ inputs.target }} pixi_bin/pixi + chmod a+x pixi_bin/pixi + echo "$(pwd)/pixi_bin" >> $GITHUB_PATH + - name: Create Directory and Move Executable + if: inputs.os == 'windows-latest' && inputs.target == 'x86_64-pc-windows-msvc' + run: | + New-Item -ItemType Directory -Force -Path "D:\.pixi" + Move-Item -Path "pixi_bin/pixi-${{ inputs.target }}${{ inputs.extension }}" -Destination "D:\.pixi\pixi.exe" + shell: pwsh + - name: Add to PATH + if: inputs.os == 'windows-latest' && inputs.target == 'x86_64-pc-windows-msvc' + run: echo "D:\.pixi" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH + shell: pwsh + - name: Verify and Use Executable + if: inputs.os == 'windows-latest' && inputs.target == 'x86_64-pc-windows-msvc' + run: | + echo "Current PATH: $env:PATH" + pixi --version + shell: pwsh + - name: Help + run: pixi --help + - name: Info + run: pixi info + + - name: Install repository + run: pixi install -v + - name: Check lockfile didn't change + run: git diff --exit-code pixi.lock + + test-examples: + name: Integration test on ${{ inputs.os }} + runs-on: ${{ inputs.os }} + steps: + - name: checkout repo + uses: actions/checkout@v4 + - name: Download binary from build + uses: actions/download-artifact@v4 + with: + name: pixi-${{ inputs.target }}${{ inputs.extension }} + path: pixi_bin + - name: Debug + run: | + pwd + - name: Setup unix binary, add to github path + if: inputs.os != 'windows-latest' + run: | + mv pixi_bin/pixi-${{ inputs.target }} pixi_bin/pixi + chmod a+x pixi_bin/pixi + echo "$(pwd)/pixi_bin" >> $GITHUB_PATH + - name: Create Directory and Move Executable + if: inputs.os == 'windows-latest' && inputs.target == 'x86_64-pc-windows-msvc' + run: | + New-Item -ItemType Directory -Force -Path "D:\.pixi" + Move-Item -Path "pixi_bin/pixi-${{ inputs.target }}${{ inputs.extension }}" -Destination "D:\.pixi\pixi.exe" + shell: pwsh + - name: Add to PATH + if: inputs.os == 'windows-latest' && inputs.target == 'x86_64-pc-windows-msvc' + run: echo "D:\.pixi" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH + shell: pwsh + - name: Verify and Use Executable + if: inputs.os == 'windows-latest' && inputs.target == 'x86_64-pc-windows-msvc' + run: | + echo "Current PATH: $env:PATH" + pixi --version + shell: pwsh + - name: Help + run: pixi --help + - name: Info + run: pixi info + - name: Install pixi + run: pixi install -v + - name: Test examples + shell: bash + run: bash tests/test_examples.sh diff --git a/Cargo.lock b/Cargo.lock index 75d8eb761..a92cc5a91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,18 +139,6 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" - [[package]] name = "assert_matches" version = "1.5.0" @@ -317,17 +305,6 @@ dependencies = [ "syn 2.0.53", ] -[[package]] -name = "async-scoped" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4042078ea593edffc452eef14e99fdb2b120caa4ad9618bcdeabc4a023b98740" -dependencies = [ - "futures", - "pin-project", - "tokio", -] - [[package]] name = "async-signal" version = "0.2.5" @@ -493,19 +470,6 @@ dependencies = [ "digest", ] -[[package]] -name = "blake3" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -864,12 +828,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "constant_time_eq" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" - [[package]] name = "core-foundation" version = "0.9.4" @@ -3049,9 +3007,6 @@ version = "0.17.0" dependencies = [ "assert_matches", "async-once-cell", - "async-recursion", - "async-scoped", - "blake3", "cfg-if", "chrono", "clap", @@ -3067,7 +3022,6 @@ dependencies = [ "dunce", "flate2", "futures", - "globset", "http-cache-reqwest", "human_bytes", "humantime", @@ -3107,7 +3061,6 @@ dependencies = [ "serde", "serde-untagged", "serde_json", - "serde_spanned", "serde_with", "serde_yaml", "serial_test", diff --git a/Cargo.toml b/Cargo.toml index 323d543df..370b30be0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,6 @@ slow_integration_tests = [] [dependencies] assert_matches = "1.5.0" async-once-cell = "0.5.3" -async-recursion = "1.1.0" -async-scoped = { version = "0.9.0", features = ["use-tokio"] } -blake3 = "1.5.1" cfg-if = "1.0" chrono = "0.4.35" clap = { version = "4.5.3", default-features = false, features = [ @@ -53,7 +50,6 @@ distribution-types = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" } dunce = "1.0.4" flate2 = "1.0.28" futures = "0.3.30" -globset = "0.4.14" http-cache-reqwest = "0.13.0" human_bytes = "0.4.3" humantime = "2.1.0" @@ -103,7 +99,6 @@ self-replace = "1.3.7" serde = "1.0.197" serde-untagged = "0.1.5" serde_json = "1.0.114" -serde_spanned = "0.6.5" serde_with = { version = "3.7.0", features = ["indexmap"] } serde_yaml = "0.9.33" shlex = "1.3.0"