Optimize window cache building for ibd (#576) #3341
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: [push, pull_request] | |
| jobs: | |
| check: | |
| name: Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Run cargo check | |
| run: cargo check --tests --workspace --benches | |
| test: | |
| name: Test Suite | |
| # runs-on: ${{ matrix.os }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| # matrix: | |
| # os: [ ubuntu-latest, macos-latest, windows-latest ] | |
| steps: | |
| # Clean unnecessary files to save disk space | |
| - name: clean unnecessary files to save space | |
| run: | | |
| docker rmi `docker images -q` | |
| sudo rm -rf /usr/share/dotnet /etc/mysql /etc/php /etc/sudo apt/sources.list.d | |
| sudo apt -y autoremove --purge | |
| sudo apt -y autoclean | |
| sudo apt clean | |
| rm --recursive --force "$AGENT_TOOLSDIRECTORY" | |
| df -h | |
| # remove large packages manually (all but llvm) | |
| sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..." | |
| sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..." | |
| sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..." | |
| df -h | |
| # Free up disk space on Ubuntu | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| # This might remove tools that are actually needed, if set to "true" but frees about 6 GB | |
| tool-cache: false | |
| # large packages, except llvm, are removed manually during the previous step | |
| # see: https://github.com/jlumbroso/free-disk-space/issues/6 | |
| # TODO: use the discussed whitelist feature when available | |
| large-packages: false | |
| swap-storage: true | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Fix CRLF on Windows | |
| if: runner.os == 'Windows' | |
| run: git config --global core.autocrlf false | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Set up cache | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install cargo-nextest | |
| run: cargo install cargo-nextest | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Run cargo build with devnet-prealloc feature | |
| run: cargo build --release --features devnet-prealloc --workspace --all --tests --benches | |
| - name: Run cargo test regular features | |
| run: cargo nextest run --release --workspace | |
| - name: Run cargo doc tests | |
| run: cargo test --doc --release --workspace | |
| - name: Run cargo test on kaspa-hashes without asm | |
| run: cargo nextest run --release -p kaspa-hashes --features=no-asm --benches | |
| - name: Run cargo doc tests with features=no-asm on kaspa-hashes | |
| run: cargo test --doc --release -p kaspa-hashes --features=no-asm | |
| - name: Run cargo doc | |
| run: cargo doc --release --no-deps --workspace | |
| - name: Run kip-10 example | |
| run: cargo run --example kip-10 | |
| # test-release: | |
| # name: Test Suite Release | |
| # runs-on: ${{ matrix.os }} | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # os: [ ubuntu-latest, macos-latest, windows-latest ] | |
| # steps: | |
| # - name: Checkout sources | |
| # uses: actions/checkout@v2 | |
| # - name: Fix CRLF on Windows | |
| # if: runner.os == 'Windows' | |
| # run: git config --global core.autocrlf false | |
| # - name: Install protoc | |
| # run: sudo apt install -y protobuf-compiler | |
| # - name: Install stable toolchain | |
| # uses: actions-rs/toolchain@v1 | |
| # with: | |
| # profile: minimal | |
| # toolchain: stable | |
| # override: true | |
| # - name: Cache | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: | | |
| # ~/.cargo/bin/ | |
| # ~/.cargo/registry/index/ | |
| # ~/.cargo/registry/cache/ | |
| # ~/.cargo/git/db/ | |
| # target/ | |
| # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| # - name: Run cargo test release regular features | |
| # uses: actions-rs/cargo@v1 | |
| # with: | |
| # command: test | |
| # args: --release | |
| # - name: Run cargo test release on hashes without asm | |
| # uses: actions-rs/cargo@v1 | |
| # with: | |
| # command: test | |
| # args: -p hashes --features=no-asm --benches --release | |
| lints: | |
| name: Lints | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Run cargo fmt | |
| run: cargo fmt --all -- --check | |
| - name: Run cargo clippy | |
| run: cargo clippy --workspace --tests --benches --examples -- -D warnings | |
| check-wasm32: | |
| name: Check WASM32 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install llvm | |
| id: install_llvm | |
| continue-on-error: true | |
| run: | | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo apt-get install -y clang-15 lldb-15 lld-15 clangd-15 clang-tidy-15 clang-format-15 clang-tools-15 llvm-15-dev lld-15 lldb-15 llvm-15-tools libomp-15-dev libc++-15-dev libc++abi-15-dev libclang-common-15-dev libclang-15-dev libclang-cpp15-dev libunwind-15-dev | |
| # Make Clang 15 default | |
| sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-15/bin/clang++ 100 | |
| sudo update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-15/bin/clang 100 | |
| sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/lib/llvm-15/bin/clang-format 100 | |
| sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/lib/llvm-15/bin/clang-tidy 100 | |
| sudo update-alternatives --install /usr/bin/run-clang-tidy run-clang-tidy /usr/lib/llvm-15/bin/run-clang-tidy 100 | |
| # Alias cc to clang | |
| sudo update-alternatives --install /usr/bin/cc cc /usr/lib/llvm-15/bin/clang 0 | |
| sudo update-alternatives --install /usr/bin/c++ c++ /usr/lib/llvm-15/bin/clang++ 0 | |
| - name: Install gcc-multilib | |
| # gcc-multilib allows clang to find gnu libraries properly | |
| run: | | |
| sudo apt-get update | |
| sudo apt install -y gcc-multilib | |
| - name: Install stable toolchain | |
| if: steps.install_llvm.outcome == 'success' && steps.install_llvm.conclusion == 'success' | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Add wasm32 target | |
| run: rustup target add wasm32-unknown-unknown | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| # append here any new wasm32 crate not already covered by the existing checks | |
| - name: Run cargo check of kaspa-wrpc-wasm for wasm32 target | |
| run: cargo clippy -p kaspa-wrpc-wasm --target wasm32-unknown-unknown | |
| - name: Run cargo check of kaspa-wallet-cli-wasm for wasm32 target | |
| run: cargo clippy -p kaspa-wallet-cli-wasm --target wasm32-unknown-unknown | |
| - name: Run cargo check of kaspa-wasm for wasm32 target | |
| run: cargo clippy -p kaspa-wasm --target wasm32-unknown-unknown | |
| build-wasm32: | |
| name: Build WASM32 SDK | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Setup Environment | |
| shell: bash | |
| run: echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install llvm | |
| id: install_llvm | |
| continue-on-error: true | |
| run: | | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo apt-get install -y clang-15 lldb-15 lld-15 clangd-15 clang-tidy-15 clang-format-15 clang-tools-15 llvm-15-dev lld-15 lldb-15 llvm-15-tools libomp-15-dev libc++-15-dev libc++abi-15-dev libclang-common-15-dev libclang-15-dev libclang-cpp15-dev libunwind-15-dev | |
| # Make Clang 15 default | |
| sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-15/bin/clang++ 100 | |
| sudo update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-15/bin/clang 100 | |
| sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/lib/llvm-15/bin/clang-format 100 | |
| sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/lib/llvm-15/bin/clang-tidy 100 | |
| sudo update-alternatives --install /usr/bin/run-clang-tidy run-clang-tidy /usr/lib/llvm-15/bin/run-clang-tidy 100 | |
| # Alias cc to clang | |
| sudo update-alternatives --install /usr/bin/cc cc /usr/lib/llvm-15/bin/clang 0 | |
| sudo update-alternatives --install /usr/bin/c++ c++ /usr/lib/llvm-15/bin/clang++ 0 | |
| - name: Install gcc-multilib | |
| # gcc-multilib allows clang to find gnu libraries properly | |
| run: | | |
| sudo apt-get update | |
| sudo apt install -y gcc-multilib | |
| - name: Install stable toolchain | |
| if: steps.install_llvm.outcome == 'success' && steps.install_llvm.conclusion == 'success' | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install wasm-pack | |
| run: cargo install wasm-pack | |
| - name: Add wasm32 target | |
| run: rustup target add wasm32-unknown-unknown | |
| - name: Install NodeJS | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Install NodeJS dependencies | |
| run: npm install --global typedoc typescript | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build wasm release | |
| run: | | |
| pushd . | |
| cd wasm | |
| bash build-release | |
| popd | |
| mv wasm/release/kaspa-wasm32-sdk.zip wasm/release/kaspa-wasm32-sdk-${{ env.SHORT_SHA }}.zip | |
| - name: Upload WASM build to GitHub | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kaspa-wasm32-sdk-${{ env.SHORT_SHA }}.zip | |
| path: wasm/release/kaspa-wasm32-sdk-${{ env.SHORT_SHA }}.zip | |
| build-release: | |
| name: Build Linux Release | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Cargo Build Outputs | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Cache Toolchain | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/x-tools | |
| key: ${{ runner.os }}-musl-${{ hashFiles('**/musl-toolchain/preset.sh') }} | |
| restore-keys: | | |
| ${{ runner.os }}-musl- | |
| - name: Build RK with musl toolchain | |
| if: runner.os == 'Linux' | |
| run: | | |
| # Run build script for musl toolchain | |
| source musl-toolchain/build.sh | |
| # Build for musl | |
| cargo --verbose build --bin kaspad --bin rothschild --bin kaspa-wallet --release --target x86_64-unknown-linux-musl |