diff --git a/.github/workflows/merge-pr.yml b/.github/workflows/merge-pr.yml index 16e8c81246..5566b6f3d6 100644 --- a/.github/workflows/merge-pr.yml +++ b/.github/workflows/merge-pr.yml @@ -1,4 +1,4 @@ -name: Run Post PR Merge Actions +name: Merge PR concurrency: group: ${{github.workflow}}-${{github.ref}} cancel-in-progress: true @@ -20,18 +20,19 @@ jobs: env: HOME: /root runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: + - name: Install Required Packages + run: | + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev - name: Check Out Repo uses: actions/checkout@v3 - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: targets: wasm32-unknown-unknown toolchain: stable - - name: Check Out Repo - uses: actions/checkout@v3 - name: Output Metadata # Run the cargo command and ignore any extra lines outside of the json result run: CARGO_INCREMENTAL=0 RUSTFLAGS="-D warnings" cargo run --features frequency-rococo-local -- export-metadata --chain=frequency-local --tmp ./js/api-augment/metadata.json @@ -51,43 +52,45 @@ jobs: env: FULL_SHA: ${{github.sha}} - name: Version Package - run: npm version --new-version "v0.0.0-${FULL_SHA:0:6}" --no-git-tag-version - working-directory: js/api-augment/dist env: FULL_SHA: ${{github.sha}} + working-directory: js/api-augment/dist + shell: bash + run: npm version --new-version "v0.0.0-${FULL_SHA:0:6}" --no-git-tag-version - name: Publish on NPM @next run: npm publish --tag next --access public working-directory: js/api-augment/dist env: NODE_AUTH_TOKEN: ${{secrets.NODE_AUTH_TOKEN}} - calc-code-coverage-main: + calc-code-coverage: name: Merge - Calculate Code Coverage - runs-on: ubuntu-20.04 + runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - - name: Check Out Repo - uses: actions/checkout@v3 - name: Install Required Packages run: | - sudo apt-get update - sudo apt install -y protobuf-compiler + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev git + git --version + - name: Check Out Repo + uses: actions/checkout@v3 - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: targets: wasm32-unknown-unknown toolchain: stable - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@044a1e5bdace8dd2f727b1af63c1d9a1d3572068 - with: - version: "0.22.0" - # Tarpaulin Docs https://github.com/xd009642/tarpaulin - # -e for exclude - args: | - -v --no-fail-fast --workspace - -e frequency frequency-cli frequency-runtime frequency-service - --exclude-files **/mock.rs **/weights.rs **/weights/* **/benchmarking.rs + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Generate code coverage + run: | + cargo llvm-cov -v --no-fail-fast --workspace --lcov --output-path lcov.info \ + --ignore-filename-regex "^.*\/(mock\.rs|weights(\.rs)?|benchmarking\.rs).*$" \ + --exclude "frequency,frequency-cli,frequency-runtime,frequency-service" \ + --features all-frequency-features - name: Upload to codecov.io uses: codecov/codecov-action@v3 with: + files: lcov.info fail_ci_if_error: false # optional (default = false) verbose: true # optional (default = false) diff --git a/.github/workflows/verify-pr-commit.yml b/.github/workflows/verify-pr-commit.yml index 5e828e09a2..551a17af08 100644 --- a/.github/workflows/verify-pr-commit.yml +++ b/.github/workflows/verify-pr-commit.yml @@ -12,6 +12,7 @@ jobs: changes: name: Determine Changed Files runs-on: ubuntu-20.04 + container: ubuntu:20.04 outputs: rust: ${{steps.filter.outputs.rust}} build-binary: ${{steps.filter.outputs.build-binary}} @@ -31,6 +32,7 @@ jobs: - '.rustfmt.toml' - '**/Cargo.toml' - '**/Cargo.lock' + - '.github/**/*.yml' build-binary: - '**/*.rs' - '**/*.hbs' @@ -39,13 +41,16 @@ jobs: - '**/Cargo.lock' - 'js/api-augment/**' - 'integration-tests/**/*.{ts,json}' + - '.github/**/*.yml' cargo-lock: - '**/Cargo.toml' - '**/Cargo.lock' + - '.github/**/*.yml' run-integration-tests: - '**/*.rs' - '**/Cargo.toml' - 'integration-tests/**/*.{ts,json}' + - '.github/**/*.yml' # Workaround to handle skipped required check inside matrix # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks @@ -53,6 +58,7 @@ jobs: needs: changes if: needs.changes.outputs.build-binary != 'true' runs-on: ubuntu-20.04 + container: ubuntu:20.04 name: Build ${{matrix.network}} Binary on ${{matrix.branch_alias}} Branch strategy: matrix: @@ -100,26 +106,19 @@ jobs: spec: frequency branch_alias: pr runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 env: - HOME: /root + # HOME: /root NETWORK: mainnet steps: - # https://github.com/LibertyDSNP/frequency/issues/1102 - - name: Debug Leftovers + - name: Install Required Packages run: | - set -x - date - whoami - cat /etc/passwd - [ -d ./target ] && sudo ls -latR ./target - [ -d ./runtime/frequency/target ] && sudo ls -latR ./runtime/frequency/target - exit 0 + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev file - name: Check Out Repo uses: actions/checkout@v3 with: ref: ${{matrix.git_branch}} - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - name: Set Env Vars run: | export BUILT_BIN_FILENAME=frequency; echo "BUILT_BIN_FILENAME=$BUILT_BIN_FILENAME" >> $GITHUB_ENV @@ -131,10 +130,6 @@ jobs: # with: # path: ${{env.BIN_DIR}}/${{env.FINAL_BIN_FILENAME}} # key: binaries-${{runner.os}}-${{env.NETWORK}}-${{github.head_ref}} - - name: Install Required Packages - run: | - sudo apt-get update - sudo apt install -y protobuf-compiler - name: Install Rust Toolchain if: steps.cache-binary.outputs.cache-hit != 'true' uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 @@ -168,9 +163,19 @@ jobs: if: needs.changes.outputs.cargo-lock == 'true' name: Check for Vulnerable Crates runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: + - name: Install Required Packages + run: | + apt-get update + apt-get install -y curl build-essential - name: Check Out Repo uses: actions/checkout@v3 + - name: Install Rust Toolchain + uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 + with: + components: rustfmt + toolchain: stable - name: Set Up Cargo Deny run: | cargo install --force cargo-deny @@ -183,7 +188,15 @@ jobs: if: needs.changes.outputs.rust == 'true' name: Verify Rust Code Format runs-on: ubuntu-20.04 + container: + image: ubuntu:20.04 + # options: --user runner steps: + - name: Install Required Packages + run: | + set -x + apt-get update + apt-get install -y curl - name: Check Out Repo uses: actions/checkout@v3 - name: Install Rust Toolchain @@ -198,28 +211,17 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' name: Lint Rust Code - env: - HOME: /root + # env: + # HOME: /root runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - # https://github.com/LibertyDSNP/frequency/issues/1102 - - name: Debug Leftovers + - name: Install Required Packages run: | - set -x - date - whoami - cat /etc/passwd - [ -d ./target ] && sudo ls -latR ./target - [ -d ./runtime/frequency/target ] && sudo ls -latR ./runtime/frequency/target - exit 0 + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev - name: Check Out Repo uses: actions/checkout@v3 - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - - name: Install Required Packages - run: | - sudo apt-get update - sudo apt install -y protobuf-compiler - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: @@ -236,28 +238,17 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' name: Verify Rust Developer Docs - env: - HOME: /root + # env: + # HOME: /root runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - # https://github.com/LibertyDSNP/frequency/issues/1102 - - name: Debug Leftovers + - name: Install Required Packages run: | - set -x - date - whoami - cat /etc/passwd - [ -d ./target ] && sudo ls -latR ./target - [ -d ./runtime/frequency/target ] && sudo ls -latR ./runtime/frequency/target - exit 0 + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev - name: Check Out Repo uses: actions/checkout@v3 - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - - name: Install Required Packages - run: | - sudo apt-get update - sudo apt install -y protobuf-compiler - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: @@ -270,29 +261,17 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' name: Verify Rust Packages and Dependencies - env: - HOME: /root + # env: + # HOME: /root runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - # https://github.com/LibertyDSNP/frequency/issues/1102 - - name: Debug Leftovers + - name: Install Required Packages run: | - set -x - set -x - date - whoami - cat /etc/passwd - [ -d ./target ] && sudo ls -latR ./target - [ -d ./runtime/frequency/target ] && sudo ls -latR ./runtime/frequency/target - exit 0 + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev - name: Check Out Repo uses: actions/checkout@v3 - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - - name: Install Required Packages - run: | - sudo apt-get update - sudo apt install -y protobuf-compiler - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: @@ -304,28 +283,17 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' name: Run Rust Tests - env: - HOME: /root + # env: + # HOME: /root runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - # https://github.com/LibertyDSNP/frequency/issues/1102 - - name: Debug Leftovers + - name: Install Required Packages run: | - set -x - date - whoami - cat /etc/passwd - [ -d ./target ] && sudo ls -latR ./target - [ -d ./runtime/frequency/target ] && sudo ls -latR ./runtime/frequency/target - exit 0 + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev - name: Check Out Repo uses: actions/checkout@v3 - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - - name: Install Required Packages - run: | - sudo apt-get update - sudo apt install -y protobuf-compiler - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: @@ -338,32 +306,33 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' name: Calculate Code Coverage - runs-on: ubuntu-20.04 + runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - - name: Check Out Repo - uses: actions/checkout@v3 - name: Install Required Packages run: | - sudo apt-get update - sudo apt install -y protobuf-compiler + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev git + git --version + - name: Check Out Repo + uses: actions/checkout@v3 - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: targets: wasm32-unknown-unknown toolchain: stable - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@044a1e5bdace8dd2f727b1af63c1d9a1d3572068 - with: - version: "0.22.0" - # Tarpaulin Docs https://github.com/xd009642/tarpaulin - # -e for exclude - args: | - -v --no-fail-fast --workspace - -e frequency frequency-cli frequency-runtime frequency-service - --exclude-files **/mock.rs **/weights.rs **/weights/* **/benchmarking.rs + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Generate code coverage + run: | + cargo llvm-cov -v --no-fail-fast --workspace --lcov --output-path lcov.info \ + --ignore-filename-regex "^.*\/(mock\.rs|weights(\.rs)?|benchmarking\.rs).*$" \ + --exclude "frequency,frequency-cli,frequency-runtime,frequency-service" \ + --features all-frequency-features - name: Upload to codecov.io uses: codecov/codecov-action@v3 with: + files: lcov.info fail_ci_if_error: false # optional (default = false) verbose: true # optional (default = false) @@ -377,6 +346,7 @@ jobs: matrix: network: [rococo, mainnet] runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: - run: echo "Just a dummy matrix to satisfy GitHub required checks that were skipped" @@ -384,8 +354,6 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' name: Verify Build Runtime for ${{matrix.network}} - env: - HOME: /root strategy: fail-fast: true matrix: @@ -402,37 +370,29 @@ jobs: runtime-dir: runtime/frequency features: frequency runs-on: [self-hosted, Linux, X64, build] + container: ubuntu:20.04 steps: - # https://github.com/LibertyDSNP/frequency/issues/1102 - - name: Debug Leftovers + - name: Install Required Packages run: | - set -x - date - whoami - cat /etc/passwd - [ -d ./target ] && sudo ls -latR ./target - [ -d ./runtime/frequency/target ] && sudo ls -latR ./runtime/frequency/target - exit 0 + apt-get update + apt-get install -y curl protobuf-compiler build-essential libclang-dev file + curl -fsSL https://get.docker.com -o get-docker.sh + sh get-docker.sh + docker run hello-world - name: Check Out Repo uses: actions/checkout@v3 - - name: Reset Runner - uses: ./.github/workflows/common/reset-self-hosted-runner - name: Install Rust Toolchain uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 with: toolchain: stable - - name: Install Docker - run: | - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh - name: Extract Runtime Spec Version run: | echo "RUNTIME_SPEC_VERSION=$(sed -nr 's/spec_version:\s*([0-9]+),/\1/p' \ ${{matrix.runtime-dir}}/src/lib.rs | tr -d '[:space:]')" >> $GITHUB_ENV - name: Validate Extracted Version + shell: bash run: | - echo "Runtime Spec Version:|${{env.RUNTIME_SPEC_VERSION}}|" - echo "Runtime Spec Version:||$RUNTIME_SPEC_VERSION||" + echo "Runtime Spec Version:${{env.RUNTIME_SPEC_VERSION}}" [[ $RUNTIME_SPEC_VERSION == ?(-)+([0-9]) ]] || \ (echo "ERROR: \"${{env.RUNTIME_SPEC_VERSION}}\" is not a valid integer" && exit 1) - name: Set Env Vars @@ -464,6 +424,7 @@ jobs: needs: build-binaries name: Verify JS API Augment runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: - name: Check Out Repo uses: actions/checkout@v3 @@ -516,7 +477,15 @@ jobs: needs: build-binaries name: Verify Docker Images runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: + - name: Install Required Packages + run: | + apt-get update + apt-get install -y curl + curl -fsSL https://get.docker.com -o get-docker.sh + sh get-docker.sh + docker run hello-world - name: Check Out Repo uses: actions/checkout@v3 - name: Set Env Vars @@ -562,16 +531,22 @@ jobs: needs: build-binaries name: Execute Binary Checks runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: + - name: Install Required Packages + run: | + apt-get update + apt-get install -y curl file - name: Check Out Repo uses: actions/checkout@v3 - name: Set Env Vars run: | echo "TEST_BIN_FILENAME=frequency.mainnet-pr" >> $GITHUB_ENV - - name: Install Rust Toolchain - uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 - with: - toolchain: stable + # # XXX Do I need to install toolchain? + # - name: Install Rust Toolchain + # uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295 + # with: + # toolchain: stable - name: Download Binaries uses: actions/download-artifact@v3 with: @@ -590,9 +565,14 @@ jobs: needs: build-binaries name: Check Metadata and Spec Version runs-on: ubuntu-20.04 + container: ubuntu:20.04 env: GITHUB_PR_LABEL: metadata-mismatch steps: + - name: Install Required Packages + run: | + apt-get update + apt-get install -y jq - name: Check Out Repo uses: actions/checkout@v3 with: @@ -615,11 +595,12 @@ jobs: id: compare-metadata working-directory: ${{env.BIN_DIR}} run: | + set -x ./$REF_BIN_FILENAME export-metadata --chain=frequency-local --tmp metadata-ref.json metadata_ref=$(cat metadata-ref.json | jq -r .result) ./$TEST_BIN_FILENAME export-metadata --chain=frequency-local --tmp metadata.json metadata=$(cat metadata.json | jq -r .result) - match=$([[ $metadata == $metadata_ref ]] && echo 'true' || echo 'false') + match=$([ "$metadata" = "$metadata_ref" ] && echo 'true' || echo 'false') echo "Metadata matches?: $match" echo "metadata_match=$match" >> $GITHUB_OUTPUT - name: Assign Metadata Mismatch Label @@ -633,7 +614,7 @@ jobs: run: | spec_version_ref=$(./$REF_BIN_FILENAME export-runtime-version | jq -r .specVersion) spec_version=$(./$TEST_BIN_FILENAME export-runtime-version | jq -r .specVersion) - [[ $spec_version -gt $spec_version_ref ]] || \ + [ $spec_version -gt $spec_version_ref ] || \ (echo "ERROR: When metadata is updated, the new spec version \($spec_version\) \ must be greater than the latest version on main \($spec_version_ref\)" && exit 1) - name: Remove Metadata Mismatch Label @@ -649,6 +630,7 @@ jobs: needs: [build-binaries, verify-js-api-augment] name: Run Integration Tests runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: - name: Check Out Repo uses: actions/checkout@v3 @@ -716,6 +698,7 @@ jobs: needs: build-binaries name: Verify Genesis State runs-on: ubuntu-20.04 + container: ubuntu:20.04 steps: - name: Set Env Vars run: | @@ -740,7 +723,7 @@ jobs: echo "Expected genesis state: $expected_genesis_state" actual_genesis_state=$(./${{env.BIN_FILENAME_ROCOCO}} export-genesis-state) echo "Actual genesis state: $actual_genesis_state" - [[ $actual_genesis_state == $expected_genesis_state ]] || \ + [ $actual_genesis_state = $expected_genesis_state ] || \ (echo "ERROR: The actual genesis state does not match the expected" && exit 1) - name: Test Mainnet Genesis State working-directory: ${{env.BIN_DIR}} @@ -749,5 +732,5 @@ jobs: echo "Expected genesis state: $expected_genesis_state" actual_genesis_state=$(./${{env.BIN_FILENAME_MAINNET}} export-genesis-state) echo "Actual genesis state: $actual_genesis_state" - [[ $actual_genesis_state == $expected_genesis_state ]] || \ + [ $actual_genesis_state = $expected_genesis_state ] || \ (echo "ERROR: The actual genesis state does not match the expected" && exit 1)