Update CI (#552)

* chop steps into different jobs

* fix https://github.com/pendulum-chain/spacewalk/actions/runs/10897911816

* fix https://github.com/pendulum-chain/spacewalk/actions/runs/10897974797/job/30240327947?pr=552

* add other installations

* add other installations

* require secrets and specify shell

* add secrets

* remove secret from the top

* passing parameters to actions

* add shell

* add shell

* add protoc for the tests and clippy

* add back the checkout for run tests job

* remove unused dependencies

* revert the actions

* cargo fmt, fix clippy, and teat

* use nightly for running tests

* let's skip the build first, since we know it'll pass

* let's skip the build first,

* use original cargo toml

* use the `cmd-all` for performing the tests

* skip running tests for rpc libraries; they don't have test cases

* break down the tests even more:
actions/runner-images#10401

* rename test names in the work flow

* fix clippy

* fix error:
https://github.com/pendulum-chain/spacewalk/actions/runs/10943093849/job/30384114712

* re-enable build check

* testing clients need nightly version of rust

* add the environments

* clean up the scripts

* run vault test in a different job

* update version to node 20

* only `integration-test`

* only `integration-test`

* clean up vault and test

* install protoc

* use ubuntu 20.04

* tauri-apps/tauri#9005

* install necessary libs

* install necessary libs

* "uses"

* remove codec error

* install libpq-dev

* switch to ubuntu-latest

* last attempt

* run test

* run whole thing

* run 4 tests in parallel

* shorten CI

* #552 (comment),
#552 (comment)

* fix cargo fmt issues

.github/actions/free-up-disk-space/action.yml

@@ -0,0 +1,33 @@
+name: 'Disk Space'
+runs:
+  using: "composite"
+  steps:
+    - name: Freeing up more disk space
+      run: |
+        sudo swapoff -a
+        sudo rm -f /mnt/swapfile
+        free -h
+        docker rmi $(docker image ls -aq)
+        sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
+        sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /usr/local/share/boost
+        sudo rm -rf /opt/hostedtoolcache
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        sudo apt-get remove -y 'php.*' --fix-missing
+        sudo apt-get remove -y '^mongodb-.*' --fix-missing
+        sudo apt-get remove -y '^mysql-.*' --fix-missing
+        sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing
+        sudo apt-get remove -y google-cloud-sdk --fix-missing
+        sudo apt-get remove -y google-cloud-cli --fix-missing
+        sudo apt-get autoremove -y
+        sudo apt-get clean          
+        df -h
+      shell: bash
+
+    # Free GitHub Actions Environment Disk Space
+    - name: Maximize Build Space
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        large-packages: false

.github/actions/install-rust-nightly/action.yml

@@ -0,0 +1,29 @@
+name: 'Rust Nightly'
+
+inputs:
+  version:
+    description: 'Nightly Version'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Remove rust-toolchain.toml for nightly
+      shell: bash
+      # To make sure that the nightly version will be used all throughout
+      run: |
+        rm /home/runner/work/spacewalk/spacewalk/rust-toolchain.toml
+
+    - name: Setup nightly Rust toolchain
+      uses: actions-rs/toolchain@v1
+#      uses: dtolnay/rust-toolchain@nightly
+      with:
+        toolchain: ${{ inputs.version }}
+        components: rustfmt, clippy
+        target: wasm32-unknown-unknown
+        override: true
+
+#    - name: Setup nightly Rust as default
+#      shell: bash
+#      run: |
+#        rustup default ${{ inputs.version }}

.github/actions/installations/action.yml

@@ -0,0 +1,29 @@
+name: 'Other Necessary Installations'
+
+inputs:
+  token:
+    description: 'A Github PAT'
+    required: true
+
+runs:
+    using: "composite"
+    steps:
+      - name: Install package
+        shell: bash
+        run: |
+          sudo docker image prune --all --force
+          echo 'APT::Get::Always-Include-Phased-Updates "false";' | sudo tee /etc/apt/apt.conf.d/99-phased-updates
+          sudo apt-get update && sudo apt-get upgrade -y
+          sudo apt-get install -y protobuf-compiler libprotobuf-dev
+
+#      - name: Use Cache
+#        uses: Swatinem/rust-cache@v2
+#        with:
+#          cache-on-failure: true
+#          key: "ubuntu-latest-cargo-${{ hashFiles('**/Cargo.lock') }}"
+#          shared-key: "shared"
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ inputs.token }}

.github/actions/prerequisite-nightly/action.yml

@@ -0,0 +1,23 @@
+name: 'Prerequisite Steps'
+description: 'Prerequite steps for every workflow: Free up some space and Install Necessary Packages'
+
+inputs:
+  token:
+    description: 'A Github PAT'
+    required: true
+  version:
+    description: 'Nightly Version'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Free up disk space and Install Necessary Packages
+      uses: ./.github/actions/prerequisite
+      with:
+        token: ${{ inputs.token }}
+
+    - name: Install Rust Nightly
+      uses: ./.github/actions/install-rust-nightly
+      with:
+        version: ${{ inputs.version }}

.github/actions/prerequisite/action.yml

@@ -0,0 +1,29 @@
+name: 'Prerequisite Steps'
+description: 'Prerequite steps for every workflow: Free up some space and Install Necessary Packages'
+
+inputs:
+  token:
+    description: 'A Github PAT'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    # Free GitHub Actions Environment Disk Space
+    - name: Maximize Build Space
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        large-packages: false
+
+    - name: Install Packages
+      uses: ./.github/actions/installations
+      with:
+        token: ${{ inputs.token }}
+
+#    - name: Use cargo.toml with polkadot-sdk fork
+#      shell: bash
+#      # To make sure that our polkadot-sdk fork will be used
+#      run: |
+#        rm /home/runner/work/spacewalk/spacewalk/Cargo.toml
+#        mv /home/runner/work/spacewalk/spacewalk/Cargo_CI.toml /home/runner/work/spacewalk/spacewalk/Cargo.toml

.github/workflows/ci-main.yml

@@ -4,119 +4,120 @@ on:
       - 'main'
 
 name: continuous-integration-main
+env:
+  RUSTUP_NIGHTLY_VERSION: nightly-2024-02-09
+  RUST_BACKTRACE: full
+  # Make sure CI fails on all warnings, including Clippy lints
+  RUSTFLAGS: "-Dwarnings"
+
 jobs:
-  ci:
-    strategy:
-      max-parallel: 2
-      matrix:
-        rust: [ stable, nightly ]
+  check-code:
+    name: Check Code
     runs-on: ubuntu-latest
-    env:
-      RUST_BACKTRACE: full
-      # Make sure CI fails on all warnings, including Clippy lints
-      RUSTFLAGS: "-Dwarnings"
 
     steps:
-      - name: Freeing up more disk space
-        run: |
-          free -h
-          sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
-          sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /usr/local/share/boost
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          sudo apt-get remove -y 'php.*' --fix-missing
-          sudo apt-get remove -y '^mongodb-.*' --fix-missing
-          sudo apt-get remove -y '^mysql-.*' --fix-missing
-          sudo apt-get clean          
-          df -h
-      - uses: actions/checkout@v2
-
-      - name: Install package
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Undergo Prerequisite
+        uses: ./.github/actions/prerequisite
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check Build
         run: |
-          sudo docker image prune --all --force
-          echo 'APT::Get::Always-Include-Phased-Updates "false";' | sudo tee /etc/apt/apt.conf.d/99-phased-updates
-          sudo apt-get update && sudo apt-get upgrade -y
-          sudo apt-get install -y protobuf-compiler libprotobuf-dev
-
-      - name: Setup stable Rust toolchain
-        if: matrix.rust == 'stable'
-        # Call `rustup show` as a hack so that the toolchain defined in rust-toolchain.toml is installed
-        run: rustup show
-
-      - name: Use cargo.toml with polkadot-sdk fork
-        if: matrix.rust == 'nightly'
-        # To make sure that our polkadot-sdk fork will be used
+          bash ./scripts/cmd-all build check "--release"
+          
+      - name: Clippy -- Libraries and Binaries
         run: |
-          rm /home/runner/work/spacewalk/spacewalk/Cargo.toml
-          mv /home/runner/work/spacewalk/spacewalk/Cargo_CI.toml /home/runner/work/spacewalk/spacewalk/Cargo.toml
+          bash ./scripts/cmd-all clippy "clippy --lib --bins" "--release -- -W clippy::all -A clippy::style -A forgetting_copy_types -A forgetting_references"
 
-      - name: Remove rust-toolchain.toml for nightly
-        if: matrix.rust == 'nightly'
-        # To make sure that the nightly version will be used all throughout
-        run: |
-          rm /home/runner/work/spacewalk/spacewalk/rust-toolchain.toml
+  check-code-nightly:
+    name: Run Clippy and Fmt
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
 
-      - name: Setup nightly Rust toolchain
-        if: matrix.rust == 'nightly'
-        uses: dtolnay/rust-toolchain@nightly
+      - name: Run Prerequisite (for Nightly)
+        uses: ./.github/actions/prerequisite-nightly
         with:
-          toolchain: nightly-2024-02-09
-          components: rustfmt, clippy
-          target: wasm32-unknown-unknown
+          token: ${{ secrets.GITHUB_TOKEN }}
+          version: ${{ env.RUSTUP_NIGHTLY_VERSION }}
 
-      - name: Setup nightly Rust as default
-        if: matrix.rust == 'nightly'
-        run: rustup default nightly-2024-02-09
+      - name: Rustfmt
+        uses: actions-rs/cargo@v1
+        with:
+          toolchain: ${{ env.RUSTUP_NIGHTLY_VERSION }}
+          command: fmt
+          args: --all -- --check
 
-      - name: Use Cache
-        uses: Swatinem/rust-cache@v2
+      - name: Clippy for Tests and Examples
+        uses: actions-rs/cargo@v1
         with:
-          cache-on-failure: true
-          key: "ubuntu-latest-cargo-${{ hashFiles('**/Cargo.lock') }}"
-          shared-key: "shared"
+          toolchain: ${{ env.RUSTUP_NIGHTLY_VERSION }}
+          command: clippy
+          args: --release --all-features --tests --benches --examples -- -A clippy::all -W clippy::correctness -A forgetting_copy_types -A forgetting_references
+
+  cargo-test:
+    name: Run Tests for Pallets and Clients
+    needs: check-code
+    if: needs.check-code.result == 'success'
+    runs-on: ubuntu-latest
+    env:
+      SOURCE_STELLAR_SECRET_MAINNET: ${{ secrets.SOURCE_STELLAR_SECRET_MAINNET }}
+      SOURCE_STELLAR_SECRET_TESTNET: ${{ secrets.SOURCE_STELLAR_SECRET_TESTNET }}
+      DEST_STELLAR_SECRET_MAINNET: ${{ secrets.DEST_STELLAR_SECRET_MAINNET }}
+      DEST_STELLAR_SECRET_TESTNET: ${{ secrets.DEST_STELLAR_SECRET_TESTNET }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
 
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+      - name: Run Prerequisite (for Nightly)
+        uses: ./.github/actions/prerequisite-nightly
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          version: ${{ env.RUSTUP_NIGHTLY_VERSION }}
 
-      - name: Check Build
-        if: matrix.rust == 'stable'
+      - name: Run Tests for Pallets
+        run: |
+            bash ./scripts/cmd-pallets test "+${{ env.RUSTUP_NIGHTLY_VERSION }} test" "--release"
+
+      - name: Run Tests for Other Libraries
         run: |
-          bash ./scripts/cmd-all build check
+          bash ./scripts/cmd-pallets test "+${{ env.RUSTUP_NIGHTLY_VERSION }} test" "--release"
+
+      - name: Run Tests for Clients
+        run: |
+          bash ./scripts/cmd-clients test "+${{ env.RUSTUP_NIGHTLY_VERSION }} test" "--release"
+
+  cargo-test-vault:
+    name: Run Tests for Vault
+    needs: check-code
+    if: needs.check-code.result == 'success'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
 
-      - name: Test
-        if: matrix.rust == 'nightly'
+      - name: Undergo Prerequisite (for Nightly)
+        uses: ./.github/actions/prerequisite-nightly
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          version: ${{ env.RUSTUP_NIGHTLY_VERSION }}
+
+      - name: Run Tests for Vault
         uses: actions-rs/cargo@v1
         env:
           SOURCE_STELLAR_SECRET_MAINNET: ${{ secrets.SOURCE_STELLAR_SECRET_MAINNET }}
           SOURCE_STELLAR_SECRET_TESTNET: ${{ secrets.SOURCE_STELLAR_SECRET_TESTNET }}
           DEST_STELLAR_SECRET_MAINNET: ${{ secrets.DEST_STELLAR_SECRET_MAINNET }}
           DEST_STELLAR_SECRET_TESTNET: ${{ secrets.DEST_STELLAR_SECRET_TESTNET }}
         with:
-          toolchain: nightly-2024-02-09
-          command: test 
-          args:  --all --all-features
-
-      - name: Rustfmt
-        if: matrix.rust == 'nightly'
-        uses: actions-rs/cargo@v1
-        with:
-          toolchain: nightly-2024-02-09
-          command: fmt
-          args: --all -- --check
-
-      - name: Clippy -- Libraries and Binaries
-        if: matrix.rust == 'stable'
-        run: |
-          bash ./scripts/cmd-all clippy "clippy --lib --bins" "-- -W clippy::all -A clippy::style -A forgetting_copy_types -A forgetting_references"           
-
-      - name: Clippy -- Tests and Examples
-        if: matrix.rust == 'nightly'
-        uses: actions-rs/cargo@v1
-        with:
-          toolchain: nightly-2024-02-09
-          command: clippy
-          args: --all-features --tests --benches --examples -- -A clippy::all -W clippy::correctness -A forgetting_copy_types -A forgetting_references
+          toolchain: ${{ env.RUSTUP_NIGHTLY_VERSION }}
+          command: test
+          # we only test the standalone-metadata
+          args: --release -p vault --features integration-test