CI: Add Ecosystem tests

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: "Test all features"
+name: "Tests"
 
 # If you modify more test jobs, ensure that you add them as required to the job "confirmTestPassed"
 # which is located at the end of this file (more info in the job)
@@ -266,6 +266,92 @@ jobs:
           export PATH=$(pwd)/target/release:$PATH
           cargo test --manifest-path integration-tests/zombienet/Cargo.toml
 
+  build-runtimes:
+    needs: [ runtime-matrix ]
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Set rust version via common env file
+        run: cat .github/env >> $GITHUB_ENV
+
+      - name: Install stable toolchain
+        run: |
+          rustup install $RUST_STABLE_VERSION
+          rustup default $RUST_STABLE_VERSION
+          rustup target add wasm32-unknown-unknown
+          rustup component add rust-src
+
+      - name: Fetch cache
+        uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2.7.0
+        with:
+          shared-key: "fellowship-cache-build-runtimes-test"
+
+      - name: Build runtime ${{ matrix.runtime.name }}
+        run: |
+          PACKAGE_NAME=${{ matrix.runtime.package }}
+          RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm
+          RUNTIME_BLOB_PATH=./target/production/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME
+          # Build wasm
+          echo "Building wasm RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH"
+
+          # Dirty hack to find out if the `metadata-hash` feature exists for the given package.
+          # Should print "error: no bin target named `something-that-does-not-exist`." if the feature doesn't exist.
+          # Otherwise it should print "error: the package 'glutton-kusama-runtime' does not contain this feature: metadata-hash".
+          if grep -v "something-that-does-not-exist" <<< $(cargo build  -p ${{ matrix.runtime.package }} --features metadata-hash --bin something-that-does-not-exist 2>&1); then
+            FEATURES="--features=metadata-hash"
+          fi
+
+          # We only enable `metadata-hash`, but not `on-chain-release-build` to still have logs enabled.
+          echo "Setting features: ${FEATURES}"
+          cargo build --profile production -p ${{ matrix.runtime.package }} $FEATURES -q --locked
+          echo "RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" >> $GITHUB_ENV
+
+      - name: Upload  ${{ matrix.runtime.name }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.runtime.name }}
+          path: ${{ env.RUNTIME_BLOB_PATH }}
+
+  ecosystem-tests:
+    needs: [runtime-matrix, build-runtimes]
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Checkout polkadot-ecosystem-tests
+        uses: actions/checkout@v4
+        with:
+          repository: open-web3-stack/polkadot-ecosystem-tests
+          path: ecosystem-tests
+
+      - name: Download WASM artifacts
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+
+      - name: Run ecosystem tests
+        run: |
+          jq --raw-output '.[] | .name, .package' .github/workflows/runtimes-matrix.json | while read -r NAME && read -r PACKAGE; do
+            NAME="$(echo $NAME | tr -d '-' | tr '[:lower:]' '[:upper:]')"
+            RUNTIME_BLOB_NAME=$(echo $PACKAGE | sed 's/-/_/g').compact.compressed.wasm
+            echo "Setting runtime override ${NAME}_WASM=$(pwd)/${RUNTIME_BLOB_NAME}"
+            # Set the path to the build runtimes
+            eval "export ${NAME}_WASM=$(pwd)/${RUNTIME_BLOB_NAME}"
+          done 
+
+          export DEBUG=api-ws
+          cd ecosystem-tests
+          yarn install
+          yarn test
+
   # This will only run if all the tests in its "needs" array passed.
   # Add this as your required job, becuase if the matrix changes size (new things get added)
   # it will still require all the steps to succeed.
@@ -279,5 +365,6 @@ jobs:
       - integration-test
       - build-chain-spec-generator
       - zombienet-smoke
+      - ecosystem-tests
     steps:
       - run: echo '### Good job! All the tests passed 🚀' >> $GITHUB_STEP_SUMMARY