chore(CI): Re-enable simtests with forked repos (#1752)

* chose(CI): Re-enable simtests with forked repos * fix cargo-simtest script * update tokio * try setting up ssh for cargo-deny * fixes * use git cli * update dependencies and fix test * use v2 deny * known hosts * lock * Add secrets * Update macros dep * fix test * Re-add use * update dependencies * update dependencies * Remove explicit patches * remove nextest expressions for now * revert and correct config patch * use revision over branch * uncomment mysticeti simtests too * Merge with new method * whoopps * add note * mover axum dep * cleanup workflows --------- Co-authored-by: Thibault Martinez <[email protected]>
iotaledger · Aug 29, 2024 · ea39e70 · ea39e70
1 parent 70d738d
commit ea39e70
Show file tree

Hide file tree

Showing 15 changed files with 340 additions and 298 deletions.
diff --git a/.github/workflows/_external_rust_lints.yml b/.github/workflows/_external_rust_lints.yml
@@ -5,6 +5,11 @@ on:
     inputs:
       isRust:
         type: boolean
+    secrets:
+      SSH_PRIVATE_KEY_IOTA_CI:
+        required: true
+      SSH_GITHUB_KNOWN_HOSTS:
+        required: true
 
 concurrency:
   group: ext-rust-lints-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -28,7 +33,11 @@ jobs:
     runs-on: [self-hosted]
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - uses: EmbarkStudios/cargo-deny-action@v2
+        with:
+          ssh-key: ${{ secrets.SSH_PRIVATE_KEY_IOTA_CI }}
+          ssh-known-hosts: ${{ secrets.SSH_GITHUB_KNOWN_HOSTS }}
+          use-git-cli: true
 
   clippy:
     if: always() && inputs.isRust

diff --git a/.github/workflows/_mysticeti_tests.yml b/.github/workflows/_mysticeti_tests.yml
@@ -136,25 +136,46 @@ jobs:
       - run: scripts/changed-files.sh
         shell: bash
 
-  # TODO: Disabled until we can get an updated fork of msim with tokio 1.38 https://github.com/iotaledger/iota/issues/1212
-  # simtest:
-  #   name: Simtest ${{ matrix.components }}
-  #   needs: changes
-  #   runs-on: [self-hosted]
-  #   strategy:
-  #     matrix:
-  #       components: ${{ fromJson(needs.changes.outputs.components) }}
-  #     fail-fast: false
-  #   env:
-  #     MSIM_WATCHDOG_TIMEOUT_MS: 60000
-  #   steps:
-  #     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # Pin v4.1.1
-  #       with:
-  #         ref: ${{ github.event.inputs.iota_repo_ref || github.ref }}
-  #     - uses: taiki-e/install-action@nextest
-  #     - name: cargo simtest
-  #       run: |
-  #         scripts/simtest/cargo-simtest simtest -E 'rdeps(${{matrix.components}})' -p ${{matrix.components}}
-  #     - name: check new tests for flakiness
-  #       run: |
-  #         scripts/simtest/stress-new-tests.sh -E 'rdeps(${{matrix.components}})' -p ${{matrix.components}}
+  simtest:
+    name: Simtest Mysticeti
+    needs: changes
+    timeout-minutes: 45
+    runs-on: [self-hosted]
+    env:
+      MSIM_WATCHDOG_TIMEOUT_MS: 60000
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
+        with:
+          ref: ${{ github.event.inputs.iota_repo_ref || github.ref }}
+      - uses: taiki-e/install-action@nextest
+      - name: setup filter
+        run: |
+          array=(${{ join(fromJson(needs.changes.outputs.components), ' ') }})
+
+          result=""
+
+          for item in "${array[@]}"; do
+              mapped_item="-E 'rdeps(${item})'"
+
+              if [ -z "$result" ]; then
+                  result="$mapped_item"
+              else
+                  result="$result $mapped_item"
+              fi
+          done
+
+          echo "filters=$result" >> "$GITHUB_ENV"
+      - name: cargo simtest
+        run: |
+          command="scripts/simtest/cargo-simtest simtest $filters"
+
+          echo "$command"
+
+          eval ${command}
+      - name: check new tests for flakiness
+        run: |
+          command="scripts/simtest/stress-new-tests.sh $filters"
+
+          echo "$command"
+
+          eval ${command}
diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml
@@ -5,6 +5,11 @@ on:
     inputs:
       isRust:
         type: boolean
+    secrets:
+      SSH_PRIVATE_KEY_IOTA_CI:
+        required: true
+      SSH_GITHUB_KNOWN_HOSTS:
+        required: true
 
 concurrency:
   group: rust-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -38,11 +43,13 @@ jobs:
     uses: ./.github/workflows/_rust_lints.yml
     with:
       isRust: ${{ inputs.isRust }}
+    secrets: inherit
 
   external-lints:
     uses: ./.github/workflows/_external_rust_lints.yml
     with:
       isRust: ${{ inputs.isRust }}
+    secrets: inherit
 
   rust-tests:
     if: github.event.pull_request.draft == false

diff --git a/.github/workflows/_rust_lints.yml b/.github/workflows/_rust_lints.yml
@@ -5,6 +5,11 @@ on:
     inputs:
       isRust:
         type: boolean
+    secrets:
+      SSH_PRIVATE_KEY_IOTA_CI:
+        required: true
+      SSH_GITHUB_KNOWN_HOSTS:
+        required: true
 
 concurrency:
   group: rust-lints-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -27,7 +32,11 @@ jobs:
     runs-on: [self-hosted]
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - uses: EmbarkStudios/cargo-deny-action@v2
+        with:
+          ssh-key: ${{ secrets.SSH_PRIVATE_KEY_IOTA_CI }}
+          ssh-known-hosts: ${{ secrets.SSH_GITHUB_KNOWN_HOSTS }}
+          use-git-cli: true
 
   clippy:
     if: always() && inputs.isRust

diff --git a/.github/workflows/_rust_tests.yml b/.github/workflows/_rust_tests.yml
@@ -110,27 +110,47 @@ jobs:
       - run: scripts/changed-files.sh
         shell: bash
 
-  # TODO: Disabled until we can get an updated fork of msim with tokio 1.38 https://github.com/iotaledger/iota/issues/1212
-  # simtest:
-  #     name: Simtest ${{ matrix.components }}
-  #     needs: changes
-  #     timeout-minutes: 45
-  #     runs-on: [self-hosted]
-  #     strategy:
-  #         matrix:
-  #             components: ${{ fromJson(needs.changes.outputs.components) }}
-  #         fail-fast: false
-  #     env:
-  #         MSIM_WATCHDOG_TIMEOUT_MS: 60000
-  #     steps:
-  #         - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
-  #         - uses: taiki-e/install-action@nextest
-  #         - name: cargo simtest
-  #           run: |
-  #               scripts/simtest/cargo-simtest simtest -E 'rdeps(${{matrix.components}})' -p ${{matrix.components}}
-  #         - name: check new tests for flakiness
-  #           run: |
-  #               scripts/simtest/stress-new-tests.sh -E 'rdeps(${{matrix.components}})' -p ${{matrix.components}}
+  simtest:
+    name: Simtest rust
+    needs: changes
+    timeout-minutes: 45
+    runs-on: [self-hosted]
+    env:
+      MSIM_WATCHDOG_TIMEOUT_MS: 60000
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
+      - uses: taiki-e/install-action@nextest
+      - name: setup filter
+        run: |
+          array=(${{ join(fromJson(needs.changes.outputs.components), ' ') }})
+
+          result=""
+
+          for item in "${array[@]}"; do
+              mapped_item="-E 'rdeps(${item})'"
+
+              if [ -z "$result" ]; then
+                  result="$mapped_item"
+              else
+                  result="$result $mapped_item"
+              fi
+          done
+
+          echo "filters=$result" >> "$GITHUB_ENV"
+      - name: cargo simtest
+        run: |
+          command="scripts/simtest/cargo-simtest simtest $filters"
+
+          echo "$command"
+
+          eval ${command}
+      - name: check new tests for flakiness
+        run: |
+          command="scripts/simtest/stress-new-tests.sh $filters"
+
+          echo "$command"
+
+          eval ${command}
 
   # # Disabled
   # rosetta-validation:

diff --git a/.github/workflows/hierarchy.yml b/.github/workflows/hierarchy.yml
@@ -115,6 +115,7 @@ jobs:
     uses: ./.github/workflows/_rust.yml
     with:
       isRust: ${{ needs.diff.outputs.isRust == 'true' }}
+    secrets: inherit
 
   e2e:
     if: github.event.pull_request.draft == false