Merge pull request #2759 from AlexandreSinger/feature-ci-test-num-pro…

…c-upgrade [CI] Improved How The CI Handles Num Cores Used
verilog-to-routing · Oct 7, 2024 · f85b422 · f85b422
2 parents 3b74db5 + 0610fff
commit f85b422
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 16 deletions.
diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh
@@ -6,9 +6,13 @@ source $(dirname "$0")/common.sh
 
 $SPACER
 
+if [[ -z "${NUM_PROC}" ]]; then
+    NUM_PROC=1
+fi
+
 start_section "vtr.build" "${GREEN}Building..${NC}"
 export FAILURE=0
-make -k BUILD_TYPE=${BUILD_TYPE} CMAKE_PARAMS="-Werror=dev ${CMAKE_PARAMS} ${CMAKE_INSTALL_PREFIX_PARAMS}" -j2 || export FAILURE=1
+make -k BUILD_TYPE=${BUILD_TYPE} CMAKE_PARAMS="-Werror=dev ${CMAKE_PARAMS} ${CMAKE_INSTALL_PREFIX_PARAMS}" -j${NUM_PROC} || export FAILURE=1
 end_section "vtr.build"
 
 # When the build fails, produce the failure output in a clear way

diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh
@@ -9,7 +9,7 @@ $(dirname "$0")/build.sh
 $SPACER
 
 start_section "vtr.test.0" "${GREEN}Testing..${NC} ${CYAN}C++ unit tests${NC}"
-make test
+make test -j${NUM_PROC}
 end_section "vtr.test.0"
 
 $SPACER
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -52,13 +52,20 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
     - name: Test
       env:
         BUILD_TYPE: ${{ matrix.build_type }}
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh VERBOSE=${{ matrix.verbose }}
@@ -80,7 +87,9 @@ jobs:
       with:
         python-version: 3.10.10
     - uses: actions/checkout@v4
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - name: Test
       run: ./dev/${{ matrix.script }}
@@ -97,11 +106,18 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - name: Test
       env:
         CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on"
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: ./.github/scripts/unittest.sh
 
 
@@ -116,7 +132,13 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
@@ -125,6 +147,7 @@ jobs:
         #In order to get compilation warnings produced per source file, we must do a non-IPO build
         #We also turn warnings into errors for this target by doing a strict compile
         CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_STRICT_COMPILE=on -DVTR_IPO_BUILD=off"
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh
@@ -214,6 +237,10 @@ jobs:
       with:
         submodules: 'true'
 
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
     - name: Install dependencies
       run: ./.github/scripts/install_dependencies.sh
 
@@ -226,10 +253,11 @@ jobs:
     - name: Test
       env:
         CMAKE_PARAMS: ${{ matrix.params }}
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh
-        ./run_reg_test.py ${{ matrix.suite }} -show_failures -j2
+        ./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count}}
 
     - name: Upload regression run files
       if: ${{ !cancelled() }}
@@ -277,7 +305,12 @@ jobs:
       with:
         submodules: 'true'
 
-    - run: ./.github/scripts/install_dependencies.sh
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
@@ -291,12 +324,13 @@ jobs:
         # depends on LLVM and all CI tests where VTR_ENABLE_SANITIZE is enabled fail. For a temporary
         # fix, we manually reduce the entropy. This quick fix should be removed in the future
         # when github deploys a more stable Ubuntu image.
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         sudo sysctl -w vm.mmap_rnd_bits=28
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh
         # We skip QoR since we are only checking for errors in sanitizer runs
-        ./run_reg_test.py ${{ matrix.suite }} -show_failures -j2 -skip_qor
+        ./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count }} -skip_qor
 
 
   Parmys:
@@ -310,18 +344,25 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-      - run: ./.github/scripts/install_dependencies.sh
+
+      - name: Get number of CPU cores
+        uses: SimenB/github-actions-cpu-cores@v2
+        id: cpu-cores
+
+      - name: Install dependencies
+        run: ./.github/scripts/install_dependencies.sh
 
       - uses: hendrikmuhs/[email protected]
 
       - name: Test
         env:
           CMAKE_PARAMS: '-DVTR_IPO_BUILD=off'
           BUILD_TYPE: debug
+          NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
         run: |
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
           ./.github/scripts/build.sh
-          ./run_reg_test.py parmys_reg_basic -show_failures -j2
+          ./run_reg_test.py parmys_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }}
 
 
   ODINII:
@@ -335,19 +376,26 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
     - name: Test
       env:
         CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on'
         BUILD_TYPE: debug
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         sudo sysctl -w vm.mmap_rnd_bits=28
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh
-        ./run_reg_test.py odin_reg_basic -show_failures -j2
+        ./run_reg_test.py odin_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }}
 
 
   VQM2BLIF:
@@ -361,13 +409,20 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
     - name: Test
       env:
         BUILD_TYPE: release
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh
@@ -396,7 +451,13 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
@@ -405,6 +466,7 @@ jobs:
         CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on"
         MATRIX_EVAL: ${{ matrix.eval }}
         BUILD_TYPE: release
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         ./.github/scripts/build.sh
@@ -431,13 +493,20 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - run: ./.github/scripts/install_dependencies.sh
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Install dependencies
+      run: ./.github/scripts/install_dependencies.sh
 
     - uses: hendrikmuhs/[email protected]
 
     - name: Test
       env:
         CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on'
+        NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
         _COVERITY_URL: 'https://scan.coverity.com/download/linux64'
         _COVERITY_MD5: 'd0d7d7df9d6609e578f85096a755fb8f'
       run: |