From e5c23924da77d6ca562545f39925ff649d888eb4 Mon Sep 17 00:00:00 2001 From: AlexandreSinger Date: Fri, 4 Oct 2024 13:29:28 -0400 Subject: [PATCH] [CI] Improved How The CI Handles Num Cores Used Found that the CI was always building and running using only 2 cores on the regular tests. However, the CI has 4 cores that it can use. The number of cores can change based on some factors. Added the ability of the tests to check how many cores they have available, and use all the cores to build and run VTR tests. --- .github/scripts/build.sh | 6 ++- .github/scripts/unittest.sh | 2 +- .github/workflows/test.yml | 101 ++++++++++++++++++++++++++++++------ 3 files changed, 92 insertions(+), 17 deletions(-) diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh index 924ba911ba1..5eea1790e0e 100755 --- a/.github/scripts/build.sh +++ b/.github/scripts/build.sh @@ -6,9 +6,13 @@ source $(dirname "$0")/common.sh $SPACER +if [[ -z "${NUM_PROC}" ]]; then + NUM_PROC=1 +fi + start_section "vtr.build" "${GREEN}Building..${NC}" export FAILURE=0 -make -k BUILD_TYPE=${BUILD_TYPE} CMAKE_PARAMS="-Werror=dev ${CMAKE_PARAMS} ${CMAKE_INSTALL_PREFIX_PARAMS}" -j2 || export FAILURE=1 +make -k BUILD_TYPE=${BUILD_TYPE} CMAKE_PARAMS="-Werror=dev ${CMAKE_PARAMS} ${CMAKE_INSTALL_PREFIX_PARAMS}" -j${NUM_PROC} || export FAILURE=1 end_section "vtr.build" # When the build fails, produce the failure output in a clear way diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh index 0aef473c515..133f8258766 100755 --- a/.github/scripts/unittest.sh +++ b/.github/scripts/unittest.sh @@ -9,7 +9,7 @@ $(dirname "$0")/build.sh $SPACER start_section "vtr.test.0" "${GREEN}Testing..${NC} ${CYAN}C++ unit tests${NC}" -make test +make test -j${NUM_PROC} end_section "vtr.test.0" $SPACER diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8035b7e0834..0c235508351 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,13 +52,20 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 - name: Test env: BUILD_TYPE: ${{ matrix.build_type }} + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh VERBOSE=${{ matrix.verbose }} @@ -80,7 +87,9 @@ jobs: with: python-version: 3.10.10 - uses: actions/checkout@v4 - - run: ./.github/scripts/install_dependencies.sh + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - name: Test run: ./dev/${{ matrix.script }} @@ -97,11 +106,18 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - name: Test env: CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: ./.github/scripts/unittest.sh @@ -116,7 +132,13 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 @@ -125,6 +147,7 @@ jobs: #In order to get compilation warnings produced per source file, we must do a non-IPO build #We also turn warnings into errors for this target by doing a strict compile CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_STRICT_COMPILE=on -DVTR_IPO_BUILD=off" + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh @@ -201,17 +224,24 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 - name: Test env: CMAKE_PARAMS: ${{ matrix.params }} + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh - ./run_reg_test.py ${{ matrix.suite }} -show_failures -j2 + ./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count}} - name: Upload regression run files if: ${{ !cancelled() }} @@ -259,7 +289,12 @@ jobs: with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 @@ -273,12 +308,13 @@ jobs: # depends on LLVM and all CI tests where VTR_ENABLE_SANITIZE is enabled fail. For a temporary # fix, we manually reduce the entropy. This quick fix should be removed in the future # when github deploys a more stable Ubuntu image. + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | sudo sysctl -w vm.mmap_rnd_bits=28 export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh # We skip QoR since we are only checking for errors in sanitizer runs - ./run_reg_test.py ${{ matrix.suite }} -show_failures -j2 -skip_qor + ./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count }} -skip_qor Parmys: @@ -292,7 +328,13 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 @@ -300,10 +342,11 @@ jobs: env: CMAKE_PARAMS: '-DVTR_IPO_BUILD=off' BUILD_TYPE: debug + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh - ./run_reg_test.py parmys_reg_basic -show_failures -j2 + ./run_reg_test.py parmys_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }} ODINII: @@ -317,7 +360,13 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 @@ -325,11 +374,12 @@ jobs: env: CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on' BUILD_TYPE: debug + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | sudo sysctl -w vm.mmap_rnd_bits=28 export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh - ./run_reg_test.py odin_reg_basic -show_failures -j2 + ./run_reg_test.py odin_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }} VQM2BLIF: @@ -343,13 +393,20 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 - name: Test env: BUILD_TYPE: release + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh @@ -378,7 +435,13 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 @@ -387,6 +450,7 @@ jobs: CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on" MATRIX_EVAL: ${{ matrix.eval }} BUILD_TYPE: release + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" ./.github/scripts/build.sh @@ -413,13 +477,20 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' - - run: ./.github/scripts/install_dependencies.sh + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: Install dependencies + run: ./.github/scripts/install_dependencies.sh - uses: hendrikmuhs/ccache-action@v1.2 - name: Test env: CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on' + NUM_PROC: ${{ steps.cpu-cores.outputs.count }} _COVERITY_URL: 'https://scan.coverity.com/download/linux64' _COVERITY_MD5: 'd0d7d7df9d6609e578f85096a755fb8f' run: |