Merge pull request #789 from ldorau/Add_Coverage

Add code coverage to CI
oneapi-src · Oct 15, 2024 · a226f59 · a226f59
2 parents 38a0e79 + c041412
commit a226f59
Show file tree

Hide file tree

Showing 17 changed files with 323 additions and 29 deletions.
diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml
@@ -11,6 +11,8 @@ env:
   UMF_VERSION: 0.10.0
   BUILD_DIR : "${{github.workspace}}/build"
   INSTL_DIR : "${{github.workspace}}/../install-dir"
+  COVERAGE_DIR : "${{github.workspace}}/coverage"
+  COVERAGE_NAME : "exports-coverage-basic"
 
 jobs:
   ubuntu-build:
@@ -122,8 +124,8 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang cmake libnuma-dev libjemalloc-dev 
-        
+        sudo apt-get install -y clang cmake libnuma-dev libjemalloc-dev lcov
+
     - name: Install TBB apt package
       if: matrix.install_tbb == 'ON'
       run: |
@@ -167,6 +169,7 @@ jobs:
         -DUMF_TESTS_FAIL_ON_SKIP=ON
         -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}}
         -DUMF_LINK_HWLOC_STATICALLY=${{matrix.link_hwloc_statically}}
+        ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
     - name: Build UMF
       run: |
@@ -177,7 +180,23 @@ jobs:
       working-directory: ${{env.BUILD_DIR}}
       run: |
         ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }}
-        ctest --output-on-failure --test-dir test
+        ctest --output-on-failure # run all tests for better coverage
+
+    - name: Check coverage
+      if:  ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }}
+      working-directory: ${{env.BUILD_DIR}}
+      run: |
+        export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}}
+        echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME"
+        ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME
+        mkdir -p ${{env.COVERAGE_DIR}}
+        mv ./$COVERAGE_FILE_NAME ${{env.COVERAGE_DIR}}
+
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      if:  ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }}
+      with:
+        name: ${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}}
+        path: ${{env.COVERAGE_DIR}}
 
     - name: Remove the installation directory
       run: rm -rf ${{env.INSTL_DIR}}

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -0,0 +1,49 @@
+# Coverage build
+name: Coverage
+
+on: workflow_call
+
+permissions:
+  contents: read
+
+env:
+  COVERAGE_DIR : "${{github.workspace}}/coverage"
+
+jobs:
+  Coverage:
+    name: Coverage build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies (ubuntu-latest)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y lcov
+
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: exports-coverage-*
+          path: coverage
+          merge-multiple: true
+
+      - name: Compute coverage
+        working-directory: ${{env.COVERAGE_DIR}}
+        run: |
+          echo "DIR: $(pwd)" && ls -al
+          ../scripts/coverage/merge_coverage_files.sh exports-coverage total_coverage
+          genhtml --no-function-coverage -o html_report total_coverage 2>&1 | tee output.txt
+          mkdir coverage_report
+          mv html_report ./coverage_report/
+          tail -n2 output.txt >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: coverage_html_report
+          path: coverage/coverage_report
diff --git a/.github/workflows/dax.yml b/.github/workflows/dax.yml
@@ -28,6 +28,8 @@ env:
   UMF_TESTS_FSDAX_PATH: "/mnt/pmem1/file"
   BUILD_DIR : "${{github.workspace}}/build"
   INSTL_DIR : "${{github.workspace}}/../install-dir"
+  COVERAGE_DIR : "${{github.workspace}}/coverage"
+  COVERAGE_NAME : "exports-coverage-dax"
 
 jobs:
   dax:
@@ -83,6 +85,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
           -DUMF_TESTS_FAIL_ON_SKIP=ON
+          ${{ matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
       - name: Build UMF
         run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $(nproc)
@@ -100,3 +103,19 @@ jobs:
           UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} ctest -C ${{matrix.build_type}} -R umf-provider_file_memory -V
           UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} ctest -C ${{matrix.build_type}} -R umf_example_dram_and_fsdax -V
           UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} ctest -C ${{matrix.build_type}} -R umf-ipc_file_prov_fsdax -V
+
+      - name: Check coverage
+        if:  ${{ matrix.build_type == 'Debug' }}
+        working-directory: ${{env.BUILD_DIR}}
+        run: |
+          export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-shared-${{matrix.shared_library}}
+          echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME"
+          ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME
+          mkdir -p ${{env.COVERAGE_DIR}}
+          mv ./$COVERAGE_FILE_NAME ${{env.COVERAGE_DIR}}
+
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        if:  ${{ matrix.build_type == 'Debug' }}
+        with:
+          name: ${{env.COVERAGE_NAME}}-shared-${{matrix.shared_library}}
+          path: ${{env.COVERAGE_DIR}}
diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml
@@ -12,26 +12,31 @@ permissions:
 env:
   BUILD_DIR : "${{github.workspace}}/build"
   INSTL_DIR : "${{github.workspace}}/../install-dir"
+  COVERAGE_DIR : "${{github.workspace}}/coverage"
 
 jobs:
   gpu:
     name: Build
     env:
-      BUILD_TYPE: Release
       VCPKG_PATH: "${{github.workspace}}/../../../../vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/../../../../vcpkg/packages/tbb_x64-windows;${{github.workspace}}/../../../../vcpkg/packages/jemalloc_x64-windows"
+      COVERAGE_NAME : "exports-coverage-gpu"
     # run only on upstream; forks will not have the HW
     if: github.repository == 'oneapi-src/unified-memory-framework'
     strategy:
       matrix:
         shared_library: ['ON', 'OFF']
         os: ['Ubuntu', 'Windows']
+        build_type: ['Debug', 'Release']
         include:
           - os: 'Ubuntu'
             compiler: {c: gcc, cxx: g++}
             number_of_processors: '$(nproc)'
           - os: 'Windows'
             compiler: {c: cl, cxx: cl}
             number_of_processors: '$Env:NUMBER_OF_PROCESSORS'
+        exclude:
+          - os: 'Windows'
+            build_type: 'Debug'
 
     runs-on: ["DSS-LEVEL_ZERO", "DSS-${{matrix.os}}"]
     steps:
@@ -51,7 +56,7 @@ jobs:
           -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
           -B ${{env.BUILD_DIR}}
           -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
-          -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+          -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
           -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
           -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
           -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
@@ -73,7 +78,7 @@ jobs:
           cmake
           -B ${{env.BUILD_DIR}}
           -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
-          -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+          -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
           -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
           -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
           -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
@@ -88,31 +93,49 @@ jobs:
           -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
           -DUMF_BUILD_CUDA_PROVIDER=OFF
           -DUMF_TESTS_FAIL_ON_SKIP=ON
+          ${{ matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
       - name: Build UMF
-        run: cmake --build ${{env.BUILD_DIR}} --config ${{env.BUILD_TYPE}} -j ${{matrix.number_of_processors}}
+        run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j ${{matrix.number_of_processors}}
 
       - name: Run tests
         working-directory: ${{env.BUILD_DIR}}
-        run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --test-dir test
+        run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
 
       - name: Run examples
         working-directory: ${{env.BUILD_DIR}}
-        run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}}
+        run: ctest --output-on-failure --test-dir examples -C ${{matrix.build_type}}
 
       - name: Run benchmarks
         working-directory: ${{env.BUILD_DIR}}
-        run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded
+        run: ctest --output-on-failure --test-dir benchmark -C ${{matrix.build_type}} --exclude-regex umf-bench-multithreaded
+
+      - name: Check coverage
+        if:  ${{ matrix.build_type == 'Debug' && matrix.os == 'Ubuntu' }}
+        working-directory: ${{env.BUILD_DIR}}
+        run: |
+          export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-shared-${{matrix.shared_library}}
+          echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME"
+          ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME
+          mkdir -p ${{env.COVERAGE_DIR}}
+          mv ./$COVERAGE_FILE_NAME ${{env.COVERAGE_DIR}}
+
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        if:  ${{ matrix.build_type == 'Debug' && matrix.os == 'Ubuntu' }}
+        with:
+          name: ${{env.COVERAGE_NAME}}-shared-${{matrix.shared_library}}
+          path: ${{env.COVERAGE_DIR}}
 
   gpu-CUDA:
     name: Build
     env:
-      BUILD_TYPE: Release
+      COVERAGE_NAME : "exports-coverage-gpu-CUDA"
     # run only on upstream; forks will not have the HW
     if: github.repository == 'oneapi-src/unified-memory-framework'
     strategy:
       matrix:
         shared_library: ['ON', 'OFF']
+        build_type: ['Debug', 'Release']
         # TODO add windows
         os: ['Ubuntu']
         include:
@@ -136,7 +159,7 @@ jobs:
       run: >
         cmake -B ${{env.BUILD_DIR}}
         -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
-        -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+        -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
         -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
         -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
         -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
@@ -151,18 +174,35 @@ jobs:
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
         -DUMF_BUILD_CUDA_PROVIDER=ON
         -DUMF_TESTS_FAIL_ON_SKIP=ON
+        ${{ matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
     - name: Build UMF
-      run: cmake --build ${{env.BUILD_DIR}} --config ${{env.BUILD_TYPE}} -j ${{matrix.number_of_processors}}
+      run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j ${{matrix.number_of_processors}}
 
     - name: Run tests
       working-directory: ${{env.BUILD_DIR}}
-      run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --test-dir test
+      run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
 
     - name: Run examples
       working-directory: ${{env.BUILD_DIR}}
-      run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}}
+      run: ctest --output-on-failure --test-dir examples -C ${{matrix.build_type}}
 
     - name: Run benchmarks
       working-directory: ${{env.BUILD_DIR}}
-      run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded
+      run: ctest --output-on-failure --test-dir benchmark -C ${{matrix.build_type}} --exclude-regex umf-bench-multithreaded
+
+    - name: Check coverage
+      if:  ${{ matrix.build_type == 'Debug' && matrix.os == 'Ubuntu' }}
+      working-directory: ${{env.BUILD_DIR}}
+      run: |
+        export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-shared-${{matrix.shared_library}}
+        echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME"
+        ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME
+        mkdir -p ${{env.COVERAGE_DIR}}
+        mv ./$COVERAGE_FILE_NAME ${{env.COVERAGE_DIR}}
+
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      if:  ${{ matrix.build_type == 'Debug' && matrix.os == 'Ubuntu' }}
+      with:
+        name: ${{env.COVERAGE_NAME}}-shared-${{matrix.shared_library}}
+        path: ${{env.COVERAGE_DIR}}
diff --git a/.github/workflows/multi_numa.yml b/.github/workflows/multi_numa.yml
@@ -6,6 +6,11 @@ on: [workflow_call]
 permissions:
   contents: read
 
+env:
+  BUILD_DIR : "${{github.workspace}}/build"
+  COVERAGE_DIR : "${{github.workspace}}/coverage"
+  COVERAGE_NAME : "exports-coverage-multinuma"
+
 jobs:
   multi_numa:
     name: ${{matrix.os}}
@@ -40,6 +45,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_TESTS_FAIL_ON_SKIP=ON
+          ${{ matrix.os == 'ubuntu-22.04' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
       - name: Build UMF
         run: cmake --build ${{github.workspace}}/build -j $(nproc)
@@ -59,3 +65,19 @@ jobs:
           ctest --output-on-failure --test-dir test -E "umf-provider_os_memory_multiple_numa_nodes"
           ./test/umf_test-provider_os_memory_multiple_numa_nodes \
             --gtest_filter="-*checkModeLocal/*:*checkModePreferredEmptyNodeset/*:testNuma.checkModeInterleave"
+
+      - name: Check coverage
+        if: matrix.os == 'ubuntu-22.04'
+        working-directory: ${{env.BUILD_DIR}}
+        run: |
+          export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-os-${{matrix.os}}
+          echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME"
+          ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME
+          mkdir -p ${{env.COVERAGE_DIR}}
+          mv ./$COVERAGE_FILE_NAME ${{env.COVERAGE_DIR}}
+
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        if: matrix.os == 'ubuntu-22.04'
+        with:
+          name: ${{env.COVERAGE_NAME}}-os-${{matrix.os}}
+          path: ${{env.COVERAGE_DIR}}
diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml
@@ -110,3 +110,6 @@ jobs:
   MultiNuma:
     needs: [Build]
     uses: ./.github/workflows/multi_numa.yml
+  Coverage:
+    needs: [Build, DevDax, GPU, MultiNuma, Qemu, ProxyLib]
+    uses: ./.github/workflows/coverage.yml
diff --git a/.github/workflows/proxy_lib.yml b/.github/workflows/proxy_lib.yml
@@ -9,6 +9,8 @@ permissions:
 env:
   BUILD_DIR : "${{github.workspace}}/build"
   INSTL_DIR : "${{github.workspace}}/../install-dir"
+  COVERAGE_DIR : "${{github.workspace}}/coverage"
+  COVERAGE_NAME : "exports-coverage-proxy"
 
 jobs:
   proxy-ubuntu:
@@ -30,7 +32,7 @@ jobs:
       - name: Install apt packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev
+          sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev lcov
 
       - name: Set ptrace value for IPC test
         run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
@@ -52,6 +54,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_TESTS_FAIL_ON_SKIP=ON
           -DUMF_PROXY_LIB_BASED_ON_POOL=${{matrix.proxy_lib_pool}}
+          ${{ matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
       - name: Build UMF
         run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -71,3 +74,19 @@ jobs:
       - name: Run "/usr/bin/date" with proxy library
         working-directory: ${{env.BUILD_DIR}}
         run: UMF_PROXY="page.disposition=shared-shm" LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/date
+
+      - name: Check coverage
+        if:  ${{ matrix.build_type == 'Debug' }}
+        working-directory: ${{env.BUILD_DIR}}
+        run: |
+          export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-proxy_lib_pool-${{matrix.proxy_lib_pool}}
+          echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME"
+          ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME
+          mkdir -p ${{env.COVERAGE_DIR}}
+          mv ./$COVERAGE_FILE_NAME ${{env.COVERAGE_DIR}}
+
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        if:  ${{ matrix.build_type == 'Debug' }}
+        with:
+          name: ${{env.COVERAGE_NAME}}-proxy_lib_pool-${{matrix.proxy_lib_pool}}
+          path: ${{env.COVERAGE_DIR}}