Merge pull request #1060 from EuphoricThinking/benchmark_p9_11_workfl…

…ow_review1

add a workflow for running UMF benchmarks

.github/workflows/benchmarks.yml

@@ -0,0 +1,31 @@
+name: Compute Benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_no:
+        description: PR number (if 0, it'll run on the main)
+        type: number
+      bench_script_params:
+        description: Parameters passed to script executing benchmark
+        type: string
+        required: false
+        default: ''
+      upload_report:
+        description: 'Upload HTML report'
+        type: boolean
+        required: false
+        default: false
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  manual:
+    name: Compute Benchmarks
+    uses: ./.github/workflows/reusable_benchmarks.yml
+    with:
+      pr_no: ${{ inputs.pr_no }}
+      bench_script_params: ${{ inputs.bench_script_params }}
+      upload_report: ${{ inputs.upload_report }}

.github/workflows/nightly.yml

@@ -194,3 +194,13 @@ jobs:
       # Beside the 2 LTS Ubuntu, we also test this on the latest Ubuntu -  to be updated
       # every 6 months, so we verify the latest version of packages (compilers, etc.).
       os: "['ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-24.10']"
+
+  Benchmarks:
+    uses: ./.github/workflows/reusable_benchmarks.yml
+    permissions:
+      contents: read
+      pull-requests: write
+    with:
+      pr_no: '0'
+      bench_script_params: ''
+      upload_report: true

.github/workflows/pr_push.yml

@@ -54,9 +54,6 @@ jobs:
     uses: ./.github/workflows/reusable_qemu.yml
     with:
       short_run: true
-  Benchmarks:
-    needs: [Build]
-    uses: ./.github/workflows/reusable_benchmarks.yml
   ProxyLib:
     needs: [Build]
     uses: ./.github/workflows/reusable_proxy_lib.yml

.github/workflows/reusable_benchmarks.yml

@@ -1,80 +1,163 @@
 # Executes benchmarks implemented in this repository
+# using scripts for benchmark results visualisation,
+# which are downloaded from Unified Runtime repository. 
 name: Benchmarks
 
-on: workflow_call
+on:
+  workflow_call:
+    inputs:
+      pr_no:
+        # even though this is a number, this is a workaround for issues with
+        # reusable workflow calls that result in "Unexpected value '0'" error.
+        type: string
+        default: '0'
+      bench_script_params:
+        required: false
+        type: string
+        default: ''
+      upload_report:
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
+  pull-requests: write
 
 env:
-  BUILD_DIR : "${{github.workspace}}/build"
-  INSTL_DIR : "${{github.workspace}}/../install-dir"
+  UMF_DIR: "${{github.workspace}}/umf-repo"
+  BUILD_DIR : "${{github.workspace}}/umf-repo/build"
 
 jobs:
   benchmarks:
     name: Benchmarks
-    env:
-      VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows"
     strategy:
       matrix:
-        os: ['ubuntu-latest', 'windows-latest']
-        include: 
-          # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command to determine the build type
-          - os: ubuntu-latest
-            extra_build_option: '-DCMAKE_BUILD_TYPE=Release'
+        os: ['ubuntu-latest']
     runs-on: ${{matrix.os}}
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-        with:
-          fetch-depth: 0
-
-      - name: Install apt packages
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
-
-      - name: Initialize vcpkg
-        if: matrix.os == 'windows-latest'
-        uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
-        with:
-          vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
-          vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg
-          vcpkgJsonGlob: '**/vcpkg.json'
-
-      - name: Install vcpkg packages
-        if: matrix.os == 'windows-latest'
-        run: vcpkg install
-        shell: pwsh # Specifies PowerShell as the shell for running the script.
-
-      - name: Configure build
-        run: >
-          cmake
-          -B ${{env.BUILD_DIR}}
-          ${{matrix.extra_build_option}}
-          -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
-          -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
-          -DUMF_BUILD_SHARED_LIBRARY=ON
-          -DUMF_BUILD_BENCHMARKS=ON
-          -DUMF_BUILD_BENCHMARKS_MT=ON
-          -DUMF_BUILD_TESTS=OFF
-          -DUMF_FORMAT_CODE_STYLE=OFF
-          -DUMF_DEVELOPER_MODE=OFF
-          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
-          -DUMF_BUILD_CUDA_PROVIDER=ON
-          -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
-          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
-
-      - name: Build UMF on Linux
-        if: matrix.os == 'ubuntu-latest'
-        run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
-
-      - name: Build UMF on Windows
-        if: matrix.os == 'windows-latest'
-        run: cmake --build ${{env.BUILD_DIR}} --config Release -j $Env:NUMBER_OF_PROCESSORS
-
-      - name: Run benchmarks
-        working-directory: ${{env.BUILD_DIR}}
-        run: ctest -V --test-dir benchmark -C Release
+    # Workspace on self-hosted runners is not cleaned automatically.
+    # We have to delete the files created outside of using actions.
+    - name: Cleanup self-hosted workspace
+      if: always()
+      run: |
+        ls -la ./
+        rm -rf ./* || true
+
+    - name: Add comment to PR
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+      if: ${{ always() && inputs.pr_no != 0 }}
+      with:
+        script: |
+          const pr_no = '${{ inputs.pr_no }}';
+          const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}';
+          const params = '${{ inputs.bench_script_params }}';
+          const body = `Compute Benchmarks run (with params: ${params}):\n${url}`;
+
+          github.rest.issues.createComment({
+            issue_number: pr_no,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body
+          })
+
+    - name: Checkout UMF
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        path: ${{env.UMF_DIR}}
+        fetch-depth: 0
+
+    # We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged.
+    - name: Fetch PR's merge commit
+      if: ${{ inputs.pr_no != 0 }}
+      working-directory: ${{env.UMF_DIR}}
+      env:
+        PR_NO: ${{ inputs.pr_no }}
+      run: |
+        git fetch -- https://github.com/${{github.repository}} +refs/pull/${PR_NO}/*:refs/remotes/origin/pr/${PR_NO}/*
+        git checkout origin/pr/${PR_NO}/merge
+        git rev-parse origin/pr/${PR_NO}/merge
+
+    - name: Install apt packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
+
+    - name: Configure build
+      run: >
+        cmake
+        -S ${{env.UMF_DIR}}
+        -B ${{env.BUILD_DIR}}
+        -DCMAKE_BUILD_TYPE=Release
+        -DUMF_BUILD_SHARED_LIBRARY=ON
+        -DUMF_BUILD_BENCHMARKS=ON
+        -DUMF_BUILD_BENCHMARKS_MT=ON
+        -DUMF_BUILD_TESTS=OFF
+        -DUMF_FORMAT_CODE_STYLE=OFF
+        -DUMF_DEVELOPER_MODE=OFF
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+        -DUMF_BUILD_CUDA_PROVIDER=ON
+        -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
+        -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
+        -DUMF_BUILD_EXAMPLES=OFF
+
+    - name: Build UMF
+      run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
+
+    # We are going to clone Unified Runtime repository in order to run
+    # the most up-to-date UR scripts for benchmark data visualisation
+    - name: Checkout UR
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        repository: oneapi-src/unified-runtime
+        path: ur-repo
+        fetch-depth: 1
+        fetch-tags: false
+
+    - name: Install pip packages for benchmarking scripts from UR
+      run: |
+        pip install --force-reinstall -r ${{github.workspace}}/ur-repo/third_party/benchmark_requirements.txt
+
+    - name: Run dedicated for UMF benchmarking scripts from UR
+      id: benchmarks
+      working-directory: ${{env.BUILD_DIR}}
+      run: >
+        ${{ github.workspace }}/ur-repo/scripts/benchmarks/main.py
+        ~/bench_workdir
+        --umf  ${{env.BUILD_DIR}}
+        ${{ inputs.upload_report && '--output-html' || '' }}
+        ${{ inputs.bench_script_params }}
+
+    - name: Add comment to PR
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+      if: ${{ always() && inputs.pr_no != 0 }}
+      with:
+        script: |
+          let markdown = ""
+          try {
+            const fs = require('fs');
+            markdown = fs.readFileSync('${{env.BUILD_DIR}}/benchmark_results.md', 'utf8');
+          } catch(err) {
+          }
+
+          const pr_no = '${{ inputs.pr_no }}';
+          const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}';
+          const test_status = '${{ steps.benchmarks.outcome }}';
+          const job_status = '${{ job.status }}';
+          const params = '${{ inputs.bench_script_params }}';
+          const body = `Compute Benchmarks run (${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`;
+
+          github.rest.issues.createComment({
+            issue_number: pr_no,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body
+          })
+  
+    - name: Upload HTML report
+      if: ${{ always() && inputs.upload_report }}
+      uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      with:
+        path: ${{env.BUILD_DIR}}/benchmark_results.html
+        key: benchmark-results-${{ github.run_id }}