From ec5f8e74ccd5783bd7f924a56d40596feefcffa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Tue, 4 Jun 2024 14:03:30 +0200 Subject: [PATCH] Update cbench workflow. (#5077) --- .github/workflows/cbench.yml | 50 ++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cbench.yml b/.github/workflows/cbench.yml index c7fa506725c..e18d85d4a07 100644 --- a/.github/workflows/cbench.yml +++ b/.github/workflows/cbench.yml @@ -9,10 +9,15 @@ on: - "quickwit/**" - "!quickwit/quickwit-ui/**" # For security reasons (to make sure the list of allowed users is - # trusted), make sure we run the workflow definition the base of the - # pull request. + # trusted), make sure we run the workflow definition from the base + # commit of the pull request. pull_request_target: +# This is required for github.rest.issues.createComment. +permissions: + issues: write + pull-requests: write + env: RUSTFLAGS: --cfg tokio_unstable @@ -26,7 +31,7 @@ jobs: # The self-hosted runner must have the system deps installed for QW and # the benchmark, because we don't have root access. runs-on: self-hosted - timeout-minutes: 40 + timeout-minutes: 60 steps: - name: Set authorized users id: authorized-users @@ -79,19 +84,54 @@ jobs: - name: Run Benchmark on SSD if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) id: bench-run-ssd - run: python3 ./run.py --search-only --storage pd-ssd --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_local}" --write-exported-run-url-to-file $GITHUB_OUTPUT + run: python3 ./run.py --search-only --storage pd-ssd --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_local}" --github-workflow-user "${{ github.actor }}" --github-workflow-run-id "${{ github.run_id }}" --comparison-reference-tag="push_main" --github-pr "${{ github.event_name == 'pull_request_target' && github.event.number || 0 }}" --comparison-reference-commit "${{ github.event_name == 'pull_request_target' && github.sha || github.event.before }}" --write-exported-run-url-to-file $GITHUB_OUTPUT working-directory: ./benchmarks - name: Run Benchmark on cloud storage if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) id: bench-run-cloud-storage - run: python3 ./run.py --search-only --storage gcs --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_gcs}" --write-exported-run-url-to-file $GITHUB_OUTPUT + run: python3 ./run.py --search-only --storage gcs --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_gcs}" --engine-config-file engines/quickwit/configs/cbench_quickwit_gcs.yaml --github-workflow-user "${{ github.actor }}" --github-workflow-run-id "${{ github.run_id }}" --comparison-reference-tag="push_main" --github-pr "${{ github.event_name == 'pull_request_target' && github.event.number || 0 }}" --comparison-reference-commit "${{ github.event_name == 'pull_request_target' && github.sha || github.event.before }}" --write-exported-run-url-to-file $GITHUB_OUTPUT working-directory: ./benchmarks - name: Show results links if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) run: | echo "::notice title=Benchmark Results on SSD::${{ steps.bench-run-ssd.outputs.url }}" + echo "::notice title=Comparison of results on SSD::${{ steps.bench-run-ssd.outputs.comparison_text }}" echo "::notice title=Benchmark Results on Cloud Storage::${{ steps.bench-run-cloud-storage.outputs.url }}" + echo "::notice title=Comparison of results on Cloud Storage::${{ steps.bench-run-cloud-storage.outputs.comparison_text }}" - name: In case of auth error if: ${{ ! contains(fromJSON(steps.authorized-users.outputs.users), github.actor) }} run: | echo "::error title=User not allowed to run the benchmark::User must be in list ${{ steps.authorized-users.outputs.users }}" + - name: Add a PR comment with comparison results + uses: actions/github-script@v6 + if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name == 'pull_request_target' + # Inspired from: https://github.com/actions/github-script/blob/60a0d83039c74a4aee543508d2ffcb1c3799cdea/.github/workflows/pull-request-test.yml + with: + script: | + // Get the existing comments. + const {data: comments} = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.number, + }) + + // Find any comment already made by the bot to update it. + const botComment = comments.find(comment => comment.user.id === 41898282) + const commentBody = "### On SSD:\n${{ steps.bench-run-ssd.outputs.comparison_text }}\n### On GCS:\n${{ steps.bench-run-cloud-storage.outputs.comparison_text }}\n" + if (botComment) { + // Update existing comment. + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }) + } else { + // New comment. + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.number, + body: commentBody + }) + }