Comment on the PR with Playwright results #3357
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Comment on the PR with Playwright results | |
| on: | |
| workflow_run: | |
| workflows: ["Playwright Tests"] | |
| types: | |
| - completed | |
| permissions: {} | |
| jobs: | |
| playwright_comment: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| if: > | |
| github.event.workflow_run.event == 'pull_request' && | |
| github.event.workflow_run.conclusion == 'success' | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 | |
| with: | |
| egress-policy: audit | |
| - name: Grab playwright-output from PR run | |
| uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 | |
| continue-on-error: true | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| run_id: ${{ github.event.workflow_run.id }} | |
| name: playwright-output | |
| - name: Grab master-screenshots-outcome from PR run | |
| uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 | |
| continue-on-error: true | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| run_id: ${{ github.event.workflow_run.id }} | |
| name: master-screenshots-outcome | |
| - name: Grab playwright-results-json from PR run | |
| uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 | |
| continue-on-error: true | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| run_id: ${{ github.event.workflow_run.id }} | |
| name: playwright-results-json | |
| - name: Dry-run grab playwright-report from PR run so we have its ID | |
| uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 | |
| id: playwright-report | |
| continue-on-error: true | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| run_id: ${{ github.event.workflow_run.id }} | |
| name: playwright-report | |
| dry_run: true | |
| - name: Store playwright-report ID | |
| id: playwright-report-artifact-id | |
| env: | |
| ARTIFACTS_JSON: ${{ steps.playwright-report.outputs.artifacts }} | |
| run: | | |
| ID=$(echo "$ARTIFACTS_JSON" | jq -r '.[0].id'); | |
| echo "id=$ID" >> "$GITHUB_OUTPUT" | |
| - name: Generate summary from playwright-results.json (expected to fail to comment) | |
| id: playwright-summary | |
| uses: daun/playwright-report-summary@v3 | |
| with: | |
| report-file: playwright-results.json | |
| - name: Load artifacts into environment variables | |
| id: playwright | |
| run: | | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| { | |
| echo "PLAYWRIGHT_OUTPUT<<$EOF"; | |
| cat ./playwright-output; | |
| echo "$EOF"; | |
| echo "MASTER_SCREENSHOTS_OUTCOME<<$EOF"; | |
| cat ./master-screenshots-outcome; | |
| echo "$EOF"; | |
| echo "FAILED=$(grep -c '^ *[0-9] failed$' ./playwright-output)"; | |
| echo "FLAKY=$(grep -c '^ *[0-9] flaky$' ./playwright-output)" | |
| } >> "$GITHUB_OUTPUT" | |
| # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks | |
| - name: Get PR context | |
| id: source-run-info | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Find the most recently updated open PR at the repo with the requested commit: | |
| SEARCH_QUERY: >- | |
| type:pr state:open sort:updated-desc | |
| repo:${{ github.repository }} | |
| ${{ github.event.workflow_run.head_sha }} | |
| # Minimal graphql search query to fetch the PR `number` field: | |
| GQL: |- | |
| query($filter: String!) { | |
| search( query: $filter, type: ISSUE, first: 1) { | |
| nodes { ... on PullRequest { number } } | |
| } | |
| } | |
| # Formats the GQL response into a `key=value` string + basic error handling | |
| JQ_FILTER: >- | |
| .data.search.nodes[0] | |
| | if (.number == null) then error("Could not find PR number") end | |
| | "pullRequestNumber=\(.number)" | |
| run: | | |
| gh api graphql --field "filter=$SEARCH_QUERY" --raw-field "query=$GQL" --jq "$JQ_FILTER" >> "${GITHUB_OUTPUT}" | |
| - name: "[Comment] Couldn't download screenshots from master branch" | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure' | |
| with: | |
| issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} | |
| message: | | |
| :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging. | |
| - name: "[Comment] Warning: Visual differences caused by this PR; please check the playwright report" | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FAILED != 0 | |
| with: | |
| issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} | |
| message: | | |
| :warning: <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${{ steps.playwright-report-artifact-id.outputs.id }}">Visual changes detected by playwright; please check the report to verify if they are desirable.</a> | |
| - name: "[Comment] Success (but flaky): No visual differences introduced by this PR (but flaky tests detected)" | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY != 0 && steps.playwright.outputs.FAILED == 0 | |
| with: | |
| issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} | |
| message: | | |
| :heavy_check_mark: <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${{ steps.playwright-report-artifact-id.outputs.id }}">No visual changes detected by playwright, but flaky tests were detected; please try to fix the tests.</a> | |
| - name: "[Comment] Success: No visual differences introduced by this PR" | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0 | |
| with: | |
| issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} | |
| message: | | |
| :heavy_check_mark: <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${{ steps.playwright-report-artifact-id.outputs.id }}">No visual changes detected by playwright.</a> | |
| update-only: true |