From 0b5593ff98ddb13a0a4a09104181a236af2d586a Mon Sep 17 00:00:00 2001 From: MarvNC Date: Mon, 28 Oct 2024 23:18:32 -0700 Subject: [PATCH 1/3] Use Playwright Report Summary Action --- .github/workflows/playwright.yml | 6 ++++++ playwright.config.js | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index a792076a98..fc262e30a6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -106,3 +106,9 @@ jobs: name: playwright-output path: playwright-output if: github.event_name == 'pull_request' + + - name: "[PR] Comment on the PR with Playwright results" + if: github.event_name == 'pull_request' + uses: daun/playwright-report-summary@v3 + with: + report-file: results.json diff --git a/playwright.config.js b/playwright.config.js index acfceb1d19..3e06911364 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -47,7 +47,10 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : void 0, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: [ + ['html'], + ['json', {outputFile: 'results.json'}], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ From 0d304765db36e614d41ac168ca75e04747ae0a02 Mon Sep 17 00:00:00 2001 From: MarvNC Date: Mon, 28 Oct 2024 23:27:01 -0700 Subject: [PATCH 2/3] Move comment logic --- .github/workflows/playwright.yml | 8 +- .github/workflows/playwright_comment.yml | 99 ++++++++++++++---------- playwright.config.js | 2 +- 3 files changed, 61 insertions(+), 48 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index fc262e30a6..924d2481ee 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -107,8 +107,8 @@ jobs: path: playwright-output if: github.event_name == 'pull_request' - - name: "[PR] Comment on the PR with Playwright results" - if: github.event_name == 'pull_request' - uses: daun/playwright-report-summary@v3 + - uses: actions/upload-artifact@v4 with: - report-file: results.json + name: playwright-results-json + path: playwright-results.json + if: github.event_name == 'pull_request' diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml index 8b3d5bb69b..49e34dca0d 100644 --- a/.github/workflows/playwright_comment.yml +++ b/.github/workflows/playwright_comment.yml @@ -35,53 +35,66 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} name: master-screenshots-outcome - - - name: Load artifacts into environment variables - id: playwright - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "PLAYWRIGHT_OUTPUT<<$EOF" >> $GITHUB_OUTPUT - cat ./playwright-output >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - echo "MASTER_SCREENSHOTS_OUTCOME<<$EOF" >> $GITHUB_OUTPUT - cat ./master-screenshots-outcome >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - echo "FAILED=$(grep -c '^ *[0-9] failed$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT - echo "FLAKY=$(grep -c '^ *[0-9] flaky$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT - - # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks - - name: "Get PR information" - uses: potiuk/get-workflow-origin@e2dae063368361e4cd1f510e8785cd73bca9352e # pin@v1_5 - id: source-run-info + - name: Grab playwright-results-json from PR run + uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # pin@v2 + continue-on-error: true with: - token: ${{ secrets.GITHUB_TOKEN }} - sourceRunId: ${{ github.event.workflow_run.id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ github.event.workflow_run.id }} + name: playwright-results-json - - name: "[Comment] Couldn't download screenshots from master branch" - uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure' + - name: "[PR] Comment on the PR with Playwright results" + if: github.event_name == 'pull_request' + uses: daun/playwright-report-summary@v3 with: - issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - message: | - :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging. + report-file: playwright-results.json - - name: "[Comment] Warning: Flaky tests or visual differences caused by this PR; please check the playwright report" - uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && (steps.playwright.outputs.FLAKY != 0 || steps.playwright.outputs.FAILED != 0) - with: - issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - message: | - :warning: Flaky tests or visual differences caused by this PR; please check the playwright report. + # - name: Load artifacts into environment variables + # id: playwright + # run: | + # EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + # echo "PLAYWRIGHT_OUTPUT<<$EOF" >> $GITHUB_OUTPUT + # cat ./playwright-output >> $GITHUB_OUTPUT + # echo "$EOF" >> $GITHUB_OUTPUT + # echo "MASTER_SCREENSHOTS_OUTCOME<<$EOF" >> $GITHUB_OUTPUT + # cat ./master-screenshots-outcome >> $GITHUB_OUTPUT + # echo "$EOF" >> $GITHUB_OUTPUT + # echo "FAILED=$(grep -c '^ *[0-9] failed$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT + # echo "FLAKY=$(grep -c '^ *[0-9] flaky$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT - View Playwright Report (note: open the "playwright-report" artifact) + # # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks + # - name: "Get PR information" + # uses: potiuk/get-workflow-origin@e2dae063368361e4cd1f510e8785cd73bca9352e # pin@v1_5 + # id: source-run-info + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # sourceRunId: ${{ github.event.workflow_run.id }} - - name: "[Comment] Success: No visual differences introduced by this PR" - uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0 - with: - issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - message: | - :heavy_check_mark: No visual differences introduced by this PR. + # - name: "[Comment] Couldn't download screenshots from master branch" + # uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + # if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure' + # with: + # issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + # message: | + # :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging. + + # - name: "[Comment] Warning: Flaky tests or visual differences caused by this PR; please check the playwright report" + # uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + # if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && (steps.playwright.outputs.FLAKY != 0 || steps.playwright.outputs.FAILED != 0) + # with: + # issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + # message: | + # :warning: Flaky tests or visual differences caused by this PR; please check the playwright report. + + # View Playwright Report (note: open the "playwright-report" artifact) + + # - name: "[Comment] Success: No visual differences introduced by this PR" + # uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + # if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0 + # with: + # issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + # message: | + # :heavy_check_mark: No visual differences introduced by this PR. - View Playwright Report (note: open the "playwright-report" artifact) - update-only: true \ No newline at end of file + # View Playwright Report (note: open the "playwright-report" artifact) + # update-only: true \ No newline at end of file diff --git a/playwright.config.js b/playwright.config.js index 3e06911364..dfbec44b67 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -49,7 +49,7 @@ export default defineConfig({ /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [ ['html'], - ['json', {outputFile: 'results.json'}], + ['json', {outputFile: 'playwright-results.json'}], ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { From dde2f897170c9ccb033beb0242baaa4554737f73 Mon Sep 17 00:00:00 2001 From: MarvNC Date: Mon, 28 Oct 2024 23:37:23 -0700 Subject: [PATCH 3/3] Use output summary --- .github/workflows/playwright_comment.yml | 92 ++++++++++++------------ 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml index 49e34dca0d..ff5f9c4377 100644 --- a/.github/workflows/playwright_comment.yml +++ b/.github/workflows/playwright_comment.yml @@ -43,58 +43,62 @@ jobs: run_id: ${{ github.event.workflow_run.id }} name: playwright-results-json - - name: "[PR] Comment on the PR with Playwright results" + - name: Generate summary from playwright-results.json (expected to fail to comment) + id: playwright-summary if: github.event_name == 'pull_request' uses: daun/playwright-report-summary@v3 with: report-file: playwright-results.json - # - name: Load artifacts into environment variables - # id: playwright - # run: | - # EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - # echo "PLAYWRIGHT_OUTPUT<<$EOF" >> $GITHUB_OUTPUT - # cat ./playwright-output >> $GITHUB_OUTPUT - # echo "$EOF" >> $GITHUB_OUTPUT - # echo "MASTER_SCREENSHOTS_OUTCOME<<$EOF" >> $GITHUB_OUTPUT - # cat ./master-screenshots-outcome >> $GITHUB_OUTPUT - # echo "$EOF" >> $GITHUB_OUTPUT - # echo "FAILED=$(grep -c '^ *[0-9] failed$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT - # echo "FLAKY=$(grep -c '^ *[0-9] flaky$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT + - name: Load artifacts into environment variables + id: playwright + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "PLAYWRIGHT_OUTPUT<<$EOF" >> $GITHUB_OUTPUT + cat ./playwright-output >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + echo "MASTER_SCREENSHOTS_OUTCOME<<$EOF" >> $GITHUB_OUTPUT + cat ./master-screenshots-outcome >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + echo "FAILED=$(grep -c '^ *[0-9] failed$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT + echo "FLAKY=$(grep -c '^ *[0-9] flaky$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT - # # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks - # - name: "Get PR information" - # uses: potiuk/get-workflow-origin@e2dae063368361e4cd1f510e8785cd73bca9352e # pin@v1_5 - # id: source-run-info - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # sourceRunId: ${{ github.event.workflow_run.id }} + # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks + - name: "Get PR information" + uses: potiuk/get-workflow-origin@e2dae063368361e4cd1f510e8785cd73bca9352e # pin@v1_5 + id: source-run-info + with: + token: ${{ secrets.GITHUB_TOKEN }} + sourceRunId: ${{ github.event.workflow_run.id }} - # - name: "[Comment] Couldn't download screenshots from master branch" - # uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - # if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure' - # with: - # issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - # message: | - # :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging. + - name: "[Comment] Couldn't download screenshots from master branch" + uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure' + with: + issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + message: | + :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging. - # - name: "[Comment] Warning: Flaky tests or visual differences caused by this PR; please check the playwright report" - # uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - # if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && (steps.playwright.outputs.FLAKY != 0 || steps.playwright.outputs.FAILED != 0) - # with: - # issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - # message: | - # :warning: Flaky tests or visual differences caused by this PR; please check the playwright report. + - name: "[Comment] Warning: Flaky tests or visual differences caused by this PR; please check the playwright report" + uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && (steps.playwright.outputs.FLAKY != 0 || steps.playwright.outputs.FAILED != 0) + with: + issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + message: ${{ steps.playwright-summary.outputs.summary }} + # message: | + # :warning: Flaky tests or visual differences caused by this PR; please check the playwright report. - # View Playwright Report (note: open the "playwright-report" artifact) + # View Playwright Report (note: open the "playwright-report" artifact) - # - name: "[Comment] Success: No visual differences introduced by this PR" - # uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - # if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0 - # with: - # issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - # message: | - # :heavy_check_mark: No visual differences introduced by this PR. + - name: "[Comment] Success: No visual differences introduced by this PR" + uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0 + with: + issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + message: ${{ steps.playwright-summary.outputs.summary }} + # message: | + # :heavy_check_mark: No visual differences introduced by this PR. - # View Playwright Report (note: open the "playwright-report" artifact) - # update-only: true \ No newline at end of file + # View Playwright Report (note: open the "playwright-report" artifact) + # TODO: uncomment + # update-only: true \ No newline at end of file