Shard playwright CI

yomidevs · Dec 29, 2024 · a39cfc4 · a39cfc4
1 parent 5425fb2
commit a39cfc4
Show file tree

Hide file tree

Showing 3 changed files with 101 additions and 35 deletions.
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -6,15 +6,15 @@ on:
 permissions:
   contents: read
 jobs:
-  playwright:
+  playwright-tests:
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4, 5]
+        shardTotal: [5]
     steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@v2
-        with:
-          egress-policy: audit
-
       - name: Remove all fonts
         run: rm -rf /usr/share/fonts
 
@@ -66,43 +66,126 @@ jobs:
       - name: "[PR] Generate new screenshots & compare against master"
         id: playwright
         run: |
-          npx playwright test 2>&1 | tee ./playwright-output || true
+          npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} 2>&1 | tee ./playwright-output || true
         continue-on-error: true
         if: github.event_name == 'pull_request' && steps.master-screenshots.outcome != 'failure'
 
       - name: "[Push] Generate new authoritative screenshots for master"
         id: playwright-master
-        run: npx playwright test -u
+        run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} -u
         if: github.event_name == 'push'
 
       - uses: actions/upload-artifact@v4
         with:
-          name: playwright-screenshots
+          name: playwright-screenshots-${{ matrix.shardIndex }}
           path: test/playwright/__screenshots__/
+          retention-days: 1
 
-      - uses: actions/upload-artifact@v4
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
         with:
-          name: playwright-report
-          path: playwright-report/
+          name: blob-report-${{ matrix.shardIndex }}
+          path: blob-report
+          retention-days: 1
 
       - name: "[PR] Store steps.master-screenshots.outcome in a file"
         run: echo ${{ steps.master-screenshots.outcome }} > ./master-screenshots-outcome
         if: github.event_name == 'pull_request'
 
       - uses: actions/upload-artifact@v4
         with:
-          name: master-screenshots-outcome
+          name: master-screenshots-outcome-${{ matrix.shardIndex }}
           path: master-screenshots-outcome
         if: github.event_name == 'pull_request'
 
       - uses: actions/upload-artifact@v4
         with:
-          name: playwright-output
+          name: playwright-output-${{ matrix.shardIndex }}
           path: playwright-output
+          retention-days: 1
         if: github.event_name == 'pull_request'
 
+  playwright:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    # Merge reports after playwright-tests, even if some shards have failed
+    if: ${{ !cancelled() }}
+    needs: [playwright-tests]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      ### playwright-report
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge into HTML Report
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+
+      - name: Merge into JSON Report
+        run: npx playwright merge-reports --reporter json ./all-blob-reports > playwright-report.json
+
       - uses: actions/upload-artifact@v4
         with:
-          name: playwright-results-json
-          path: playwright-results.json
-        if: github.event_name == 'pull_request'
+          name: playwright-report
+          path: playwright-report/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report-json
+          path: playwright-report.json
+
+      ### playwright-screenshots
+      - name: Download & merge screenshots from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: test/playwright/__screenshots__
+          pattern: playwright-screenshots-*
+          merge-multiple: true
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: playwright-screenshots
+          path: test/playwright/__screenshots__/
+
+      ### master-screenshots-outcome
+      - name: Download master-screenshots-outcome files
+        uses: actions/download-artifact@v4
+        with:
+          path: master-screenshots-outcomes
+          pattern: master-screenshots-outcome-*
+          merge-multiple: true
+
+      - name: Check if any shard failed to download master screenshots
+        run: |
+          if grep -q "failure" master-screenshots-outcomes/*; then
+            echo "failure" > merged-master-screenshots-outcome
+          else
+            echo "success" > merged-master-screenshots-outcome
+          fi
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: master-screenshots-outcome
+          path: merged-master-screenshots-outcome
+
+      ### playwright-output
+      - name: Download playwright-output files
+        uses: actions/download-artifact@v4
+        with:
+          path: playwright-outputs
+          pattern: playwright-output-*
+          merge-multiple: true
+
+      - name: Merge playwright output files
+        run: cat playwright-outputs/* > merged-playwright-output
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: playwright-output
+          path: merged-playwright-output
diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml
@@ -36,14 +36,6 @@ jobs:
           run_id: ${{ github.event.workflow_run.id }}
           name: master-screenshots-outcome
 
-      - name: Grab playwright-results-json from PR run
-        uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
-        continue-on-error: true
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ github.event.workflow_run.id }}
-          name: playwright-results-json
-
       - name: Dry-run grab playwright-report from PR run so we have its ID
         uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
         id: playwright-report
@@ -62,12 +54,6 @@ jobs:
           ID=$(echo "$ARTIFACTS_JSON" | jq -r '.[0].id');
           echo "id=$ID" >> "$GITHUB_OUTPUT"
 
-      - name: Generate summary from playwright-results.json (expected to fail to comment)
-        id: playwright-summary
-        uses: daun/playwright-report-summary@v3
-        with:
-          report-file: playwright-results.json
-
       - name: Load artifacts into environment variables
         id: playwright
         run: |

diff --git a/playwright.config.js b/playwright.config.js
@@ -47,10 +47,7 @@ export default defineConfig({
     /* Opt out of parallel tests on CI. */
     workers: process.env.CI ? 1 : void 0,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-    reporter: [
-        ['html'],
-        ['json', {outputFile: 'playwright-results.json'}],
-    ],
+    reporter: process.env.CI ? 'blob' : 'html',
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
         /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */