From 7f324546a7a64708911f0541632bed046a5259bd Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Sun, 29 Dec 2024 13:43:16 +0900 Subject: [PATCH] Shard playwright CI --- .github/workflows/playwright.yml | 95 +++++++++++++++++++----- .github/workflows/playwright_comment.yml | 14 ---- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 5c2f1db42..27a3864b3 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -6,15 +6,15 @@ on: permissions: contents: read jobs: - playwright: + playwright-tests: timeout-minutes: 60 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - egress-policy: audit - - name: Remove all fonts run: rm -rf /usr/share/fonts @@ -66,24 +66,28 @@ jobs: - name: "[PR] Generate new screenshots & compare against master" id: playwright run: | - npx playwright test 2>&1 | tee ./playwright-output || true + npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} 2>&1 | tee ./playwright-output || true continue-on-error: true if: github.event_name == 'pull_request' && steps.master-screenshots.outcome != 'failure' - name: "[Push] Generate new authoritative screenshots for master" id: playwright-master - run: npx playwright test -u + run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} -u if: github.event_name == 'push' - uses: actions/upload-artifact@v4 with: - name: playwright-screenshots + name: playwright-screenshots-${{ matrix.shardIndex }} path: test/playwright/__screenshots__/ + retention-days: 1 - - uses: actions/upload-artifact@v4 + - name: Upload blob report to GitHub Actions Artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 with: - name: playwright-report - path: playwright-report/ + name: blob-report-${{ matrix.shardIndex }} + path: blob-report + retention-days: 1 - name: "[PR] Store steps.master-screenshots.outcome in a file" run: echo ${{ steps.master-screenshots.outcome }} > ./master-screenshots-outcome @@ -91,18 +95,75 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: master-screenshots-outcome + name: master-screenshots-outcome-${{ matrix.shardIndex }} path: master-screenshots-outcome if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v4 with: - name: playwright-output + name: playwright-output-${{ matrix.shardIndex }} path: playwright-output + retention-days: 1 if: github.event_name == 'pull_request' + playwright: + timeout-minutes: 60 + runs-on: ubuntu-latest + # Merge reports after playwright-tests, even if some shards have failed + if: ${{ !cancelled() }} + needs: [playwright-tests] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + + ### playwright-report + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: all-blob-reports + pattern: blob-report-* + merge-multiple: true + + - name: Merge into HTML Report + run: npx playwright merge-reports --reporter html ./all-blob-reports + - uses: actions/upload-artifact@v4 with: - name: playwright-results-json - path: playwright-results.json - if: github.event_name == 'pull_request' + name: playwright-report + path: playwright-report/ + + ### playwright-screenshots + - name: Download & merge screenshots from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: test/playwright/__screenshots__ + pattern: playwright-screenshots-* + merge-multiple: true + + - uses: actions/upload-artifact@v4 + with: + name: playwright-screenshots + path: test/playwright/__screenshots__/ + + ### master-screenshots-outcome + - name: Download master-screenshots-outcome files + uses: actions/download-artifact@v4 + with: + path: master-screenshots-outcomes + pattern: master-screenshots-outcome-* + merge-multiple: true + + - name: Check if any shard failed to download master screenshots + run: | + if grep -q "failure" master-screenshots-outcomes/*; then + echo "failure" > merged-master-screenshots-outcome + else + echo "success" > merged-master-screenshots-outcome + fi + + - uses: actions/upload-artifact@v4 + with: + name: master-screenshots-outcome + path: merged-master-screenshots-outcome + + diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml index 9a9938e24..25b6b9d3b 100644 --- a/.github/workflows/playwright_comment.yml +++ b/.github/workflows/playwright_comment.yml @@ -36,14 +36,6 @@ jobs: run_id: ${{ github.event.workflow_run.id }} name: master-screenshots-outcome - - name: Grab playwright-results-json from PR run - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 - continue-on-error: true - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ github.event.workflow_run.id }} - name: playwright-results-json - - name: Dry-run grab playwright-report from PR run so we have its ID uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 id: playwright-report @@ -62,12 +54,6 @@ jobs: ID=$(echo "$ARTIFACTS_JSON" | jq -r '.[0].id'); echo "id=$ID" >> "$GITHUB_OUTPUT" - - name: Generate summary from playwright-results.json (expected to fail to comment) - id: playwright-summary - uses: daun/playwright-report-summary@v3 - with: - report-file: playwright-results.json - - name: Load artifacts into environment variables id: playwright run: |