forked from FooSoft/yomichan
-
Notifications
You must be signed in to change notification settings - Fork 112
142 lines (129 loc) · 6.87 KB
/
playwright_comment.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: Comment on the PR with Playwright results
on:
workflow_run:
workflows: ["Playwright Tests"]
types:
- completed
permissions: {}
jobs:
playwright_comment:
runs-on: ubuntu-latest
permissions:
pull-requests: write
if: >
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success'
steps:
- name: Harden Runner
uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
with:
egress-policy: audit
- name: Grab playwright-output from PR run
uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: playwright-output
- name: Grab master-screenshots-outcome from PR run
uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: master-screenshots-outcome
- name: Grab playwright-results-json from PR run
uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: playwright-results-json
- name: Dry-run grab playwright-report from PR run so we have its ID
uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
id: playwright-report
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: playwright-report
dry_run: true
- name: Store playwright-report ID
id: playwright-report-artifact-id
env:
ARTIFACTS_JSON: ${{ steps.playwright-report.outputs.artifacts }}
run: |
ID=$(echo "$ARTIFACTS_JSON" | jq -r '.[0].id');
echo "id=$ID" >> "$GITHUB_OUTPUT"
- name: Generate summary from playwright-results.json (expected to fail to comment)
id: playwright-summary
uses: daun/playwright-report-summary@v3
with:
report-file: playwright-results.json
- name: Load artifacts into environment variables
id: playwright
run: |
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
{
echo "PLAYWRIGHT_OUTPUT<<$EOF";
cat ./playwright-output;
echo "$EOF";
echo "MASTER_SCREENSHOTS_OUTCOME<<$EOF";
cat ./master-screenshots-outcome;
echo "$EOF";
echo "FAILED=$(grep -c '^ *[0-9] failed$' ./playwright-output)";
echo "FLAKY=$(grep -c '^ *[0-9] flaky$' ./playwright-output)"
} >> "$GITHUB_OUTPUT"
# this is required because github.event.workflow_run.pull_requests is not available for PRs from forks
- name: Get PR context
id: source-run-info
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Find the most recently updated open PR at the repo with the requested commit:
SEARCH_QUERY: >-
type:pr state:open sort:updated-desc
repo:${{ github.repository }}
${{ github.event.workflow_run.head_sha }}
# Minimal graphql search query to fetch the PR `number` field:
GQL: |-
query($filter: String!) {
search( query: $filter, type: ISSUE, first: 1) {
nodes { ... on PullRequest { number } }
}
}
# Formats the GQL response into a `key=value` string + basic error handling
JQ_FILTER: >-
.data.search.nodes[0]
| if (.number == null) then error("Could not find PR number") end
| "pullRequestNumber=\(.number)"
run: |
gh api graphql --field "filter=$SEARCH_QUERY" --raw-field "query=$GQL" --jq "$JQ_FILTER" >> "${GITHUB_OUTPUT}"
- name: "[Comment] Couldn't download screenshots from master branch"
uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure'
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
message: |
:heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging.
- name: "[Comment] Warning: Visual differences caused by this PR; please check the playwright report"
uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FAILED != 0
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
message: |
:warning: <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${{ steps.playwright-report-artifact-id.outputs.id }}">Visual changes detected by playwright; please check the report to verify if they are desirable.</a>
- name: "[Comment] Success (but flaky): No visual differences introduced by this PR (but flaky tests detected)"
uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY != 0 && steps.playwright.outputs.FAILED == 0
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
message: |
:heavy_check_mark: <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${{ steps.playwright-report-artifact-id.outputs.id }}">No visual changes detected by playwright, but flaky tests were detected; please try to fix the tests.</a>
- name: "[Comment] Success: No visual differences introduced by this PR"
uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
message: |
:heavy_check_mark: <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${{ steps.playwright-report-artifact-id.outputs.id }}">No visual changes detected by playwright.</a>
update-only: true