From f527c6e01f4d8cdaac1b224b776baa662b0d16f9 Mon Sep 17 00:00:00 2001 From: Richard West Date: Sat, 22 Jul 2023 23:00:05 -0400 Subject: [PATCH 1/8] Fix a bug in the CI workflow Forgot to define the pull request number in the environment --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 34978ebde3..9861422ef7 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -347,6 +347,8 @@ jobs: - name: Prepare Results for PR Comment if : ${{ github.event_name == 'pull_request' }} + env: + PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > summary.txt echo "## Regression Testing Results" >> summary.txt From 80b651ff900444ae9a3362c169adb527546b1910 Mon Sep 17 00:00:00 2001 From: Richard West Date: Sun, 23 Jul 2023 09:05:22 -0400 Subject: [PATCH 2/8] Annotate script shouldn't fail on pushes. The workflow is triggered by things other than pull requests, but only works for pull requests. This should make it not fail on other events. Also reports the github.event.workflow_run in case there are clues in there that would let us filter it differently. --- .github/workflows/annotate.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/annotate.yml b/.github/workflows/annotate.yml index 4e2a1d93a2..2ddaa55048 100644 --- a/.github/workflows/annotate.yml +++ b/.github/workflows/annotate.yml @@ -19,6 +19,7 @@ jobs: steps: - run: echo 'The triggering workflow passed' - name: 'Download regression_summary artifact' + continue-on-error: true uses: actions/github-script@v6 with: script: | @@ -39,16 +40,22 @@ jobs: let fs = require('fs'); fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/regression_summary.zip`, Buffer.from(download.data)); + - name: Report the github.event.workflow_run as json + if: ${{ error() }} + run: echo '${{ toJson(github.event.workflow_run) }}' + - name: 'Unzip artifact' run: unzip regression_summary.zip + if : ${{ success() }} - name: 'Get pull request number' - id: pr_number + if: ${{ success() }} run: | echo "PR_NUMBER=$( head -n 1 summary.txt )" >> $GITHUB_ENV # remove the first line from the file sed -i '1d' summary.txt - name: Write summary to pull request as a comment + if: ${{ success() }} uses: thollander/actions-comment-pull-request@v2 with: filePath: summary.txt From d64364228df23522d162d9016b8157533ca7b020 Mon Sep 17 00:00:00 2001 From: Richard West Date: Sun, 23 Jul 2023 11:56:46 -0400 Subject: [PATCH 3/8] Update .github/workflows/annotate.yml --- .github/workflows/annotate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/annotate.yml b/.github/workflows/annotate.yml index 2ddaa55048..5e49ea2e56 100644 --- a/.github/workflows/annotate.yml +++ b/.github/workflows/annotate.yml @@ -41,7 +41,7 @@ jobs: fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/regression_summary.zip`, Buffer.from(download.data)); - name: Report the github.event.workflow_run as json - if: ${{ error() }} + if: ${{ failure() }} run: echo '${{ toJson(github.event.workflow_run) }}' - name: 'Unzip artifact' From 234c0f9b805ec48c55f3c2c976850a81ba4ea1aa Mon Sep 17 00:00:00 2001 From: Richard West Date: Sun, 23 Jul 2023 22:56:07 -0400 Subject: [PATCH 4/8] Futzing with annotation workflow. The continue-on-error:true makes it think that step succeeded (though it didn't) so the "do on failure" doesn't run, and then the next step fails. Now we let it fail, and at least report in json the information that may let us try something else. One idea is to cange the javascript around line 32 that gets the matchArtifact and if there isn't one then return there and behave differently. --- .github/workflows/annotate.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/annotate.yml b/.github/workflows/annotate.yml index 5e49ea2e56..c07c1ffe00 100644 --- a/.github/workflows/annotate.yml +++ b/.github/workflows/annotate.yml @@ -19,7 +19,6 @@ jobs: steps: - run: echo 'The triggering workflow passed' - name: 'Download regression_summary artifact' - continue-on-error: true uses: actions/github-script@v6 with: script: | @@ -40,27 +39,24 @@ jobs: let fs = require('fs'); fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/regression_summary.zip`, Buffer.from(download.data)); - - name: Report the github.event.workflow_run as json - if: ${{ failure() }} - run: echo '${{ toJson(github.event.workflow_run) }}' - - name: 'Unzip artifact' run: unzip regression_summary.zip - if : ${{ success() }} - name: 'Get pull request number' - if: ${{ success() }} run: | echo "PR_NUMBER=$( head -n 1 summary.txt )" >> $GITHUB_ENV # remove the first line from the file sed -i '1d' summary.txt - name: Write summary to pull request as a comment - if: ${{ success() }} uses: thollander/actions-comment-pull-request@v2 with: filePath: summary.txt pr_number: ${{ env.PR_NUMBER }} + - name: Report the github.event.workflow_run as json + if: ${{ failure() }} + run: echo '${{ toJson(github.event.workflow_run) }}' + on-failure: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'failure' }} From df6128957cdb06609cfdfd9572a75e50422798d6 Mon Sep 17 00:00:00 2001 From: Richard West Date: Mon, 24 Jul 2023 09:59:32 -0400 Subject: [PATCH 5/8] Only run annotation workflow on pull requests. Hoping this works. Also fix a problem with escaping ' inside JSON when logging the github.event.workflow_run context. --- .github/workflows/annotate.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/annotate.yml b/.github/workflows/annotate.yml index c07c1ffe00..dd95bcef61 100644 --- a/.github/workflows/annotate.yml +++ b/.github/workflows/annotate.yml @@ -15,7 +15,7 @@ on: jobs: on-success: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' }} steps: - run: echo 'The triggering workflow passed' - name: 'Download regression_summary artifact' @@ -47,6 +47,7 @@ jobs: echo "PR_NUMBER=$( head -n 1 summary.txt )" >> $GITHUB_ENV # remove the first line from the file sed -i '1d' summary.txt + - name: Write summary to pull request as a comment uses: thollander/actions-comment-pull-request@v2 with: @@ -55,11 +56,13 @@ jobs: - name: Report the github.event.workflow_run as json if: ${{ failure() }} - run: echo '${{ toJson(github.event.workflow_run) }}' + run: echo $JSON + env: + JSON: ${{ toJson(github.event.workflow_run) }} on-failure: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'pull_request' }} steps: - run: echo 'The triggering workflow failed' From 2e1c05d339edbfc7e1e9001627f5959264afa54b Mon Sep 17 00:00:00 2001 From: Richard West Date: Mon, 24 Jul 2023 11:28:38 -0400 Subject: [PATCH 6/8] Give the annotation runs a better name. --- .github/workflows/annotate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/annotate.yml b/.github/workflows/annotate.yml index dd95bcef61..505be2ce6a 100644 --- a/.github/workflows/annotate.yml +++ b/.github/workflows/annotate.yml @@ -12,6 +12,8 @@ on: types: - completed +run-name: Annotate PR ${{ github.event.workflow_run.event.display_title }}. + jobs: on-success: runs-on: ubuntu-latest From 6147eebfedb72229c1c51a63271da54c4e2ee25b Mon Sep 17 00:00:00 2001 From: Richard West Date: Mon, 24 Jul 2023 11:29:17 -0400 Subject: [PATCH 7/8] Just skip annotation runs on failed pull requests. Rather than run a pointless "the workflow failed" job, we just skip it. --- .github/workflows/annotate.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/annotate.yml b/.github/workflows/annotate.yml index 505be2ce6a..6a67c9b9e9 100644 --- a/.github/workflows/annotate.yml +++ b/.github/workflows/annotate.yml @@ -61,10 +61,3 @@ jobs: run: echo $JSON env: JSON: ${{ toJson(github.event.workflow_run) }} - - on-failure: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'pull_request' }} - steps: - - run: echo 'The triggering workflow failed' - From c3e4ce02c87548fab6b17204aa0d31b379f41785 Mon Sep 17 00:00:00 2001 From: Richard West Date: Mon, 24 Jul 2023 13:39:08 -0400 Subject: [PATCH 8/8] Put regression results into workflow summary even for not PRs The Continuous Integration workflow runs on - pushes - pull requests - a schedule - workflow calls from database testing With this change, we create the regression test summary and cat it into the workflow summary for all job types, though it is still only uploaded as an artifact for pull requests. --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9861422ef7..a13e807029 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -346,9 +346,9 @@ jobs: fi - name: Prepare Results for PR Comment - if : ${{ github.event_name == 'pull_request' }} + if: ${{ env.REFERENCE_JOB == 'false' }} env: - PR_NUMBER: ${{ github.event.number }} + PR_NUMBER: ${{ github.event.number || github.event.after || github.event_name }} run: | echo $PR_NUMBER > summary.txt echo "## Regression Testing Results" >> summary.txt