[ui, ci] UI test look-back and comparison for PRs #5037
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test-ui | |
| on: | |
| pull_request: | |
| paths: | |
| - "ui/**" | |
| push: | |
| branches: | |
| - main | |
| - release/** | |
| - test-ui | |
| paths: | |
| - "ui/**" | |
| jobs: | |
| pre-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| defaults: | |
| run: | |
| working-directory: ui | |
| outputs: | |
| nonce: ${{ steps.nonce.outputs.nonce }} | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - uses: ./.github/actions/setup-js | |
| - name: lint:js | |
| run: yarn run lint:js | |
| - name: lint:hbs | |
| run: yarn run lint:hbs | |
| - id: nonce | |
| name: nonce | |
| run: echo "nonce=${{ github.run_id }}-$(date +%s)" >> "$GITHUB_OUTPUT" | |
| tests: | |
| needs: | |
| - pre-test | |
| runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }} | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| defaults: | |
| run: | |
| working-directory: ui | |
| strategy: | |
| matrix: | |
| partition: [1, 2, 3, 4] | |
| split: [4] | |
| # Note: If we ever change the number of partitions, we'll need to update the | |
| # finalize.combine step to match | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - uses: ./.github/actions/setup-js | |
| - uses: browser-actions/setup-chrome@facf10a55b9caf92e0cc749b4f82bf8220989148 # v1.7.2 | |
| - name: Retrieve Vault-hosted Secrets | |
| if: endsWith(github.repository, '-enterprise') | |
| id: vault | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| with: | |
| url: ${{ vars.CI_VAULT_URL }} | |
| method: ${{ vars.CI_VAULT_METHOD }} | |
| path: ${{ vars.CI_VAULT_PATH }} | |
| jwtGithubAudience: ${{ vars.CI_VAULT_AUD }} | |
| secrets: |- | |
| kv/data/teams/nomad/ui PERCY_TOKEN ; | |
| - name: ember exam | |
| id: exam | |
| env: | |
| PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }} | |
| PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }} | |
| run: | | |
| yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json | |
| # We have continue-on-error set to true, but we still want to alert the author if | |
| # there are test failures or timeouts. Without it, we'll get errors in our output, | |
| # but the workflow will still succeed / have a green checkmark. | |
| - name: Express timeout failure | |
| if: ${{ failure() }} | |
| run: exit 1 | |
| - name: Check test status | |
| if: steps.exam.outcome != 'success' | |
| run: | | |
| echo "Tests failed or timed out in partition ${{ matrix.partition }}" | |
| exit 1 | |
| - name: Upload partition test results | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: test-results-${{ matrix.partition }} | |
| path: ui/test-results/test-results.json | |
| retention-days: 90 | |
| finalize: | |
| needs: | |
| - pre-test | |
| - tests | |
| runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }} | |
| timeout-minutes: 30 | |
| defaults: | |
| run: | |
| working-directory: ui | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - uses: ./.github/actions/setup-js | |
| - name: Retrieve Vault-hosted Secrets | |
| if: endsWith(github.repository, '-enterprise') | |
| id: vault | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| with: | |
| url: ${{ vars.CI_VAULT_URL }} | |
| method: ${{ vars.CI_VAULT_METHOD }} | |
| path: ${{ vars.CI_VAULT_PATH }} | |
| jwtGithubAudience: ${{ vars.CI_VAULT_AUD }} | |
| secrets: |- | |
| kv/data/teams/nomad/ui PERCY_TOKEN ; | |
| - name: Download all test results | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request' | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: test-results-* | |
| path: test-results | |
| - name: Combine test results for comparison | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request' | |
| run: node ../scripts/combine-ui-test-results.js | |
| - name: Upload combined results for comparison | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: test-results-${{ github.sha }} | |
| path: ui/combined-test-results.json | |
| retention-days: 90 | |
| - name: Upload Current PR results | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step | |
| path: ui/combined-test-results.json | |
| retention-days: 7 | |
| - name: finalize | |
| env: | |
| PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }} | |
| PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }} | |
| run: yarn percy build:finalize | |
| analyze-times: | |
| # TODO: temporary comment-out with hardcoded sha | |
| # needs: [tests, finalize] | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: ui | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| # Debug step to show environment | |
| - name: Debug environment | |
| run: | | |
| echo "GITHUB_SHA: ${{ github.sha }}" | |
| echo "GITHUB_EVENT_NAME: ${{ github.event_name }}" | |
| echo "GITHUB_REF: ${{ github.ref }}" | |
| echo "RUN_ID: ${{ github.run_id }}" | |
| # Try to list available artifacts first | |
| - name: List artifacts | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| // run_id: context.runId | |
| run_id: 12163157778 | |
| }); | |
| console.log('Available artifacts:'); | |
| console.log(JSON.stringify(artifacts.data, null, 2)); | |
| - name: Download current PR results | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| # name: test-results-${{ github.sha }} | |
| name: pr-test-results-fe7ca11e9afc42bc98d79fe521155a37634bd232 # TODO: temporary hardcoded sha from previous run | |
| path: ui | |
| run-id: 12163157778 | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # - name: Download historical results | |
| # uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| # with: | |
| # pattern: test-results-* | |
| # path: historical-results | |
| # merge-multiple: true | |
| # Download historical results from previous main branch runs | |
| - name: Download historical results | |
| uses: actions/github-script@v7 | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const historicalDir = path.join('ui', 'historical-results'); | |
| // Clean up any existing directory | |
| if (fs.existsSync(historicalDir)) { | |
| fs.rmSync(historicalDir, { recursive: true, force: true }); | |
| } | |
| fs.mkdirSync(historicalDir, { recursive: true }); | |
| const artifacts = await github.rest.actions.listArtifactsForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| per_page: 100 | |
| }); | |
| // Log out the names of each artifact | |
| console.log('Available artifacts:'); | |
| artifacts.data.artifacts.forEach(artifact => { | |
| console.log(`- ${artifact.name}`); | |
| }); | |
| const testArtifacts = artifacts.data.artifacts.filter(artifact => | |
| artifact.name.startsWith('test-results-') | |
| ); | |
| console.log(`Found ${testArtifacts.length} test result artifacts`); | |
| for (const artifact of testArtifacts) { | |
| try { | |
| console.log(`Downloading ${artifact.name}`); | |
| // Post a link to the artifact in the PR | |
| await github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `[View ${artifact.name} run](${artifact.archive_download_url})` | |
| }); | |
| // Create a temporary directory for this artifact | |
| const tempDir = path.join(historicalDir, `temp-${artifact.id}`); | |
| fs.mkdirSync(tempDir, { recursive: true }); | |
| // Download to temp directory | |
| await exec.exec('gh', [ | |
| 'run', | |
| 'download', | |
| '-n', | |
| artifact.name, | |
| '--repo', | |
| `${context.repo.owner}/${context.repo.repo}`, | |
| '--dir', | |
| tempDir, | |
| artifact.workflow_run.id.toString() | |
| ]); | |
| // Move the JSON file to the historical directory with a unique name | |
| const jsonFile = path.join(tempDir, 'test-results.json'); | |
| if (fs.existsSync(jsonFile)) { | |
| fs.renameSync( | |
| jsonFile, | |
| path.join(historicalDir, `${artifact.name}-${artifact.id}.json`) | |
| ); | |
| } | |
| // Clean up temp directory | |
| fs.rmSync(tempDir, { recursive: true, force: true }); | |
| console.log(`Successfully processed ${artifact.name}`); | |
| } catch (error) { | |
| console.log(`Error processing ${artifact.name}:`, error.message); | |
| } | |
| } | |
| # Debug what we got | |
| - name: Debug directories | |
| run: | | |
| echo "Current directory structure:" | |
| ls -la | |
| printf "\nHistorical results directory:\n" | |
| ls -la historical-results || echo "historical-results directory not found" | |
| - name: Analyze test times | |
| run: node ../scripts/analyze-ui-test-times.js | |
| - name: Comment PR | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json')); | |
| let body = `### Test Time Analysis\n\n`; | |
| body += `- Total Tests: ${analysis.summary.totalTests}\n`; | |
| body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`; | |
| body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`; | |
| if (analysis.testComparisons.length > 0) { | |
| body += `#### Most Significant Changes:\n\n`; | |
| analysis.testComparisons | |
| .filter(comp => comp.percentDiff != null) // Skip invalid comparisons | |
| .slice(0, 5) | |
| .forEach(comp => { | |
| body += `**${comp.name}**\n`; | |
| body += `- Current: ${comp.currentDuration}ms\n`; | |
| body += `- Historical Avg: ${comp.historicalAverage}ms\n`; | |
| body += `- Change: ${comp.percentDiff?.toFixed(1) || 'N/A'}%\n\n`; | |
| }); | |
| } | |
| await github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body | |
| }); | |
| permissions: | |
| contents: read | |
| id-token: write |