Skip to content

[ui, ci] UI test look-back and comparison for PRs #5027

[ui, ci] UI test look-back and comparison for PRs

[ui, ci] UI test look-back and comparison for PRs #5027

Workflow file for this run

name: test-ui
on:
pull_request:
paths:
- "ui/**"
push:
branches:
- main
- release/**
- test-ui
paths:
- "ui/**"
jobs:
pre-test:
runs-on: ubuntu-latest
timeout-minutes: 30
defaults:
run:
working-directory: ui
outputs:
nonce: ${{ steps.nonce.outputs.nonce }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/setup-js
- name: lint:js
run: yarn run lint:js
- name: lint:hbs
run: yarn run lint:hbs
- id: nonce
name: nonce
run: echo "nonce=${{ github.run_id }}-$(date +%s)" >> "$GITHUB_OUTPUT"
tests:
needs:
- pre-test
runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
timeout-minutes: 30
continue-on-error: true
defaults:
run:
working-directory: ui
strategy:
matrix:
partition: [1, 2, 3, 4]
split: [4]
# Note: If we ever change the number of partitions, we'll need to update the
# finalize.combine step to match
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/setup-js
- uses: browser-actions/setup-chrome@facf10a55b9caf92e0cc749b4f82bf8220989148 # v1.7.2
- name: Retrieve Vault-hosted Secrets
if: endsWith(github.repository, '-enterprise')
id: vault
uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
with:
url: ${{ vars.CI_VAULT_URL }}
method: ${{ vars.CI_VAULT_METHOD }}
path: ${{ vars.CI_VAULT_PATH }}
jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
secrets: |-
kv/data/teams/nomad/ui PERCY_TOKEN ;
- name: ember exam
id: exam
env:
PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
run: |
yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
# We have continue-on-error set to true, but we still want to alert the author if
# there are test failures or timeouts. Without it, we'll get errors in our output,
# but the workflow will still succeed / have a green checkmark.
- name: Express timeout failure
if: ${{ failure() }}
run: exit 1
- name: Check test status
if: steps.exam.outcome != 'success'
run: |
echo "Tests failed or timed out in partition ${{ matrix.partition }}"
exit 1
- name: Upload partition test results
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: test-results-${{ matrix.partition }}
path: ui/test-results/test-results.json
retention-days: 90
finalize:
needs:
- pre-test
- tests
runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
timeout-minutes: 30
defaults:
run:
working-directory: ui
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/setup-js
- name: Retrieve Vault-hosted Secrets
if: endsWith(github.repository, '-enterprise')
id: vault
uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
with:
url: ${{ vars.CI_VAULT_URL }}
method: ${{ vars.CI_VAULT_METHOD }}
path: ${{ vars.CI_VAULT_PATH }}
jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
secrets: |-
kv/data/teams/nomad/ui PERCY_TOKEN ;
- name: Download all test results
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: test-results-*
path: test-results
- name: Combine test results for comparison
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
run: node ../scripts/combine-ui-test-results.js
- name: Upload combined results for comparison
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: test-results-${{ github.sha }}
path: ui/combined-test-results.json
retention-days: 90
- name: Upload Current PR results
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step
path: ui/combined-test-results.json
retention-days: 7
- name: finalize
env:
PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
run: yarn percy build:finalize
analyze-times:
# TODO: temporary comment-out with hardcoded sha
# needs: [tests, finalize]
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
defaults:
run:
working-directory: ui
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
# Debug step to show environment
- name: Debug environment
run: |
echo "GITHUB_SHA: ${{ github.sha }}"
echo "GITHUB_EVENT_NAME: ${{ github.event_name }}"
echo "GITHUB_REF: ${{ github.ref }}"
echo "RUN_ID: ${{ github.run_id }}"
# Try to list available artifacts first
- name: List artifacts
uses: actions/github-script@v7
with:
script: |
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
// run_id: context.runId
run_id: 12163157778
});
console.log('Available artifacts:');
console.log(JSON.stringify(artifacts.data, null, 2));
- name: Download current PR results
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
# name: test-results-${{ github.sha }}
name: pr-test-results-fe7ca11e9afc42bc98d79fe521155a37634bd232 # TODO: temporary hardcoded sha from previous run
path: ui
run-id: 12163157778
github-token: ${{ secrets.GITHUB_TOKEN }}
# - name: Download historical results
# uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
# with:
# pattern: test-results-*
# path: historical-results
# merge-multiple: true
# Download historical results from previous main branch runs
- name: Download historical results
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const https = require('https');
// Create directory
const historicalDir = path.join('ui', 'historical-results');
fs.mkdirSync(historicalDir, { recursive: true });
// Helper function to download file
async function downloadFile(url, outputPath) {
return new Promise((resolve, reject) => {
const file = fs.createWriteStream(outputPath);
https.get(url, {
headers: {
'Authorization': `token ${process.env.GITHUB_TOKEN}`,
'Accept': 'application/vnd.github.v3.raw'
}
}, response => {
response.pipe(file);
file.on('finish', () => {
file.close();
resolve();
});
}).on('error', reject);
});
}
// Get the last 10 workflow runs on main
const runs = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'test-ui.yml',
branch: 'main',
per_page: 10
});
// Download artifacts from each run
for (const run of runs.data.workflow_runs) {
console.log(`Checking run ${run.id} from ${run.created_at}`);
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: run.id
});
for (const artifact of artifacts.data.artifacts) {
if (artifact.name.startsWith('test-results-')) {
console.log(`Downloading ${artifact.name} from run ${run.id}`);
// Get download URL
const download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: artifact.id,
archive_format: 'json' // Request JSON directly instead of ZIP
});
// Save the JSON file
fs.writeFileSync(
path.join(historicalDir, `${artifact.name}.json`),
JSON.stringify(download.data, null, 2)
);
}
}
}
console.log('Contents of historical-results:');
console.log(fs.readdirSync(historicalDir));
# Debug what we got
- name: Debug directories
run: |
echo "Current directory structure:"
ls -la
printf "\nHistorical results directory:\n"
ls -la historical-results || echo "historical-results directory not found"
- name: Analyze test times
run: node ../scripts/analyze-ui-test-times.js
- name: Comment PR
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const fs = require('fs');
const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));
console.log('=== analysis', analysis);
let body = `### Test Time Analysis\n\n`;
body += `- Total Tests: ${analysis.summary.totalTests}\n`;
body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;
if (analysis.testComparisons.length > 0) {
body += `#### Most Significant Changes:\n\n`;
analysis.testComparisons.slice(0, 5).forEach(comp => {
body += `**${comp.name}**\n`;
body += `- Current: ${comp.currentDuration}ms\n`;
body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
body += `- Change: ${comp.percentDiff.toFixed(1)}%\n\n`;
});
}
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body
});
permissions:
contents: read
id-token: write