Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ui, ci] UI test look-back and comparison for PRs #24604

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 64 additions & 1 deletion .github/workflows/test-ui.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
- pre-test
runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
timeout-minutes: 30
continue-on-error: true
defaults:
run:
working-directory: ui
Expand All @@ -61,15 +62,23 @@
secrets: |-
kv/data/teams/nomad/ui PERCY_TOKEN ;
- name: ember exam
id: exam
env:
PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
run: |
yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
continue-on-error: true
# We have continue-on-error set to true, but we still want to alert the author if
# there are test failures or timeouts. Without it, we'll get errors in our output,
# but the workflow will still succeed / have a green checkmark.
- name: Express timeout failure
if: ${{ failure() }}
run: exit 1
- name: Check test status
if: steps.exam.outcome != 'success'
run: |
echo "Tests failed or timed out in partition ${{ matrix.partition }}"
exit 1
- name: Upload partition test results
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
Expand Down Expand Up @@ -123,6 +132,60 @@
PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
run: yarn percy build:finalize

analyze-times:
needs: [tests, finalize]
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Fixed Show fixed Hide fixed

- uses: actions/download-artifact@v4
Fixed Show fixed Hide fixed
with:
name: test-results-${{ github.sha }}
path: ui

- name: Download historical results
uses: actions/download-artifact@v4
Fixed Show fixed Hide fixed
Fixed Show fixed Hide fixed
with:
pattern: test-results-*
path: historical-results
merge-multiple: true

- name: Analyze test times
run: node scripts/analyze-test-times.js

- name: Comment PR
uses: actions/github-script@v7
Fixed Show fixed Hide fixed
with:
script: |
const fs = require('fs');
const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));

console.log('=== analysis', analysis);

let body = `### Test Time Analysis\n\n`;
body += `- Total Tests: ${analysis.summary.totalTests}\n`;
body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;

if (analysis.testComparisons.length > 0) {
body += `#### Most Significant Changes:\n\n`;
analysis.testComparisons.slice(0, 5).forEach(comp => {
body += `**${comp.name}**\n`;
body += `- Current: ${comp.currentDuration}ms\n`;
body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
body += `- Change: ${comp.percentDiff.toFixed(1)}%\n\n`;
});
}

github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body
});

permissions:
contents: read
id-token: write
107 changes: 107 additions & 0 deletions scripts/analyze-ui-test-times.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#!/usr/bin/env node
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/

'use strict';
const fs = require('fs');

async function analyzeTestTimes() {
const currentResults = JSON.parse(
fs.readFileSync('../ui/combined-test-results.json')
);

// Create a map of test names to their durations
const currentTestTimes = new Map();
currentResults.tests.forEach(test => {
currentTestTimes.set(test.name, test.duration);
});

// Load and process historical results
const historicalAverages = new Map();
const historicalCounts = new Map();

// Read each historical result file
const historicalFiles = fs.readdirSync('../historical-results');
historicalFiles.forEach(file => {
const historical = JSON.parse(
fs.readFileSync(`../historical-results/${file}`)
);

historical.tests.forEach(test => {
const current = historicalAverages.get(test.name) || 0;
const count = historicalCounts.get(test.name) || 0;
historicalAverages.set(test.name, current + test.duration);
historicalCounts.set(test.name, count + 1);
});
});

// Calculate averages and compare
const analysis = {
timestamp: new Date().toISOString(),
sha: process.env.GITHUB_SHA,
summary: {
totalTests: currentResults.tests.length,
significantlySlower: 0,
significantlyFaster: 0
},
testComparisons: []
};

currentTestTimes.forEach((currentDuration, testName) => {
const totalHistorical = historicalAverages.get(testName) || 0;
const count = historicalCounts.get(testName) || 1;
const historicalAverage = totalHistorical / count;

// Consider a test significantly different if it's 25% slower/faster
const percentDiff = ((currentDuration - historicalAverage) / historicalAverage) * 100;

if (Math.abs(percentDiff) >= 25) {
analysis.testComparisons.push({
name: testName,
currentDuration,
historicalAverage,
percentDiff,
samples: count
});

if (percentDiff > 0) {
analysis.summary.significantlySlower++;
} else {
analysis.summary.significantlyFaster++;
}
}
});

// Sort by most significant differences first
analysis.testComparisons.sort((a, b) => Math.abs(b.percentDiff) - Math.abs(a.percentDiff));

// Write analysis results
fs.writeFileSync(
'../ui/test-time-analysis.json',
JSON.stringify(analysis, null, 2)
);

// Output summary to console for GitHub Actions
console.log('\nTest Time Analysis Summary:');
console.log(`Total Tests: ${analysis.summary.totalTests}`);
console.log(`Significantly Slower: ${analysis.summary.significantlySlower}`);
console.log(`Significantly Faster: ${analysis.summary.significantlyFaster}`);

if (analysis.testComparisons.length > 0) {
console.log('\nMost Significant Changes:');
analysis.testComparisons.slice(0, 5).forEach(comp => {
console.log(`${comp.name}:`);
console.log(` Current: ${comp.currentDuration}ms`);
console.log(` Historical Avg: ${comp.historicalAverage}ms`);
console.log(` Change: ${comp.percentDiff.toFixed(1)}%`);
});
}
}

if (require.main === module) {
analyzeTestTimes();
}

module.exports = analyzeTestTimes;
Loading