[ui, ci] UI test look-back and comparison for PRs

.github/workflows/test-ui.yml

@@ -36,6 +36,7 @@
       - pre-test
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     timeout-minutes: 30
+    continue-on-error: true
     defaults:
       run:
         working-directory: ui
@@ -61,15 +62,23 @@
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: ember exam
+        id: exam
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: |
           yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
-        continue-on-error: true
+      # We have continue-on-error set to true, but we still want to alert the author if
+      # there are test failures or timeouts. Without it, we'll get errors in our output,
+      # but the workflow will still succeed / have a green checkmark.
       - name: Express timeout failure
         if: ${{ failure() }}
         run: exit 1
+      - name: Check test status
+        if: steps.exam.outcome != 'success'
+        run: |
+          echo "Tests failed or timed out in partition ${{ matrix.partition }}"
+          exit 1
       - name: Upload partition test results
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
@@ -123,6 +132,60 @@
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: yarn percy build:finalize
+
+  analyze-times:
+    needs: [tests, finalize]
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: test-results-${{ github.sha }}
+          path: ui
+
+      - name: Download historical results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: test-results-*
+          path: historical-results
+          merge-multiple: true
+
+      - name: Analyze test times
+        run: node scripts/analyze-test-times.js
+
+      - name: Comment PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));
+
+            console.log('=== analysis', analysis);
+
+            let body = `### Test Time Analysis\n\n`;
+            body += `- Total Tests: ${analysis.summary.totalTests}\n`;
+            body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
+            body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;
+
+            if (analysis.testComparisons.length > 0) {
+              body += `#### Most Significant Changes:\n\n`;
+              analysis.testComparisons.slice(0, 5).forEach(comp => {
+                body += `**${comp.name}**\n`;
+                body += `- Current: ${comp.currentDuration}ms\n`;
+                body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
+                body += `- Change: ${comp.percentDiff.toFixed(1)}%\n\n`;
+              });
+            }
+
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            });
+
 permissions:
   contents: read
   id-token: write

scripts/analyze-ui-test-times.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+'use strict';
+const fs = require('fs');
+
+async function analyzeTestTimes() {
+  const currentResults = JSON.parse(
+    fs.readFileSync('../ui/combined-test-results.json')
+  );
+
+  // Create a map of test names to their durations
+  const currentTestTimes = new Map();
+  currentResults.tests.forEach(test => {
+    currentTestTimes.set(test.name, test.duration);
+  });
+
+  // Load and process historical results
+  const historicalAverages = new Map();
+  const historicalCounts = new Map();
+
+  // Read each historical result file
+  const historicalFiles = fs.readdirSync('../historical-results');
+  historicalFiles.forEach(file => {
+    const historical = JSON.parse(
+      fs.readFileSync(`../historical-results/${file}`)
+    );
+
+    historical.tests.forEach(test => {
+      const current = historicalAverages.get(test.name) || 0;
+      const count = historicalCounts.get(test.name) || 0;
+      historicalAverages.set(test.name, current + test.duration);
+      historicalCounts.set(test.name, count + 1);
+    });
+  });
+
+  // Calculate averages and compare
+  const analysis = {
+    timestamp: new Date().toISOString(),
+    sha: process.env.GITHUB_SHA,
+    summary: {
+      totalTests: currentResults.tests.length,
+      significantlySlower: 0,
+      significantlyFaster: 0
+    },
+    testComparisons: []
+  };
+
+  currentTestTimes.forEach((currentDuration, testName) => {
+    const totalHistorical = historicalAverages.get(testName) || 0;
+    const count = historicalCounts.get(testName) || 1;
+    const historicalAverage = totalHistorical / count;
+
+    // Consider a test significantly different if it's 25% slower/faster
+    const percentDiff = ((currentDuration - historicalAverage) / historicalAverage) * 100;
+
+    if (Math.abs(percentDiff) >= 25) {
+      analysis.testComparisons.push({
+        name: testName,
+        currentDuration,
+        historicalAverage,
+        percentDiff,
+        samples: count
+      });
+
+      if (percentDiff > 0) {
+        analysis.summary.significantlySlower++;
+      } else {
+        analysis.summary.significantlyFaster++;
+      }
+    }
+  });
+
+  // Sort by most significant differences first
+  analysis.testComparisons.sort((a, b) => Math.abs(b.percentDiff) - Math.abs(a.percentDiff));
+
+  // Write analysis results
+  fs.writeFileSync(
+    '../ui/test-time-analysis.json',
+    JSON.stringify(analysis, null, 2)
+  );
+
+  // Output summary to console for GitHub Actions
+  console.log('\nTest Time Analysis Summary:');
+  console.log(`Total Tests: ${analysis.summary.totalTests}`);
+  console.log(`Significantly Slower: ${analysis.summary.significantlySlower}`);
+  console.log(`Significantly Faster: ${analysis.summary.significantlyFaster}`);
+
+  if (analysis.testComparisons.length > 0) {
+    console.log('\nMost Significant Changes:');
+    analysis.testComparisons.slice(0, 5).forEach(comp => {
+      console.log(`${comp.name}:`);
+      console.log(`  Current: ${comp.currentDuration}ms`);
+      console.log(`  Historical Avg: ${comp.historicalAverage}ms`);
+      console.log(`  Change: ${comp.percentDiff.toFixed(1)}%`);
+    });
+  }
+}
+
+if (require.main === module) {
+  analyzeTestTimes();
+}
+
+module.exports = analyzeTestTimes;