From 7b8ca143152b576e20a8e6051a2ac09d75e0621c Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Wed, 4 Dec 2024 09:37:27 -0500
Subject: [PATCH 1/8] Continue-on-error placement was causing failing exams to
 get a green checkmark

---
 .github/workflows/test-ui.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
index 5478460877c..8b72f8ddb90 100644
--- a/.github/workflows/test-ui.yml
+++ b/.github/workflows/test-ui.yml
@@ -36,6 +36,7 @@ jobs:
       - pre-test
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     timeout-minutes: 30
+    continue-on-error: true
     defaults:
       run:
         working-directory: ui
@@ -61,15 +62,23 @@ jobs:
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: ember exam
+        id: exam
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: |
           yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
-        continue-on-error: true
+      # We have continue-on-error set to true, but we still want to alert the author if
+      # there are test failures or timeouts. Without it, we'll get errors in our output,
+      # but the workflow will still succeed / have a green checkmark.
       - name: Express timeout failure
         if: ${{ failure() }}
         run: exit 1
+      - name: Check test status
+        if: steps.exam.outcome != 'success'
+        run: |
+          echo "Tests failed or timed out in partition ${{ matrix.partition }}"
+          exit 1
       - name: Upload partition test results
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

From bea9b3a5829ec9d8ccdeb3545ceff84ba7cc8bf1 Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Wed, 4 Dec 2024 09:50:35 -0500
Subject: [PATCH 2/8] First pass at lookback step

---
 .github/workflows/test-ui.yml    |  54 ++++++++++++++++
 scripts/analyze-ui-test-times.js | 107 +++++++++++++++++++++++++++++++
 2 files changed, 161 insertions(+)
 create mode 100644 scripts/analyze-ui-test-times.js

diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
index 8b72f8ddb90..6ac8c96db00 100644
--- a/.github/workflows/test-ui.yml
+++ b/.github/workflows/test-ui.yml
@@ -132,6 +132,60 @@ jobs:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: yarn percy build:finalize
+
+  analyze-times:
+    needs: [tests, finalize]
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: actions/download-artifact@v4
+        with:
+          name: test-results-${{ github.sha }}
+          path: ui
+          
+      - name: Download historical results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: test-results-*
+          path: historical-results
+          merge-multiple: true
+          
+      - name: Analyze test times
+        run: node scripts/analyze-test-times.js
+
+      - name: Comment PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));
+
+            console.log('=== analysis', analysis);
+            
+            let body = `### Test Time Analysis\n\n`;
+            body += `- Total Tests: ${analysis.summary.totalTests}\n`;
+            body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
+            body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;
+            
+            if (analysis.testComparisons.length > 0) {
+              body += `#### Most Significant Changes:\n\n`;
+              analysis.testComparisons.slice(0, 5).forEach(comp => {
+                body += `**${comp.name}**\n`;
+                body += `- Current: ${comp.currentDuration}ms\n`;
+                body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
+                body += `- Change: ${comp.percentDiff.toFixed(1)}%\n\n`;
+              });
+            }
+            
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            });
+
 permissions:
   contents: read
   id-token: write
diff --git a/scripts/analyze-ui-test-times.js b/scripts/analyze-ui-test-times.js
new file mode 100644
index 00000000000..ebc394ef7db
--- /dev/null
+++ b/scripts/analyze-ui-test-times.js
@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+'use strict';
+const fs = require('fs');
+
+async function analyzeTestTimes() {
+  const currentResults = JSON.parse(
+    fs.readFileSync('../ui/combined-test-results.json')
+  );
+
+  // Create a map of test names to their durations
+  const currentTestTimes = new Map();
+  currentResults.tests.forEach(test => {
+    currentTestTimes.set(test.name, test.duration);
+  });
+
+  // Load and process historical results
+  const historicalAverages = new Map();
+  const historicalCounts = new Map();
+
+  // Read each historical result file
+  const historicalFiles = fs.readdirSync('../historical-results');
+  historicalFiles.forEach(file => {
+    const historical = JSON.parse(
+      fs.readFileSync(`../historical-results/${file}`)
+    );
+
+    historical.tests.forEach(test => {
+      const current = historicalAverages.get(test.name) || 0;
+      const count = historicalCounts.get(test.name) || 0;
+      historicalAverages.set(test.name, current + test.duration);
+      historicalCounts.set(test.name, count + 1);
+    });
+  });
+
+  // Calculate averages and compare
+  const analysis = {
+    timestamp: new Date().toISOString(),
+    sha: process.env.GITHUB_SHA,
+    summary: {
+      totalTests: currentResults.tests.length,
+      significantlySlower: 0,
+      significantlyFaster: 0
+    },
+    testComparisons: []
+  };
+
+  currentTestTimes.forEach((currentDuration, testName) => {
+    const totalHistorical = historicalAverages.get(testName) || 0;
+    const count = historicalCounts.get(testName) || 1;
+    const historicalAverage = totalHistorical / count;
+
+    // Consider a test significantly different if it's 25% slower/faster
+    const percentDiff = ((currentDuration - historicalAverage) / historicalAverage) * 100;
+    
+    if (Math.abs(percentDiff) >= 25) {
+      analysis.testComparisons.push({
+        name: testName,
+        currentDuration,
+        historicalAverage,
+        percentDiff,
+        samples: count
+      });
+
+      if (percentDiff > 0) {
+        analysis.summary.significantlySlower++;
+      } else {
+        analysis.summary.significantlyFaster++;
+      }
+    }
+  });
+
+  // Sort by most significant differences first
+  analysis.testComparisons.sort((a, b) => Math.abs(b.percentDiff) - Math.abs(a.percentDiff));
+
+  // Write analysis results
+  fs.writeFileSync(
+    '../ui/test-time-analysis.json',
+    JSON.stringify(analysis, null, 2)
+  );
+
+  // Output summary to console for GitHub Actions
+  console.log('\nTest Time Analysis Summary:');
+  console.log(`Total Tests: ${analysis.summary.totalTests}`);
+  console.log(`Significantly Slower: ${analysis.summary.significantlySlower}`);
+  console.log(`Significantly Faster: ${analysis.summary.significantlyFaster}`);
+  
+  if (analysis.testComparisons.length > 0) {
+    console.log('\nMost Significant Changes:');
+    analysis.testComparisons.slice(0, 5).forEach(comp => {
+      console.log(`${comp.name}:`);
+      console.log(`  Current: ${comp.currentDuration}ms`);
+      console.log(`  Historical Avg: ${comp.historicalAverage}ms`);
+      console.log(`  Change: ${comp.percentDiff.toFixed(1)}%`);
+    });
+  }
+}
+
+if (require.main === module) {
+  analyzeTestTimes();
+}
+
+module.exports = analyzeTestTimes;

From 4b984adfae4bbd0e06ef09702f9eb5d0ebba50df Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Wed, 4 Dec 2024 10:00:51 -0500
Subject: [PATCH 3/8] Pinned github action artifacts

---
 .github/workflows/test-ui.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
index 6ac8c96db00..15d88301dc2 100644
--- a/.github/workflows/test-ui.yml
+++ b/.github/workflows/test-ui.yml
@@ -138,15 +138,14 @@ jobs:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      
-      - uses: actions/download-artifact@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: test-results-${{ github.sha }}
           path: ui
           
       - name: Download historical results
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           pattern: test-results-*
           path: historical-results
@@ -156,7 +155,7 @@ jobs:
         run: node scripts/analyze-test-times.js
 
       - name: Comment PR
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           script: |
             const fs = require('fs');

From cdbff0bed96b31dfc44ef6855c1747b626c658fc Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Wed, 4 Dec 2024 10:03:21 -0500
Subject: [PATCH 4/8] no-op to trigger ui tests

---
 ui/app/index.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ui/app/index.html b/ui/app/index.html
index df5eb739390..9d49fe8a579 100644
--- a/ui/app/index.html
+++ b/ui/app/index.html
@@ -24,7 +24,6 @@
     {{content-for "body"}}
 
     <script src="{{rootURL}}assets/vendor.js"></script>
-
     <script src="{{rootURL}}assets/nomad-ui.js"></script>
 
     {{content-for "body-footer"}}

From 65fe72a47eff8a3e051433fc26096db27666b99b Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Wed, 4 Dec 2024 10:37:07 -0500
Subject: [PATCH 5/8] Retain currently-run PR results for 7 days and
 differentiate between them and main

---
 .github/workflows/test-ui.yml    | 15 ++++++++++++---
 scripts/analyze-ui-test-times.js | 21 ++++++++++++++-------
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
index 15d88301dc2..a52c0e7b61c 100644
--- a/.github/workflows/test-ui.yml
+++ b/.github/workflows/test-ui.yml
@@ -80,7 +80,7 @@ jobs:
           echo "Tests failed or timed out in partition ${{ matrix.partition }}"
           exit 1
       - name: Upload partition test results
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: test-results-${{ matrix.partition }}
@@ -110,15 +110,16 @@ jobs:
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: Download all test results
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           pattern: test-results-*
           path: test-results
 
       - name: Combine test results for comparison
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
         run: node ../scripts/combine-ui-test-results.js
+
       - name: Upload combined results for comparison
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
@@ -127,6 +128,14 @@ jobs:
           path: ui/combined-test-results.json
           retention-days: 90
 
+      - name: Upload Current PR results
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step
+          path: ui/combined-test-results.json
+          retention-days: 7
+
       - name: finalize
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
diff --git a/scripts/analyze-ui-test-times.js b/scripts/analyze-ui-test-times.js
index ebc394ef7db..bad4c738372 100644
--- a/scripts/analyze-ui-test-times.js
+++ b/scripts/analyze-ui-test-times.js
@@ -19,22 +19,29 @@ async function analyzeTestTimes() {
   });
 
   // Load and process historical results
+  console.log('[analyze-test-times] Processing historical results...\n');
   const historicalAverages = new Map();
   const historicalCounts = new Map();
 
   // Read each historical result file
+  console.log('[analyze-test-times] Reading historical results directory...\n');
   const historicalFiles = fs.readdirSync('../historical-results');
-  historicalFiles.forEach(file => {
+  historicalFiles.forEach((file, index) => {
+    console.log(`[analyze-test-times] Reading ${file} (${index + 1} of ${historicalFiles.length})...`);
     const historical = JSON.parse(
       fs.readFileSync(`../historical-results/${file}`)
     );
 
-    historical.tests.forEach(test => {
-      const current = historicalAverages.get(test.name) || 0;
-      const count = historicalCounts.get(test.name) || 0;
-      historicalAverages.set(test.name, current + test.duration);
-      historicalCounts.set(test.name, count + 1);
-    });
+    if (historical.summary.failed === 0) {
+      historical.tests.forEach(test => {
+        const current = historicalAverages.get(test.name) || 0;
+        const count = historicalCounts.get(test.name) || 0;
+        historicalAverages.set(test.name, current + test.duration);
+        historicalCounts.set(test.name, count + 1);
+      });
+    } else {
+      console.log(`[analyze-test-times] Skipping ${file} because it has failed tests`);
+    }
   });
 
   // Calculate averages and compare

From 90eae68d1bcc6c3f6e00c62838d03ead939868e4 Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Wed, 4 Dec 2024 10:57:40 -0500
Subject: [PATCH 6/8] Debugging run

---
 .github/workflows/test-ui.yml    | 176 +++++++++++++++++++++++++++----
 scripts/analyze-ui-test-times.js |  91 ++++++++++++----
 2 files changed, 228 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
index a52c0e7b61c..359a248c054 100644
--- a/.github/workflows/test-ui.yml
+++ b/.github/workflows/test-ui.yml
@@ -127,6 +127,11 @@ jobs:
           name: test-results-${{ github.sha }}
           path: ui/combined-test-results.json
           retention-days: 90
+      - name: Delete partition test results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
+        uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0
+        with:
+          name: test-results-*
 
       - name: Upload Current PR results
         if: github.event_name == 'pull_request'
@@ -134,7 +139,7 @@ jobs:
         with:
           name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step
           path: ui/combined-test-results.json
-          retention-days: 7
+          retention-days: 1
 
       - name: finalize
         env:
@@ -143,25 +148,156 @@ jobs:
         run: yarn percy build:finalize
 
   analyze-times:
-    needs: [tests, finalize]
+    # TODO: temporary comment-out with hardcoded sha
+    # needs: [tests, finalize]
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ui
+
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+
+      # Debug step to show environment
+      - name: Debug environment
+        run: |
+          echo "GITHUB_SHA: ${{ github.sha }}"
+          echo "GITHUB_EVENT_NAME: ${{ github.event_name }}"
+          echo "GITHUB_REF: ${{ github.ref }}"
+          echo "RUN_ID: ${{ github.run_id }}"
+          
+      # Try to list available artifacts first
+      - name: List artifacts
+        uses: actions/github-script@v7
         with:
-          name: test-results-${{ github.sha }}
+          script: |
+            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              // run_id: context.runId
+              run_id: 12163157778
+            });
+            console.log('Available artifacts:');
+            console.log(JSON.stringify(artifacts.data, null, 2));
+
+      - name: Download current PR results
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          # name: test-results-${{ github.sha }}
+          name: pr-test-results-fe7ca11e9afc42bc98d79fe521155a37634bd232 # TODO: temporary hardcoded sha from previous run
           path: ui
-          
+          run-id: 12163157778
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # - name: Download historical results
+      #   uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      #   with:
+      #     pattern: test-results-*
+      #     path: historical-results
+      #     merge-multiple: true
+
+      # Download historical results from previous main branch runs
       - name: Download historical results
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/github-script@v7
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          pattern: test-results-*
-          path: historical-results
-          merge-multiple: true
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const historicalDir = path.join('ui', 'historical-results');
+            // Clean up any existing directory
+            if (fs.existsSync(historicalDir)) {
+              fs.rmSync(historicalDir, { recursive: true, force: true });
+            }
+            fs.mkdirSync(historicalDir, { recursive: true });
+
+            const artifacts = await github.rest.actions.listArtifactsForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              per_page: 100
+            });
+
+            // Log out the names of each artifact
+            console.log('Available artifacts:');
+            artifacts.data.artifacts.forEach(artifact => {
+              console.log(`- ${artifact.name}`);
+            });
+
+            const testArtifacts = artifacts.data.artifacts.filter(artifact => 
+              artifact.name.startsWith('pr-test-results-')
+            );
+
+            console.log(`Found ${testArtifacts.length} test result artifacts`);
+
+            for (const artifact of testArtifacts) {
+              try {
+                console.log(`Downloading ${artifact.name}`);
+                
+                // Create a temporary directory for this artifact
+                const tempDir = path.join(historicalDir, `temp-${artifact.id}`);
+                fs.mkdirSync(tempDir, { recursive: true });
+                
+                try {
+                  // Download to temp directory
+                  await exec.exec('gh', [
+                    'run',
+                    'download',
+                    '-n',
+                    artifact.name,
+                    '--repo',
+                    `${context.repo.owner}/${context.repo.repo}`,
+                    '--dir',
+                    tempDir,
+                    artifact.workflow_run.id.toString()
+                  ]);
+                  
+                  // Move the JSON file to the historical directory with a unique name
+                  const jsonFile = path.join(tempDir, 'combined-test-results.json');
+                  if (fs.existsSync(jsonFile)) {
+                    fs.renameSync(
+                      jsonFile,
+                      path.join(historicalDir, `${artifact.name}-${artifact.id}.json`)
+                    );
+                    console.log(`Successfully processed ${artifact.name}`);
+                  } else {
+                    const files = fs.readdirSync(tempDir);
+                    console.log(`Warning: No test results JSON found in ${artifact.name}. Found files:`, files);
+                    console.log(`Warning: No combined-test-results.json found in ${artifact.name}`);
+                  }
+                } finally {
+                  // Always clean up temp directory
+                  if (fs.existsSync(tempDir)) {
+                    fs.rmSync(tempDir, { recursive: true, force: true });
+                  }
+                }
+              } catch (error) {
+                console.log(`Error processing ${artifact.name}:`, error.message);
+                // Continue with next artifact
+              }
+            }
+      
+      # Debug what we got
+      - name: Debug directories
+        run: |
+          echo "Current directory structure:"
+          ls -la
+          printf "\nHistorical results directory:\n"
+          ls -la historical-results || echo "historical-results directory not found"
           
+          cd historical-results
+          echo -e "\nContents of each file (first 10 lines):"
+          for file in *.json; do
+            if [ -f "$file" ]; then
+              echo -e "\n=== $file ==="
+              head -n 10 "$file"
+            fi
+          done
+
       - name: Analyze test times
-        run: node scripts/analyze-test-times.js
+        run: node ../scripts/analyze-ui-test-times.js
 
       - name: Comment PR
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -169,8 +305,6 @@ jobs:
           script: |
             const fs = require('fs');
             const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));
-
-            console.log('=== analysis', analysis);
             
             let body = `### Test Time Analysis\n\n`;
             body += `- Total Tests: ${analysis.summary.totalTests}\n`;
@@ -179,15 +313,18 @@ jobs:
             
             if (analysis.testComparisons.length > 0) {
               body += `#### Most Significant Changes:\n\n`;
-              analysis.testComparisons.slice(0, 5).forEach(comp => {
-                body += `**${comp.name}**\n`;
-                body += `- Current: ${comp.currentDuration}ms\n`;
-                body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
-                body += `- Change: ${comp.percentDiff.toFixed(1)}%\n\n`;
-              });
+              analysis.testComparisons
+                .filter(comp => comp.percentDiff != null)  // Skip invalid comparisons
+                .slice(0, 5)
+                .forEach(comp => {
+                  body += `**${comp.name}**\n`;
+                  body += `- Current: ${comp.currentDuration}ms\n`;
+                  body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
+                  body += `- Change: ${comp.percentDiff?.toFixed(1) || 'N/A'}%\n\n`;
+                });
             }
             
-            github.rest.issues.createComment({
+            await github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
@@ -197,3 +334,4 @@ jobs:
 permissions:
   contents: read
   id-token: write
+  pull-requests: write
diff --git a/scripts/analyze-ui-test-times.js b/scripts/analyze-ui-test-times.js
index bad4c738372..42b0364b0ce 100644
--- a/scripts/analyze-ui-test-times.js
+++ b/scripts/analyze-ui-test-times.js
@@ -6,10 +6,11 @@
 
 'use strict';
 const fs = require('fs');
+const path = require('path');
 
 async function analyzeTestTimes() {
   const currentResults = JSON.parse(
-    fs.readFileSync('../ui/combined-test-results.json')
+    fs.readFileSync('combined-test-results.json')
   );
 
   // Create a map of test names to their durations
@@ -18,21 +19,47 @@ async function analyzeTestTimes() {
     currentTestTimes.set(test.name, test.duration);
   });
 
-  // Load and process historical results
-  console.log('[analyze-test-times] Processing historical results...\n');
-  const historicalAverages = new Map();
-  const historicalCounts = new Map();
-
-  // Read each historical result file
-  console.log('[analyze-test-times] Reading historical results directory...\n');
-  const historicalFiles = fs.readdirSync('../historical-results');
-  historicalFiles.forEach((file, index) => {
-    console.log(`[analyze-test-times] Reading ${file} (${index + 1} of ${historicalFiles.length})...`);
-    const historical = JSON.parse(
-      fs.readFileSync(`../historical-results/${file}`)
-    );
-
-    if (historical.summary.failed === 0) {
+// Load and process historical results
+console.log('[analyze-test-times] Processing historical results...\n');
+const historicalAverages = new Map();
+const historicalCounts = new Map();
+
+// Read each historical result file
+console.log('[analyze-test-times] Reading historical results directory...\n');
+const historicalDir = 'historical-results';
+const historicalFiles = fs.readdirSync(historicalDir)
+  .filter(file => file.endsWith('.json'));
+
+console.log(`[analyze-test-times] Found ${historicalFiles.length} JSON files`);
+
+  // Debug: Show content of first file
+  if (historicalFiles.length > 0) {
+    const firstFile = fs.readFileSync(path.join(historicalDir, historicalFiles[0]), 'utf8');
+    console.log('\n[analyze-test-times] First file content sample:');
+    console.log(firstFile.substring(0, 500) + '...');
+    console.log('\n[analyze-test-times] First file parsed:');
+    const parsed = JSON.parse(firstFile);
+    console.log(JSON.stringify(parsed, null, 2).substring(0, 500) + '...');
+  }
+
+historicalFiles.forEach((file, index) => {
+  console.log(`[analyze-test-times] Reading ${file} (${index + 1} of ${historicalFiles.length})...`);
+  try {
+    const historical = JSON.parse(fs.readFileSync(path.join(historicalDir, file), 'utf8'));
+    
+    // Debug output
+    console.log(`[analyze-test-times] File ${file}:`);
+    console.log(`  - Has summary: ${!!historical.summary}`);
+    if (historical.summary) {
+      console.log(`  - Failed tests: ${historical.summary.failed}`);
+      console.log(`  - Total tests: ${historical.summary.total}`);
+    }
+    console.log(`  - Has tests array: ${!!historical.tests}`);
+    if (historical.tests) {
+      console.log(`  - Number of tests: ${historical.tests.length}`);
+    }
+
+    if (historical.summary && historical.summary.failed === 0) {
       historical.tests.forEach(test => {
         const current = historicalAverages.get(test.name) || 0;
         const count = historicalCounts.get(test.name) || 0;
@@ -40,10 +67,25 @@ async function analyzeTestTimes() {
         historicalCounts.set(test.name, count + 1);
       });
     } else {
-      console.log(`[analyze-test-times] Skipping ${file} because it has failed tests`);
+      console.log(`[analyze-test-times] Skipping ${file} because it has failed tests or invalid format`);
     }
-  });
+  } catch (error) {
+    console.log(`[analyze-test-times] Error processing ${file}:`, error.message);
+  }
+});
 
+// Debug output after processing
+console.log('\n[analyze-test-times] Processing complete');
+console.log(`Total unique tests found: ${historicalAverages.size}`);
+if (historicalAverages.size > 0) {
+  console.log('Sample of processed tests:');
+  let i = 0;
+  for (const [name, total] of historicalAverages) {
+    if (i++ >= 3) break;
+    const count = historicalCounts.get(name);
+    console.log(`- ${name}: ${total}ms total, ${count} samples`);
+  }
+}
   // Calculate averages and compare
   const analysis = {
     timestamp: new Date().toISOString(),
@@ -56,20 +98,29 @@ async function analyzeTestTimes() {
     testComparisons: []
   };
 
+  // In the analyzeTestTimes function:
+  console.log('[analyze-test-times] Comparing current test times with historical averages...');
   currentTestTimes.forEach((currentDuration, testName) => {
     const totalHistorical = historicalAverages.get(testName) || 0;
     const count = historicalCounts.get(testName) || 1;
     const historicalAverage = totalHistorical / count;
 
+    // Skip tests with no historical data
+    if (historicalAverage === 0) {
+      console.log(`[analyze-test-times] Skipping ${testName} - no historical data`);
+      return;
+    }
+
     // Consider a test significantly different if it's 25% slower/faster
     const percentDiff = ((currentDuration - historicalAverage) / historicalAverage) * 100;
     
     if (Math.abs(percentDiff) >= 25) {
+      console.log(`[analyze-test-times] Found significant difference in ${testName}: ${percentDiff.toFixed(1)}% change`);
       analysis.testComparisons.push({
         name: testName,
         currentDuration,
         historicalAverage,
-        percentDiff,
+        percentDiff,  // This will now always be a number
         samples: count
       });
 
@@ -86,7 +137,7 @@ async function analyzeTestTimes() {
 
   // Write analysis results
   fs.writeFileSync(
-    '../ui/test-time-analysis.json',
+    'test-time-analysis.json',
     JSON.stringify(analysis, null, 2)
   );
 

From aed33ae4d587bb7597233ebeda535ac647c4d83f Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Thu, 5 Dec 2024 10:45:27 -0500
Subject: [PATCH 7/8] A little debugging for historical timings

---
 scripts/analyze-ui-test-times.js | 69 ++++++++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 3 deletions(-)

diff --git a/scripts/analyze-ui-test-times.js b/scripts/analyze-ui-test-times.js
index 42b0364b0ce..e3d9cc422d8 100644
--- a/scripts/analyze-ui-test-times.js
+++ b/scripts/analyze-ui-test-times.js
@@ -23,6 +23,8 @@ async function analyzeTestTimes() {
 console.log('[analyze-test-times] Processing historical results...\n');
 const historicalAverages = new Map();
 const historicalCounts = new Map();
+const variablesTimings = new Set();
+const jobACLDisabledTimings = new Set();
 
 // Read each historical result file
 console.log('[analyze-test-times] Reading historical results directory...\n');
@@ -65,6 +67,15 @@ historicalFiles.forEach((file, index) => {
         const count = historicalCounts.get(test.name) || 0;
         historicalAverages.set(test.name, current + test.duration);
         historicalCounts.set(test.name, count + 1);
+        // Log out all timings for "Acceptance | variables > Job Variables Page: If the user has variable read access, but no variables, the subnav exists but contains only a message"
+        if (test.name === "Acceptance | variables > Job Variables Page: If the user has variable read access, but no variables, the subnav exists but contains only a message") {
+          console.log(`[analyze-test-times] Timings for ${test.name}: ${test.duration}`);
+          variablesTimings.add(test.duration);
+        }
+        if (test.name === "Unit | Ability | job: it permits job run when ACLs are disabled") {
+          console.log(`[analyze-test-times] Timings for ${test.name}: ${test.duration}`);
+          jobACLDisabledTimings.add(test.duration);
+        }
       });
     } else {
       console.log(`[analyze-test-times] Skipping ${file} because it has failed tests or invalid format`);
@@ -86,7 +97,58 @@ if (historicalAverages.size > 0) {
     console.log(`- ${name}: ${total}ms total, ${count} samples`);
   }
 }
-  // Calculate averages and compare
+// Log out variablesTimings
+console.log(`[analyze-test-times] Variables timings: ${Array.from(variablesTimings).join(', ')}`);
+console.log(`[analyze-test-times] Job ACL disabled timings: ${Array.from(jobACLDisabledTimings).join(', ')}`);
+
+// After processing all files, show statistics
+console.log('\n[analyze-test-times] Sample count analysis:');
+console.log(`Total unique tests found: ${historicalAverages.size}`);
+
+
+// Sort tests by sample count to see which ones might be missing data
+const testStats = Array.from(historicalCounts.entries())
+  .sort((a, b) => b[1] - a[1]); // Sort by count, descending
+
+console.log('\nSample counts per test:');
+console.log('Format: Test name (count/total files)');
+
+// Create a Map to store timings for each test
+const testTimings = new Map();
+
+// First pass: collect all timings
+historicalFiles.forEach(file => {
+  try {
+    const historical = JSON.parse(fs.readFileSync(path.join(historicalDir, file), 'utf8'));
+    if (historical.tests) {
+      historical.tests.forEach(test => {
+        if (!testTimings.has(test.name)) {
+          testTimings.set(test.name, []);
+        }
+        testTimings.get(test.name).push(test.duration);
+      });
+    }
+  } catch (error) {
+    console.log(`Error reading file ${file}:`, error.message);
+  }
+});
+
+// Second pass: display results
+testStats.forEach(([testName, count]) => {
+  const percentage = ((count / historicalFiles.length) * 100).toFixed(1);
+  const timings = testTimings.get(testName) || [];
+  const timingsStr = timings.join(', ');
+  
+  if (count < historicalFiles.length) {
+    console.log(`⚠️  ${testName}: ${count}/${historicalFiles.length} (${percentage}%)`);
+    console.log(`   Timings: [${timingsStr}]`);
+  } else {
+    console.log(`✓ ${testName}: ${count}/${historicalFiles.length} (${percentage}%)`);
+    console.log(`   Timings: [${timingsStr}]`);
+  }
+});
+
+// Calculate averages and compare
   const analysis = {
     timestamp: new Date().toISOString(),
     sha: process.env.GITHUB_SHA,
@@ -102,11 +164,12 @@ if (historicalAverages.size > 0) {
   console.log('[analyze-test-times] Comparing current test times with historical averages...');
   currentTestTimes.forEach((currentDuration, testName) => {
     const totalHistorical = historicalAverages.get(testName) || 0;
-    const count = historicalCounts.get(testName) || 1;
+    const count = historicalCounts.get(testName);
     const historicalAverage = totalHistorical / count;
 
     // Skip tests with no historical data
-    if (historicalAverage === 0) {
+    // if (historicalAverage === 0) {
+    if (!count) {
       console.log(`[analyze-test-times] Skipping ${testName} - no historical data`);
       return;
     }

From b8f9c286af42bb0962888a32c0d1170d63165d1b Mon Sep 17 00:00:00 2001
From: Phil Renaud <phil@riotindustries.com>
Date: Thu, 5 Dec 2024 11:27:17 -0500
Subject: [PATCH 8/8] OK but what if I made a data visualization for very good
 reasons

---
 .github/workflows/test-ui.yml    |  1 +
 scripts/analyze-ui-test-times.js | 23 ++++++++++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
index 359a248c054..bc1a5a9ecc2 100644
--- a/.github/workflows/test-ui.yml
+++ b/.github/workflows/test-ui.yml
@@ -321,6 +321,7 @@ jobs:
                   body += `- Current: ${comp.currentDuration}ms\n`;
                   body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
                   body += `- Change: ${comp.percentDiff?.toFixed(1) || 'N/A'}%\n\n`;
+                  body += `- Trend: ${comp.trend}\n\n`;
                 });
             }
             
diff --git a/scripts/analyze-ui-test-times.js b/scripts/analyze-ui-test-times.js
index e3d9cc422d8..86643859b4d 100644
--- a/scripts/analyze-ui-test-times.js
+++ b/scripts/analyze-ui-test-times.js
@@ -184,7 +184,9 @@ testStats.forEach(([testName, count]) => {
         currentDuration,
         historicalAverage,
         percentDiff,  // This will now always be a number
-        samples: count
+        samples: count,
+        historicalTimings: testTimings.get(testName) || [],
+        trend: createSparkline(testTimings.get(testName) || [], currentDuration)      
       });
 
       if (percentDiff > 0) {
@@ -221,6 +223,25 @@ testStats.forEach(([testName, count]) => {
   }
 }
 
+function createSparkline(timings, currentValue) {
+  const blocks = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
+  const min = Math.min(...timings);
+  const max = Math.max(...timings);
+  const range = max - min;
+  
+  const sparkline = timings.map(value => {
+    const normalized = range === 0 ? 0 : (value - min) / range;
+    const blockIndex = Math.floor(normalized * (blocks.length - 1));
+    return blocks[blockIndex];
+  }).join('');
+  
+  const avgHistorical = timings.reduce((a, b) => a + b, 0) / timings.length;
+  const trend = currentValue > avgHistorical ? '↑' : currentValue < avgHistorical ? '↓' : '→';
+  const trendColor = currentValue > avgHistorical ? '🔴' : currentValue < avgHistorical ? '🟢' : '⚪';
+  
+  return `${sparkline} ${trend} ${trendColor}`;
+}
+
 if (require.main === module) {
   analyzeTestTimes();
 }