[ui, ci] UI test look-back and comparison for PRs

.github/workflows/test-ui.yml

@@ -36,6 +36,7 @@
       - pre-test
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     timeout-minutes: 30
+    continue-on-error: true
     defaults:
       run:
         working-directory: ui
@@ -61,17 +62,25 @@
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: ember exam
+        id: exam
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: |
           yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
-        continue-on-error: true
+      # We have continue-on-error set to true, but we still want to alert the author if
+      # there are test failures or timeouts. Without it, we'll get errors in our output,
+      # but the workflow will still succeed / have a green checkmark.
       - name: Express timeout failure
         if: ${{ failure() }}
         run: exit 1
+      - name: Check test status
+        if: steps.exam.outcome != 'success'
+        run: |
+          echo "Tests failed or timed out in partition ${{ matrix.partition }}"
+          exit 1
       - name: Upload partition test results
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: test-results-${{ matrix.partition }}
@@ -101,28 +110,229 @@
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: Download all test results
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           pattern: test-results-*
           path: test-results
 
       - name: Combine test results for comparison
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
         run: node ../scripts/combine-ui-test-results.js
+
       - name: Upload combined results for comparison
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: test-results-${{ github.sha }}
           path: ui/combined-test-results.json
           retention-days: 90
+      - name: Delete partition test results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
+        uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0
+        with:
+          name: test-results-*
+
+      - name: Upload Current PR results
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step
+          path: ui/combined-test-results.json
+          retention-days: 1
 
       - name: finalize
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: yarn percy build:finalize
+
+  analyze-times:
+    # TODO: temporary comment-out with hardcoded sha
+    # needs: [tests, finalize]
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ui
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      # Debug step to show environment
+      - name: Debug environment
+        run: |
+          echo "GITHUB_SHA: ${{ github.sha }}"
+          echo "GITHUB_EVENT_NAME: ${{ github.event_name }}"
+          echo "GITHUB_REF: ${{ github.ref }}"
+          echo "RUN_ID: ${{ github.run_id }}"
+
+      # Try to list available artifacts first
+      - name: List artifacts
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              // run_id: context.runId
+              run_id: 12163157778
+            });
+            console.log('Available artifacts:');
+            console.log(JSON.stringify(artifacts.data, null, 2));
+
+      - name: Download current PR results
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          # name: test-results-${{ github.sha }}
+          name: pr-test-results-fe7ca11e9afc42bc98d79fe521155a37634bd232 # TODO: temporary hardcoded sha from previous run
+          path: ui
+          run-id: 12163157778
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # - name: Download historical results
+      #   uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      #   with:
+      #     pattern: test-results-*
+      #     path: historical-results
+      #     merge-multiple: true
+
+      # Download historical results from previous main branch runs
+      - name: Download historical results
+        uses: actions/github-script@v7
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const historicalDir = path.join('ui', 'historical-results');
+            // Clean up any existing directory
+            if (fs.existsSync(historicalDir)) {
+              fs.rmSync(historicalDir, { recursive: true, force: true });
+            }
+            fs.mkdirSync(historicalDir, { recursive: true });
+
+            const artifacts = await github.rest.actions.listArtifactsForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              per_page: 100
+            });
+
+            // Log out the names of each artifact
+            console.log('Available artifacts:');
+            artifacts.data.artifacts.forEach(artifact => {
+              console.log(`- ${artifact.name}`);
+            });
+
+            const testArtifacts = artifacts.data.artifacts.filter(artifact => 
+              artifact.name.startsWith('pr-test-results-')
+            );
+
+            console.log(`Found ${testArtifacts.length} test result artifacts`);
+
+            for (const artifact of testArtifacts) {
+              try {
+                console.log(`Downloading ${artifact.name}`);
+
+                // Create a temporary directory for this artifact
+                const tempDir = path.join(historicalDir, `temp-${artifact.id}`);
+                fs.mkdirSync(tempDir, { recursive: true });
+
+                try {
+                  // Download to temp directory
+                  await exec.exec('gh', [
+                    'run',
+                    'download',
+                    '-n',
+                    artifact.name,
+                    '--repo',
+                    `${context.repo.owner}/${context.repo.repo}`,
+                    '--dir',
+                    tempDir,
+                    artifact.workflow_run.id.toString()
+                  ]);
+
+                  // Move the JSON file to the historical directory with a unique name
+                  const jsonFile = path.join(tempDir, 'combined-test-results.json');
+                  if (fs.existsSync(jsonFile)) {
+                    fs.renameSync(
+                      jsonFile,
+                      path.join(historicalDir, `${artifact.name}-${artifact.id}.json`)
+                    );
+                    console.log(`Successfully processed ${artifact.name}`);
+                  } else {
+                    const files = fs.readdirSync(tempDir);
+                    console.log(`Warning: No test results JSON found in ${artifact.name}. Found files:`, files);
+                    console.log(`Warning: No combined-test-results.json found in ${artifact.name}`);
+                  }
+                } finally {
+                  // Always clean up temp directory
+                  if (fs.existsSync(tempDir)) {
+                    fs.rmSync(tempDir, { recursive: true, force: true });
+                  }
+                }
+              } catch (error) {
+                console.log(`Error processing ${artifact.name}:`, error.message);
+                // Continue with next artifact
+              }
+            }
+
+      # Debug what we got
+      - name: Debug directories
+        run: |
+          echo "Current directory structure:"
+          ls -la
+          printf "\nHistorical results directory:\n"
+          ls -la historical-results || echo "historical-results directory not found"
+
+          cd historical-results
+          echo -e "\nContents of each file (first 10 lines):"
+          for file in *.json; do
+            if [ -f "$file" ]; then
+              echo -e "\n=== $file ==="
+              head -n 10 "$file"
+            fi
+          done
+
+      - name: Analyze test times
+        run: node ../scripts/analyze-ui-test-times.js
+
+      - name: Comment PR
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const fs = require('fs');
+            const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));
+
+            let body = `### Test Time Analysis\n\n`;
+            body += `- Total Tests: ${analysis.summary.totalTests}\n`;
+            body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
+            body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;
+
+            if (analysis.testComparisons.length > 0) {
+              body += `#### Most Significant Changes:\n\n`;
+              analysis.testComparisons
+                .filter(comp => comp.percentDiff != null)  // Skip invalid comparisons
+                .slice(0, 5)
+                .forEach(comp => {
+                  body += `**${comp.name}**\n`;
+                  body += `- Current: ${comp.currentDuration}ms\n`;
+                  body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
+                  body += `- Change: ${comp.percentDiff?.toFixed(1) || 'N/A'}%\n\n`;
+                  body += `- Trend: ${comp.trend}\n\n`;
+                });
+            }
+
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            });
+
 permissions:
   contents: read
   id-token: write
+  pull-requests: write