test-ui.yml

name: test-ui
on:
  pull_request:
    paths:
      - "ui/**"
  push:
    branches:
      - main
      - release/**
      - test-ui
    paths:
      - "ui/**"

jobs:
  pre-test:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    defaults:
      run:
        working-directory: ui
    outputs:
      nonce: ${{ steps.nonce.outputs.nonce }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup-js
      - name: lint:js
        run: yarn run lint:js
      - name: lint:hbs
        run: yarn run lint:hbs
      - id: nonce
        name: nonce
        run: echo "nonce=${{ github.run_id }}-$(date +%s)" >> "$GITHUB_OUTPUT"

  tests:
    needs:
      - pre-test
    runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
    timeout-minutes: 30
    continue-on-error: true
    defaults:
      run:
        working-directory: ui
    strategy:
      matrix:
        partition: [1, 2, 3, 4]
        split: [4]
        # Note: If we ever change the number of partitions, we'll need to update the
        # finalize.combine step to match
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup-js
      - uses: browser-actions/setup-chrome@facf10a55b9caf92e0cc749b4f82bf8220989148 # v1.7.2
      - name: Retrieve Vault-hosted Secrets
        if: endsWith(github.repository, '-enterprise')
        id: vault
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          url: ${{ vars.CI_VAULT_URL }}
          method: ${{ vars.CI_VAULT_METHOD }}
          path: ${{ vars.CI_VAULT_PATH }}
          jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
          secrets: |-
            kv/data/teams/nomad/ui PERCY_TOKEN ;
      - name: ember exam
        id: exam
        env:
          PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
          PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
        run: |
          yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
      # We have continue-on-error set to true, but we still want to alert the author if
      # there are test failures or timeouts. Without it, we'll get errors in our output,
      # but the workflow will still succeed / have a green checkmark.
      - name: Express timeout failure
        if: ${{ failure() }}
        run: exit 1
      - name: Check test status
        if: steps.exam.outcome != 'success'
        run: |
          echo "Tests failed or timed out in partition ${{ matrix.partition }}"
          exit 1
      - name: Upload partition test results
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: test-results-${{ matrix.partition }}
          path: ui/test-results/test-results.json
          retention-days: 90
  finalize:
    needs:
      - pre-test
      - tests
    runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
    timeout-minutes: 30
    defaults:
      run:
        working-directory: ui
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup-js
      - name: Retrieve Vault-hosted Secrets
        if: endsWith(github.repository, '-enterprise')
        id: vault
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          url: ${{ vars.CI_VAULT_URL }}
          method: ${{ vars.CI_VAULT_METHOD }}
          path: ${{ vars.CI_VAULT_PATH }}
          jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
          secrets: |-
            kv/data/teams/nomad/ui PERCY_TOKEN ;
      - name: Download all test results
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: test-results-*
          path: test-results

      - name: Combine test results for comparison
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
        run: node ../scripts/combine-ui-test-results.js

      - name: Upload combined results for comparison
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: test-results-${{ github.sha }}
          path: ui/combined-test-results.json
          retention-days: 90

      - name: Upload Current PR results
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step
          path: ui/combined-test-results.json
          retention-days: 7

      - name: finalize
        env:
          PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
          PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
        run: yarn percy build:finalize

  analyze-times:
    # TODO: temporary comment-out with hardcoded sha
    # needs: [tests, finalize]
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ui

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      # Debug step to show environment
      - name: Debug environment
        run: |
          echo "GITHUB_SHA: ${{ github.sha }}"
          echo "GITHUB_EVENT_NAME: ${{ github.event_name }}"
          echo "GITHUB_REF: ${{ github.ref }}"
          echo "RUN_ID: ${{ github.run_id }}"
          
      # Try to list available artifacts first
      - name: List artifacts
        uses: actions/github-script@v7
        with:
          script: |
            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
              owner: context.repo.owner,
              repo: context.repo.repo,
              // run_id: context.runId
              run_id: 12163157778
            });
            console.log('Available artifacts:');
            console.log(JSON.stringify(artifacts.data, null, 2));

      - name: Download current PR results
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          # name: test-results-${{ github.sha }}
          name: pr-test-results-fe7ca11e9afc42bc98d79fe521155a37634bd232 # TODO: temporary hardcoded sha from previous run
          path: ui
          run-id: 12163157778
          github-token: ${{ secrets.GITHUB_TOKEN }}

      # - name: Download historical results
      #   uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      #   with:
      #     pattern: test-results-*
      #     path: historical-results
      #     merge-multiple: true

      # Download historical results from previous main branch runs
      - name: Download historical results
        uses: actions/github-script@v7
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          script: |
            const fs = require('fs');
            const path = require('path');
            
            const historicalDir = path.join('ui', 'historical-results');
            // Clean up any existing directory
            if (fs.existsSync(historicalDir)) {
              fs.rmSync(historicalDir, { recursive: true, force: true });
            }
            fs.mkdirSync(historicalDir, { recursive: true });
            
            const artifacts = await github.rest.actions.listArtifactsForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              per_page: 100
            });
            
            const testArtifacts = artifacts.data.artifacts.filter(artifact => 
              artifact.name.startsWith('test-results-')
            );
            
            console.log(`Found ${testArtifacts.length} test result artifacts`);
            
            for (const artifact of testArtifacts) {
              try {
                console.log(`Downloading ${artifact.name}`);
                
                // Create a temporary directory for this artifact
                const tempDir = path.join(historicalDir, `temp-${artifact.id}`);
                fs.mkdirSync(tempDir, { recursive: true });
                
                // Download to temp directory
                await exec.exec('gh', [
                  'run',
                  'download',
                  '-n',
                  artifact.name,
                  '--repo',
                  `${context.repo.owner}/${context.repo.repo}`,
                  '--dir',
                  tempDir,
                  artifact.workflow_run.id.toString()
                ]);
                
                // Move the JSON file to the historical directory with a unique name
                const jsonFile = path.join(tempDir, 'test-results.json');
                if (fs.existsSync(jsonFile)) {
                  fs.renameSync(
                    jsonFile,
                    path.join(historicalDir, `${artifact.name}-${artifact.id}.json`)
                  );
                }
                
                // Clean up temp directory
                fs.rmSync(tempDir, { recursive: true, force: true });
                
                console.log(`Successfully processed ${artifact.name}`);
              } catch (error) {
                console.log(`Error processing ${artifact.name}:`, error.message);
              }
            }
      
      # Debug what we got
      - name: Debug directories
        run: |
          echo "Current directory structure:"
          ls -la
          printf "\nHistorical results directory:\n"
          ls -la historical-results || echo "historical-results directory not found"

      - name: Analyze test times
        run: node ../scripts/analyze-ui-test-times.js

      - name: Comment PR
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          script: |
            const fs = require('fs');
            const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));
            
            let body = `### Test Time Analysis\n\n`;
            body += `- Total Tests: ${analysis.summary.totalTests}\n`;
            body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
            body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;
            
            if (analysis.testComparisons.length > 0) {
              body += `#### Most Significant Changes:\n\n`;
              analysis.testComparisons
                .filter(comp => comp.percentDiff != null)  // Skip invalid comparisons
                .slice(0, 5)
                .forEach(comp => {
                  body += `**${comp.name}**\n`;
                  body += `- Current: ${comp.currentDuration}ms\n`;
                  body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
                  body += `- Change: ${comp.percentDiff?.toFixed(1) || 'N/A'}%\n\n`;
                });
            }
            
            await github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body
            });

permissions:
  contents: read
  id-token: write