test-ui.yml

name: test-ui
on:
  pull_request:
    paths:
      - "ui/**"
  push:
    branches:
      - main
      - release/**
      - test-ui
    paths:
      - "ui/**"

jobs:
  pre-test:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    defaults:
      run:
        working-directory: ui
    outputs:
      nonce: ${{ steps.nonce.outputs.nonce }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup-js
      - name: lint:js
        run: yarn run lint:js
      - name: lint:hbs
        run: yarn run lint:hbs
      - id: nonce
        name: nonce
        run: echo "nonce=${{ github.run_id }}-$(date +%s)" >> "$GITHUB_OUTPUT"

  tests:
    needs:
      - pre-test
    runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
    timeout-minutes: 30
    continue-on-error: true
    defaults:
      run:
        working-directory: ui
    strategy:
      matrix:
        partition: [1, 2, 3, 4]
        split: [4]
        # Note: If we ever change the number of partitions, we'll need to update the
        # finalize.combine step to match
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup-js
      - uses: browser-actions/setup-chrome@facf10a55b9caf92e0cc749b4f82bf8220989148 # v1.7.2
      - name: Retrieve Vault-hosted Secrets
        if: endsWith(github.repository, '-enterprise')
        id: vault
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          url: ${{ vars.CI_VAULT_URL }}
          method: ${{ vars.CI_VAULT_METHOD }}
          path: ${{ vars.CI_VAULT_PATH }}
          jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
          secrets: |-
            kv/data/teams/nomad/ui PERCY_TOKEN ;
      - name: ember exam
        id: exam
        env:
          PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
          PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
        run: |
          yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
      # We have continue-on-error set to true, but we still want to alert the author if
      # there are test failures or timeouts. Without it, we'll get errors in our output,
      # but the workflow will still succeed / have a green checkmark.
      - name: Express timeout failure
        if: ${{ failure() }}
        run: exit 1
      - name: Check test status
        if: steps.exam.outcome != 'success'
        run: |
          echo "Tests failed or timed out in partition ${{ matrix.partition }}"
          exit 1
      - name: Upload partition test results
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: test-results-${{ matrix.partition }}
          path: ui/test-results/test-results.json
          retention-days: 90
  finalize:
    needs:
      - pre-test
      - tests
    runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
    timeout-minutes: 30
    defaults:
      run:
        working-directory: ui
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup-js
      - name: Retrieve Vault-hosted Secrets
        if: endsWith(github.repository, '-enterprise')
        id: vault
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          url: ${{ vars.CI_VAULT_URL }}
          method: ${{ vars.CI_VAULT_METHOD }}
          path: ${{ vars.CI_VAULT_PATH }}
          jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
          secrets: |-
            kv/data/teams/nomad/ui PERCY_TOKEN ;
      - name: Download all test results
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: test-results-*
          path: test-results

      - name: Combine test results for comparison
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
        run: node ../scripts/combine-ui-test-results.js

      - name: Upload combined results for comparison
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: test-results-${{ github.sha }}
          path: ui/combined-test-results.json
          retention-days: 90

      - name: Upload Current PR results
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: pr-test-results-${{ github.sha }} # Prefix with "pr-" to avoid comparing with main during analyze step
          path: ui/combined-test-results.json
          retention-days: 7

      - name: finalize
        env:
          PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
          PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
        run: yarn percy build:finalize

  analyze-times:
    # TODO: temporary comment-out with hardcoded sha
    # needs: [tests, finalize]
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ui

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      # Debug step to show environment
      - name: Debug environment
        run: |
          echo "GITHUB_SHA: ${{ github.sha }}"
          echo "GITHUB_EVENT_NAME: ${{ github.event_name }}"
          echo "GITHUB_REF: ${{ github.ref }}"
          echo "RUN_ID: ${{ github.run_id }}"
          
      # Try to list available artifacts first
      - name: List artifacts
        uses: actions/github-script@v7
        with:
          script: |
            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
              owner: context.repo.owner,
              repo: context.repo.repo,
              // run_id: context.runId
              run_id: 12163157778
            });
            console.log('Available artifacts:');
            console.log(JSON.stringify(artifacts.data, null, 2));

      - name: Download current PR results
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          # name: test-results-${{ github.sha }}
          name: pr-test-results-fe7ca11e9afc42bc98d79fe521155a37634bd232 # TODO: temporary hardcoded sha from previous run
          path: ui
          run-id: 12163157778
          github-token: ${{ secrets.GITHUB_TOKEN }}

      # - name: Download historical results
      #   uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      #   with:
      #     pattern: test-results-*
      #     path: historical-results
      #     merge-multiple: true

      # Download historical results from previous main branch runs
      - name: Download historical results
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const path = require('path');
            const stream = require('stream');
            const { promisify } = require('util');
            const pipeline = promisify(stream.pipeline);
            
            // Create directory
            const historicalDir = path.join('ui', 'historical-results');
            fs.mkdirSync(historicalDir, { recursive: true });
            
            // Get the last 10 workflow runs on main
            const runs = await github.rest.actions.listWorkflowRuns({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'test-ui.yml',
              branch: 'main',
              per_page: 10
            });
            
            // Process each run
            for (const run of runs.data.workflow_runs) {
              console.log(`Checking run ${run.id} from ${run.created_at}`);
              
              try {
                const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  run_id: run.id
                });
                
                for (const artifact of artifacts.data.artifacts) {
                  if (artifact.name.startsWith('test-results-')) {
                    console.log(`Found ${artifact.name} from run ${run.id}`);
                    
                    // Use the download-artifact action for each artifact
                    await exec.exec('gh', [
                      'run',
                      'download',
                      '-n',
                      artifact.name,
                      '--repo',
                      `${context.repo.owner}/${context.repo.repo}`,
                      '--dir',
                      historicalDir,
                      run.id.toString()
                    ]);
                  }
                }
              } catch (error) {
                console.log(`Error processing run ${run.id}:`, error.message);
              }
            }
            
            console.log('Contents of historical-results:');
            console.log(fs.readdirSync(historicalDir));

      # Debug what we got
      - name: Debug directories
        run: |
          echo "Current directory structure:"
          ls -la
          printf "\nHistorical results directory:\n"
          ls -la historical-results || echo "historical-results directory not found"

      - name: Analyze test times
        run: node ../scripts/analyze-ui-test-times.js

      - name: Comment PR
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          script: |
            const fs = require('fs');
            const analysis = JSON.parse(fs.readFileSync('ui/test-time-analysis.json'));

            console.log('=== analysis', analysis);
            
            let body = `### Test Time Analysis\n\n`;
            body += `- Total Tests: ${analysis.summary.totalTests}\n`;
            body += `- Significantly Slower: ${analysis.summary.significantlySlower}\n`;
            body += `- Significantly Faster: ${analysis.summary.significantlyFaster}\n\n`;
            
            if (analysis.testComparisons.length > 0) {
              body += `#### Most Significant Changes:\n\n`;
              analysis.testComparisons.slice(0, 5).forEach(comp => {
                body += `**${comp.name}**\n`;
                body += `- Current: ${comp.currentDuration}ms\n`;
                body += `- Historical Avg: ${comp.historicalAverage}ms\n`;
                body += `- Change: ${comp.percentDiff.toFixed(1)}%\n\n`;
              });
            }
            
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body
            });

permissions:
  contents: read
  id-token: write