From 4b91c17dfa2e55c370054361ae56de476114f244 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Tue, 3 Dec 2024 09:56:06 -0500 Subject: [PATCH] [ui, ci] retain artifacts from test runs including test timing (#24555) * retain artifacts from test runs including test timing * Pinning commit hashes for action helpers * trigger for ui-test run * Trying to isolate down to a simple upload * Once more with mkdir * What if we just wrote our own test reporter tho * Let the partitioned runs handle placement * Filter out common token logs, add a summary at the end, and note failures in logtime * Custom reporter cannot also have an output file, he finds out two days late * Aggregate summary, duration, and removing failure case * Conditional test report generation * Timeouts are errors * Trying with un-partitioned input json file * Remove the commented-out lines for main-only runs * combine-ui-test-results as its own script --- .github/workflows/test-ui.yml | 40 +++++++- scripts/combine-ui-test-results.js | 51 +++++++++ ui/app/index.html | 1 + ui/test-reporter.js | 159 +++++++++++++++++++++++++++++ ui/testem.js | 23 +++++ 5 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 scripts/combine-ui-test-results.js create mode 100644 ui/test-reporter.js diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml index f52096c47ab..5478460877c 100644 --- a/.github/workflows/test-ui.yml +++ b/.github/workflows/test-ui.yml @@ -2,14 +2,14 @@ name: test-ui on: pull_request: paths: - - 'ui/**' + - "ui/**" push: branches: - main - release/** - test-ui paths: - - 'ui/**' + - "ui/**" jobs: pre-test: @@ -36,7 +36,6 @@ jobs: - pre-test runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }} timeout-minutes: 30 - continue-on-error: true defaults: run: working-directory: ui @@ -44,6 +43,8 @@ jobs: matrix: partition: [1, 2, 3, 4] split: [4] + # Note: If we ever change the number of partitions, we'll need to update the + # finalize.combine step to match steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/setup-js @@ -63,8 +64,19 @@ jobs: env: PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }} PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }} - run: yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} - + run: | + yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json + continue-on-error: true + - name: Express timeout failure + if: ${{ failure() }} + run: exit 1 + - name: Upload partition test results + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: test-results-${{ matrix.partition }} + path: ui/test-results/test-results.json + retention-days: 90 finalize: needs: - pre-test @@ -88,6 +100,24 @@ jobs: jwtGithubAudience: ${{ vars.CI_VAULT_AUD }} secrets: |- kv/data/teams/nomad/ui PERCY_TOKEN ; + - name: Download all test results + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: test-results-* + path: test-results + + - name: Combine test results for comparison + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: node ../scripts/combine-ui-test-results.js + - name: Upload combined results for comparison + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: test-results-${{ github.sha }} + path: ui/combined-test-results.json + retention-days: 90 + - name: finalize env: PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }} diff --git a/scripts/combine-ui-test-results.js b/scripts/combine-ui-test-results.js new file mode 100644 index 00000000000..8d78f423cce --- /dev/null +++ b/scripts/combine-ui-test-results.js @@ -0,0 +1,51 @@ +#!/usr/bin/env node +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +'use strict'; +const fs = require('fs'); + +const NUM_PARTITIONS = 4; + +function combineResults() { + const results = []; + let duration = 0; + let aggregateSummary = { total: 0, passed: 0, failed: 0 }; + + for (let i = 1; i <= NUM_PARTITIONS; i++) { + try { + const data = JSON.parse( + fs.readFileSync(`../test-results/test-results-${i}/test-results.json`).toString() + ); + results.push(...data.tests); + duration += data.duration; + aggregateSummary.total += data.summary.total; + aggregateSummary.passed += data.summary.passed; + aggregateSummary.failed += data.summary.failed; + } catch (err) { + console.error(`Error reading partition ${i}:`, err); + } + } + + const output = { + timestamp: new Date().toISOString(), + sha: process.env.GITHUB_SHA, + summary: { + total: aggregateSummary.total, + passed: aggregateSummary.passed, + failed: aggregateSummary.failed + }, + duration, + tests: results + }; + + fs.writeFileSync('../ui/combined-test-results.json', JSON.stringify(output, null, 2)); +} + +if (require.main === module) { + combineResults(); +} + +module.exports = combineResults; diff --git a/ui/app/index.html b/ui/app/index.html index 9d49fe8a579..df5eb739390 100644 --- a/ui/app/index.html +++ b/ui/app/index.html @@ -24,6 +24,7 @@ {{content-for "body"}} + {{content-for "body-footer"}} diff --git a/ui/test-reporter.js b/ui/test-reporter.js new file mode 100644 index 00000000000..ddad7268714 --- /dev/null +++ b/ui/test-reporter.js @@ -0,0 +1,159 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +/* eslint-env node */ +/* eslint-disable no-console */ + +const fs = require('fs'); +const path = require('path'); + +class JsonReporter { + constructor(out, socket, config) { + this.out = out || process.stdout; + this.results = []; + + // Get output file from Testem config, which is set by the --json-report=path argument + this.outputFile = config?.fileOptions?.custom_report_file; + this.generateReport = !!this.outputFile; + + if (this.generateReport) { + console.log( + `[Reporter] Initializing with output file: ${this.outputFile}` + ); + + try { + fs.mkdirSync(path.dirname(this.outputFile), { recursive: true }); + + // Initialize the results file + fs.writeFileSync( + this.outputFile, + JSON.stringify( + { + summary: { total: 0, passed: 0, failed: 0 }, + timestamp: new Date().toISOString(), + tests: [], + }, + null, + 2 + ) + ); + console.log('[Reporter] Initialized results file'); + } catch (err) { + console.error('[Reporter] Error initializing results file:', err); + } + } else { + console.log('[Reporter] No report file configured, skipping JSON output'); + } + + process.on('SIGINT', () => { + console.log('[Reporter] Received SIGINT, finishing up...'); + this.finish(); + process.exit(0); + }); + + this.testCounter = 0; + this.startTime = Date.now(); + } + + filterLogs(logs) { + return logs.filter((log) => { + // Filter out token-related logs + if ( + log.text && + (log.text.includes('Accessor:') || + log.text.includes('log in with a JWT') || + log.text === 'TOKENS:' || + log.text === '=====================================') + ) { + return false; + } + + // Keep non-warning logs that aren't token-related + return log.type !== 'warn'; + }); + } + + report(prefix, data) { + if (!data || !data.name) { + console.log(`[Reporter] Skipping invalid test result: ${data.name}`); + return; + } + + this.testCounter++; + console.log(`[Reporter] Test #${this.testCounter}: ${data.name}`); + + const partitionMatch = data.name.match(/^Exam Partition (\d+) - (.*)/); + + const result = { + name: partitionMatch ? partitionMatch[2] : data.name.trim(), + partition: partitionMatch ? parseInt(partitionMatch[1], 10) : null, + browser: prefix, + passed: !data.failed, + duration: data.runDuration, + error: data.failed ? data.error : null, + logs: this.filterLogs(data.logs || []), + }; + + if (result.passed) { + console.log('- [PASS]'); + } else { + console.log('- [FAIL]'); + console.log('- Error:', result.error); + console.log('- Logs:', result.logs); + } + + this.results.push(result); + } + + writeCurrentResults() { + console.log('[Reporter] Writing current results...'); + try { + const passed = this.results.filter((r) => r.passed).length; + const failed = this.results.filter((r) => !r.passed).length; + const total = this.results.length; + const duration = Date.now() - this.startTime; + + const output = { + summary: { total, passed, failed }, + timestamp: new Date().toISOString(), + duration, + tests: this.results, + }; + + if (this.generateReport) { + fs.writeFileSync(this.outputFile, JSON.stringify(output, null, 2)); + } + + // Print a summary + console.log('\n[Reporter] Test Summary:'); + console.log(`- Total: ${total}`); + console.log(`- Passed: ${passed}`); + console.log(`- Failed: ${failed}`); + console.log(`- Duration: ${duration}ms`); + if (failed > 0) { + console.log('\n[Reporter] Failed Tests:'); + this.results + .filter((r) => !r.passed) + .forEach((r) => { + console.log(`❌ ${r.name}`); + if (r.error) { + console.error(r.error); + } + }); + } + + console.log('[Reporter] Successfully wrote results'); + } catch (err) { + console.error('[Reporter] Error writing results:', err); + } + } + finish() { + console.log('[Reporter] Finishing up...'); + this.writeCurrentResults(); + console.log('[Reporter] Done.'); + } +} + +module.exports = JsonReporter; diff --git a/ui/testem.js b/ui/testem.js index 7d1869af9ef..c937a5760fe 100644 --- a/ui/testem.js +++ b/ui/testem.js @@ -3,7 +3,24 @@ * SPDX-License-Identifier: BUSL-1.1 */ +// @ts-check + 'use strict'; +const JsonReporter = require('./test-reporter'); + +/** + * Get the path for the test results file based on the command line arguments + * @returns {string} The path to the test results file + */ +const getReportPath = () => { + const jsonReportArg = process.argv.find((arg) => + arg.startsWith('--json-report=') + ); + if (jsonReportArg) { + return jsonReportArg.split('=')[1]; + } + return null; +}; const config = { test_page: 'tests/index.html?hidepassed', @@ -13,6 +30,12 @@ const config = { browser_start_timeout: 120, parallel: -1, framework: 'qunit', + reporter: JsonReporter, + custom_report_file: getReportPath(), + // NOTE: we output this property as custom_report_file instead of report_file. + // See https://github.com/testem/testem/issues/1073, report_file + custom reporter results in double output. + debug: true, + browser_args: { // New format in testem/master, but not in a release yet // Chrome: {