-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ui, ci] retain artifacts from test runs including test timing (#24555)
* retain artifacts from test runs including test timing * Pinning commit hashes for action helpers * trigger for ui-test run * Trying to isolate down to a simple upload * Once more with mkdir * What if we just wrote our own test reporter tho * Let the partitioned runs handle placement * Filter out common token logs, add a summary at the end, and note failures in logtime * Custom reporter cannot also have an output file, he finds out two days late * Aggregate summary, duration, and removing failure case * Conditional test report generation * Timeouts are errors * Trying with un-partitioned input json file * Remove the commented-out lines for main-only runs * combine-ui-test-results as its own script
- Loading branch information
1 parent
97d14c9
commit 4b91c17
Showing
5 changed files
with
269 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| #!/usr/bin/env node | ||
| /** | ||
| * Copyright (c) HashiCorp, Inc. | ||
| * SPDX-License-Identifier: BUSL-1.1 | ||
| */ | ||
|
|
||
| 'use strict'; | ||
| const fs = require('fs'); | ||
|
|
||
| const NUM_PARTITIONS = 4; | ||
|
|
||
| function combineResults() { | ||
| const results = []; | ||
| let duration = 0; | ||
| let aggregateSummary = { total: 0, passed: 0, failed: 0 }; | ||
|
|
||
| for (let i = 1; i <= NUM_PARTITIONS; i++) { | ||
| try { | ||
| const data = JSON.parse( | ||
| fs.readFileSync(`../test-results/test-results-${i}/test-results.json`).toString() | ||
| ); | ||
| results.push(...data.tests); | ||
| duration += data.duration; | ||
| aggregateSummary.total += data.summary.total; | ||
| aggregateSummary.passed += data.summary.passed; | ||
| aggregateSummary.failed += data.summary.failed; | ||
| } catch (err) { | ||
| console.error(`Error reading partition ${i}:`, err); | ||
| } | ||
| } | ||
|
|
||
| const output = { | ||
| timestamp: new Date().toISOString(), | ||
| sha: process.env.GITHUB_SHA, | ||
| summary: { | ||
| total: aggregateSummary.total, | ||
| passed: aggregateSummary.passed, | ||
| failed: aggregateSummary.failed | ||
| }, | ||
| duration, | ||
| tests: results | ||
| }; | ||
|
|
||
| fs.writeFileSync('../ui/combined-test-results.json', JSON.stringify(output, null, 2)); | ||
| } | ||
|
|
||
| if (require.main === module) { | ||
| combineResults(); | ||
| } | ||
|
|
||
| module.exports = combineResults; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,159 @@ | ||
| /** | ||
| * Copyright (c) HashiCorp, Inc. | ||
| * SPDX-License-Identifier: BUSL-1.1 | ||
| */ | ||
|
|
||
| /* eslint-env node */ | ||
| /* eslint-disable no-console */ | ||
|
|
||
| const fs = require('fs'); | ||
| const path = require('path'); | ||
|
|
||
| class JsonReporter { | ||
| constructor(out, socket, config) { | ||
| this.out = out || process.stdout; | ||
| this.results = []; | ||
|
|
||
| // Get output file from Testem config, which is set by the --json-report=path argument | ||
| this.outputFile = config?.fileOptions?.custom_report_file; | ||
| this.generateReport = !!this.outputFile; | ||
|
|
||
| if (this.generateReport) { | ||
| console.log( | ||
| `[Reporter] Initializing with output file: ${this.outputFile}` | ||
| ); | ||
|
|
||
| try { | ||
| fs.mkdirSync(path.dirname(this.outputFile), { recursive: true }); | ||
|
|
||
| // Initialize the results file | ||
| fs.writeFileSync( | ||
| this.outputFile, | ||
| JSON.stringify( | ||
| { | ||
| summary: { total: 0, passed: 0, failed: 0 }, | ||
| timestamp: new Date().toISOString(), | ||
| tests: [], | ||
| }, | ||
| null, | ||
| 2 | ||
| ) | ||
| ); | ||
| console.log('[Reporter] Initialized results file'); | ||
| } catch (err) { | ||
| console.error('[Reporter] Error initializing results file:', err); | ||
| } | ||
| } else { | ||
| console.log('[Reporter] No report file configured, skipping JSON output'); | ||
| } | ||
|
|
||
| process.on('SIGINT', () => { | ||
| console.log('[Reporter] Received SIGINT, finishing up...'); | ||
| this.finish(); | ||
| process.exit(0); | ||
| }); | ||
|
|
||
| this.testCounter = 0; | ||
| this.startTime = Date.now(); | ||
| } | ||
|
|
||
| filterLogs(logs) { | ||
| return logs.filter((log) => { | ||
| // Filter out token-related logs | ||
| if ( | ||
| log.text && | ||
| (log.text.includes('Accessor:') || | ||
| log.text.includes('log in with a JWT') || | ||
| log.text === 'TOKENS:' || | ||
| log.text === '=====================================') | ||
| ) { | ||
| return false; | ||
| } | ||
|
|
||
| // Keep non-warning logs that aren't token-related | ||
| return log.type !== 'warn'; | ||
| }); | ||
| } | ||
|
|
||
| report(prefix, data) { | ||
| if (!data || !data.name) { | ||
| console.log(`[Reporter] Skipping invalid test result: ${data.name}`); | ||
| return; | ||
| } | ||
|
|
||
| this.testCounter++; | ||
| console.log(`[Reporter] Test #${this.testCounter}: ${data.name}`); | ||
|
|
||
| const partitionMatch = data.name.match(/^Exam Partition (\d+) - (.*)/); | ||
|
|
||
| const result = { | ||
| name: partitionMatch ? partitionMatch[2] : data.name.trim(), | ||
| partition: partitionMatch ? parseInt(partitionMatch[1], 10) : null, | ||
| browser: prefix, | ||
| passed: !data.failed, | ||
| duration: data.runDuration, | ||
| error: data.failed ? data.error : null, | ||
| logs: this.filterLogs(data.logs || []), | ||
| }; | ||
|
|
||
| if (result.passed) { | ||
| console.log('- [PASS]'); | ||
| } else { | ||
| console.log('- [FAIL]'); | ||
| console.log('- Error:', result.error); | ||
| console.log('- Logs:', result.logs); | ||
| } | ||
|
|
||
| this.results.push(result); | ||
| } | ||
|
|
||
| writeCurrentResults() { | ||
| console.log('[Reporter] Writing current results...'); | ||
| try { | ||
| const passed = this.results.filter((r) => r.passed).length; | ||
| const failed = this.results.filter((r) => !r.passed).length; | ||
| const total = this.results.length; | ||
| const duration = Date.now() - this.startTime; | ||
|
|
||
| const output = { | ||
| summary: { total, passed, failed }, | ||
| timestamp: new Date().toISOString(), | ||
| duration, | ||
| tests: this.results, | ||
| }; | ||
|
|
||
| if (this.generateReport) { | ||
| fs.writeFileSync(this.outputFile, JSON.stringify(output, null, 2)); | ||
| } | ||
|
|
||
| // Print a summary | ||
| console.log('\n[Reporter] Test Summary:'); | ||
| console.log(`- Total: ${total}`); | ||
| console.log(`- Passed: ${passed}`); | ||
| console.log(`- Failed: ${failed}`); | ||
| console.log(`- Duration: ${duration}ms`); | ||
| if (failed > 0) { | ||
| console.log('\n[Reporter] Failed Tests:'); | ||
| this.results | ||
| .filter((r) => !r.passed) | ||
| .forEach((r) => { | ||
| console.log(`❌ ${r.name}`); | ||
| if (r.error) { | ||
| console.error(r.error); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| console.log('[Reporter] Successfully wrote results'); | ||
| } catch (err) { | ||
| console.error('[Reporter] Error writing results:', err); | ||
| } | ||
| } | ||
| finish() { | ||
| console.log('[Reporter] Finishing up...'); | ||
| this.writeCurrentResults(); | ||
| console.log('[Reporter] Done.'); | ||
| } | ||
| } | ||
|
|
||
| module.exports = JsonReporter; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters