diff --git a/CHANGELOG.md b/CHANGELOG.md index f51bc650..3176d1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +## Added +- New server-side APIs to accept a batch of results instead of a single result #TINY-11177 + +## Changed +- Reverted TINY-10708 which was a server-side fix +- Client no longer waits for log requests to complete between tests, which should speed up remote testing #TINY-11177 +- Console HUD no longer updates for individual tests #TINY-11177 +- Client now posts test status only in batches every 30 seconds, this is the only time the console HUD will update #TINY-11177 + +## Removed +- Single result server-side API #TINY-11177 +- Server-side monitoring of single test timeouts. This is still monitored client side. #TINY-11177 +- The Promise polyfill is no longer allowed on modern NodeJS frameworks so it has been removed. #TINY-11177 + ## 14.1.4 - 2024-03-27 ### Fixed diff --git a/Jenkinsfile b/Jenkinsfile index a20104b4..b0fbdeca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,19 +21,35 @@ timestamps { stage("test") { exec('yarn test') - - bedrockBrowsers( - prepareTests: { - yarnInstall() - exec('yarn build') - }, - testDirs: [ 'modules/sample/src/test/ts/**/pass' ], - custom: '--config modules/sample/tsconfig.json --customRoutes modules/sample/routes.json --polyfills Promise Symbol' - ) } + } + + // Testing + stage("bedrock testing") { + bedrockRemoteBrowsers( + platforms: [ + [ browser: 'chrome', provider: 'aws', buckets: 2 ], + [ browser: 'firefox', provider: 'aws', buckets: 2 ], + [ browser: 'edge', provider: 'lambdatest', buckets: 1 ], + [ browser: 'chrome', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1 ], + [ browser: 'firefox', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1 ], + [ browser: 'safari', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1 ], + ], + prepareTests: { + yarnInstall() + sh 'yarn build' + }, + testDirs: [ 'modules/sample/src/test/ts/**/pass' ], + custom: '--config modules/sample/tsconfig.json --customRoutes modules/sample/routes.json --polyfills Promise Symbol' + ) + } - if (isReleaseBranch()) { - stage("publish") { + // Publish + if (isReleaseBranch()) { + stage("publish") { + tinyPods.node() { + yarnInstall() + sh 'yarn build' tinyNpm.withNpmPublishCredentials { // We need to tell git to ignore the changes to .npmrc when publishing exec('git update-index --assume-unchanged .npmrc') diff --git a/lerna.json b/lerna.json index f33665ff..53b20d12 100644 --- a/lerna.json +++ b/lerna.json @@ -1,7 +1,7 @@ { "npmClient": "yarn", "useWorkspaces": true, - "version": "14.1.4", + "version": "15.0.0-alpha.4", "publish": { "push": false } diff --git a/modules/runner/package.json b/modules/runner/package.json index 2f0a708b..e378f581 100644 --- a/modules/runner/package.json +++ b/modules/runner/package.json @@ -1,6 +1,6 @@ { "name": "@ephox/bedrock-runner", - "version": "14.1.1", + "version": "15.0.0-alpha.4", "author": "Tiny Technologies Inc", "license": "Apache-2.0", "scripts": { @@ -11,7 +11,6 @@ }, "dependencies": { "@ephox/bedrock-common": "^14.1.1", - "@ephox/wrap-promise-polyfill": "^2.2.0", "jquery": "^3.4.1", "querystringify": "^2.1.1" }, diff --git a/modules/runner/src/main/ts/api/Main.ts b/modules/runner/src/main/ts/api/Main.ts index 667f8296..3547ddfd 100644 --- a/modules/runner/src/main/ts/api/Main.ts +++ b/modules/runner/src/main/ts/api/Main.ts @@ -1,5 +1,4 @@ import { Failure, Global } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import * as Globals from '../core/Globals'; import * as TestLoader from '../core/TestLoader'; import { UrlParams } from '../core/UrlParams'; diff --git a/modules/runner/src/main/ts/core/TestLoader.ts b/modules/runner/src/main/ts/core/TestLoader.ts index fcab9f50..c1cddec1 100644 --- a/modules/runner/src/main/ts/core/TestLoader.ts +++ b/modules/runner/src/main/ts/core/TestLoader.ts @@ -1,4 +1,3 @@ -import Promise from '@ephox/wrap-promise-polyfill'; import { ErrorCatcher } from '../errors/ErrorCatcher'; export const load = (scriptUrl: string): Promise => @@ -38,4 +37,4 @@ export const load = (scriptUrl: string): Promise => // Add the script to the dom to load it document.body.appendChild(script); - }); \ No newline at end of file + }); diff --git a/modules/runner/src/main/ts/core/Utils.ts b/modules/runner/src/main/ts/core/Utils.ts index 7225207c..10b72385 100644 --- a/modules/runner/src/main/ts/core/Utils.ts +++ b/modules/runner/src/main/ts/core/Utils.ts @@ -1,5 +1,4 @@ import { Suite, Test } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import sourceMappedStackTrace from 'sourcemapped-stacktrace'; // eslint-disable-next-line @typescript-eslint/no-empty-function @@ -62,4 +61,4 @@ export const setStack = (error: Error, stack: string | undefined): void => { } catch (err) { // Do nothing } -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/reporter/Callbacks.ts b/modules/runner/src/main/ts/reporter/Callbacks.ts index ce10424e..f4ac2e5c 100644 --- a/modules/runner/src/main/ts/reporter/Callbacks.ts +++ b/modules/runner/src/main/ts/reporter/Callbacks.ts @@ -1,7 +1,16 @@ import { ErrorData, Global } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; + import { HarnessResponse } from '../core/ServerTypes'; +export interface TestReport { + file: string; + name: string; + passed: boolean; + time: string; + skipped: string | null; + error: TestErrorData | null +} + export interface TestErrorData { readonly data: ErrorData; readonly text: string; @@ -11,15 +20,24 @@ export interface Callbacks { readonly loadHarness: () => Promise readonly sendKeepAlive: (session: string) => Promise; readonly sendInit: (session: string) => Promise; - readonly sendTestStart: (session: string, totalTests: number, file: string, name: string) => Promise; - readonly sendTestResult: (session: string, file: string, name: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string | null) => Promise; + readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string) => Promise; + readonly sendTestResults: (session: string, results: TestReport[]) => Promise; readonly sendDone: (session: string, error?: string) => Promise; } declare const $: JQueryStatic; -const sendJson = (url: string, data: any): Promise => { +function generateErrorMessage(xhr: JQuery.jqXHR, onError: (reason?: any) => void, url: string, requestDetails: string, statusText: 'timeout' | 'error' | 'abort' | 'parsererror', e: string) { + if (xhr.readyState === 0) { + onError(`Unable to open connection to ${url}, ${requestDetails}. Status text "${statusText}", error thrown "${e}"`); + } else { + onError(`Response status ${xhr.status} connecting to ${url}, ${requestDetails}. Status text "${statusText}", error thrown "${e}"`); + } +} + +const sendJson = (url: string, jsonData: any): Promise => { return new Promise((onSuccess, onError) => { + const data = JSON.stringify(jsonData); $.ajax({ method: 'post', url, @@ -27,9 +45,9 @@ const sendJson = (url: string, data: any): Promise => { dataType: 'json', success: onSuccess, error: (xhr, statusText, e) => { - onError(e); + generateErrorMessage(xhr, onError, url, `sending ${data}`, statusText, e); }, - data: JSON.stringify(data), + data, }); }); }; @@ -41,7 +59,7 @@ const getJson = (url: string): Promise => { dataType: 'json', success: onSuccess, error: (xhr, statusText, e) => { - onError(e); + generateErrorMessage(xhr, onError, url, 'as a get request', statusText, e); } }); })); @@ -64,8 +82,9 @@ export const Callbacks = (): Callbacks => { }); }; - const sendTestStart = (session: string, totalTests: number, file: string, name: string): Promise => { + const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string): Promise => { return sendJson('/tests/start', { + number, totalTests, session, file, @@ -73,15 +92,10 @@ export const Callbacks = (): Callbacks => { }); }; - const sendTestResult = (session: string, file: string, name: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string | null): Promise => { - return sendJson('/tests/result', { + const sendTestResults = (session: string, results: TestReport[]): Promise => { + return sendJson('/tests/results', { session, - file, - name, - passed, - skipped, - time, - error, + results, }); }; @@ -101,7 +115,7 @@ export const Callbacks = (): Callbacks => { sendInit, sendKeepAlive, sendTestStart, - sendTestResult, + sendTestResults, sendDone }; -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/reporter/Reporter.ts b/modules/runner/src/main/ts/reporter/Reporter.ts index 75aafd69..767fafd1 100644 --- a/modules/runner/src/main/ts/reporter/Reporter.ts +++ b/modules/runner/src/main/ts/reporter/Reporter.ts @@ -1,22 +1,23 @@ -import { LoggedError, Reporter as ErrorReporter } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; -import { Callbacks } from './Callbacks'; -import { UrlParams } from '../core/UrlParams'; -import { formatElapsedTime, mapStackTrace, setStack } from '../core/Utils'; +import {LoggedError, Reporter as ErrorReporter} from '@ephox/bedrock-common'; +import {Callbacks, TestReport} from './Callbacks'; +import {UrlParams} from '../core/UrlParams'; +import {formatElapsedTime, mapStackTrace, setStack} from '../core/Utils'; type LoggedError = LoggedError.LoggedError; export interface TestReporter { - readonly start: () => Promise; - readonly retry: () => Promise; - readonly pass: () => Promise; - readonly skip: (reason: string) => Promise; - readonly fail: (e: LoggedError) => Promise; + readonly start: () => void; + readonly retry: () => void; + readonly pass: () => void; + readonly skip: (reason: string) => void; + readonly fail: (e: LoggedError) => void; } export interface Reporter { readonly summary: () => { offset: number; passed: number; failed: number; skipped: number }; readonly test: (file: string, name: string, totalNumTests: number) => TestReporter; + readonly waitForResults: () => Promise; + readonly retry: () => Promise; readonly done: (error?: LoggedError) => void; } @@ -42,16 +43,30 @@ const mapError = (e: LoggedError) => mapStackTrace(e.stack).then((mappedStack) = e.logs = logs.replace(originalStack, mappedStack).split('\n'); } - return Promise.resolve(e); + return e; }); export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi): Reporter => { const initial = new Date(); + let timeSinceLastReport = initial; let currentCount = params.offset || 0; let passCount = 0; let skipCount = 0; let failCount = 0; + // A list of test results we are going to send as a batch to the server + const testResults: TestReport[] = []; + + // A global list of requests that were sent to the server, we must wait for these before sending `/done` or it may confuse the HUD + const requestsInFlight: Promise[] = []; + + const sendCurrentResults = () => { + if (testResults.length > 0) { + requestsInFlight.push(callbacks.sendTestResults(params.session, testResults)); + testResults.length = 0; + } + }; + const summary = () => ({ offset: Math.max(0, currentCount - 1), passed: passCount + (params.offset - params.failed - params.skipped), @@ -60,73 +75,98 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi }); const test = (file: string, name: string, totalNumTests: number) => { - let starttime: Date; + let starttime = new Date(); let reported = false; let started = false; const testUi = ui.test(); - const start = (): Promise => { - if (started) { - return Promise.resolve(); - } else { + const start = (): void => { + if (!started) { started = true; starttime = new Date(); currentCount++; testUi.start(file, name); - return callbacks.sendTestStart(params.session, totalNumTests, file, name); + + if (currentCount === 1) { + // we need to send test start once to establish the session + requestsInFlight.push(callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name)); + } else if (starttime.getTime() - timeSinceLastReport.getTime() > 30 * 1000) { + // ping the server with results every 30 seconds or so, otherwise the result data could be gigantic + sendCurrentResults(); + timeSinceLastReport = new Date(); + } } }; - const retry = (): Promise => { + const retry = (): void => { + // a test has used `this.retries()` and wants to retry without reloading the page starttime = new Date(); - return Promise.resolve(); }; - const pass = (): Promise => { - if (reported) { - return Promise.resolve(); - } else { + const pass = (): void => { + if (!reported) { reported = true; passCount++; const testTime = elapsed(starttime); testUi.pass(testTime, currentCount); - return callbacks.sendTestResult(params.session, file, name, true, testTime, null, null); + testResults.push({ + file, + name, + passed: true, + time: testTime, + error: null, + skipped: null, + }); } }; - const skip = (reason: string): Promise => { - if (reported) { - return Promise.resolve(); - } else { + const skip = (reason: string): void => { + if (!reported) { reported = true; skipCount++; const testTime = elapsed(starttime); testUi.skip(testTime, currentCount); - return callbacks.sendTestResult(params.session, file, name, false, testTime, null, reason); + + testResults.push({ + file, + name, + passed: false, + time: testTime, + error: null, + skipped: reason, + }); } }; - const fail = (e: LoggedError): Promise => { - if (reported) { - return Promise.resolve(); - } else { + const fail = (e: LoggedError): void => { + if (!reported) { reported = true; failCount++; const testTime = elapsed(starttime); - return mapError(e).then((err) => { + + // `sourcemapped-stacktrace` is async, so we need to wait for it + requestsInFlight.push(mapError(e).then((err) => { const errorData = ErrorReporter.data(err); const error = { data: errorData, text: ErrorReporter.dataText(errorData) }; + testResults.push({ + file, + name, + passed: false, + time: testTime, + error, + skipped: null, + }); + testUi.fail(err, testTime, currentCount); - return callbacks.sendTestResult(params.session, file, name, false, testTime, error, null); - }); + })); } }; @@ -139,6 +179,32 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi }; }; + const waitForResults = async (): Promise => { + sendCurrentResults(); + if (requestsInFlight.length > 0) { + const currentRequests = requestsInFlight.slice(0); + requestsInFlight.length = 0; + return Promise.all(currentRequests).then(() => { + // if more things have been queued, such as a failing test stack trace, wait for those as well + waitForResults(); + }); + } + }; + + // the page is about to reload to retry a test + const retry = (): Promise => { + // remove the last test failure from the stack so we don't confuse the server + return Promise.all(requestsInFlight).then(() => { + const last = testResults.pop(); + if (last && last.error === null) { + // something isn't right, the last test didn't fail, put it back + testResults.push(last); + } + // now push the results to the server + return waitForResults(); + }); + }; + const done = (error?: LoggedError): void => { const setAsDone = (): void => { const totalTime = elapsed(initial); @@ -146,12 +212,18 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi }; const textError = error !== undefined ? ErrorReporter.text(error) : undefined; - callbacks.sendDone(params.session, textError).then(setAsDone, setAsDone); + + // make sure any in progress updates are sent before we clean up + waitForResults().then(() => + callbacks.sendDone(params.session, textError).then(setAsDone, setAsDone) + ); }; return { summary, test, + retry, + waitForResults, done }; -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/runner/Hooks.ts b/modules/runner/src/main/ts/runner/Hooks.ts index 05e8f7e5..9606c14c 100644 --- a/modules/runner/src/main/ts/runner/Hooks.ts +++ b/modules/runner/src/main/ts/runner/Hooks.ts @@ -1,5 +1,4 @@ import { Hook, HookType, RunnableState, Suite, Test } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import * as Context from '../core/Context'; import { SkipError } from '../errors/Errors'; import { runWithErrorCatcher, runWithTimeout } from './Run'; @@ -75,4 +74,4 @@ export const runAfterEach = (currentTest: Test): Promise => { } else { return Promise.resolve(); } -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/runner/Run.ts b/modules/runner/src/main/ts/runner/Run.ts index 48bb4109..b273314c 100644 --- a/modules/runner/src/main/ts/runner/Run.ts +++ b/modules/runner/src/main/ts/runner/Run.ts @@ -1,5 +1,4 @@ import { Context, ExecuteFn, Failure, Runnable, RunnableState, TestThrowable } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import { isInternalError, MultipleDone, SkipError } from '../errors/Errors'; import { ErrorCatcher } from '../errors/ErrorCatcher'; import { Timer } from './Timer'; @@ -128,4 +127,4 @@ export const runWithTimeout = (runnable: Runnable, context: Context, defaultTime }).then(resolveIfNotTimedOut, reject); }); } -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/runner/Runner.ts b/modules/runner/src/main/ts/runner/Runner.ts index e732935e..2c22dcf4 100644 --- a/modules/runner/src/main/ts/runner/Runner.ts +++ b/modules/runner/src/main/ts/runner/Runner.ts @@ -1,14 +1,13 @@ -import { Suite } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; -import { HarnessResponse } from '../core/ServerTypes'; -import { UrlParams } from '../core/UrlParams'; -import { noop } from '../core/Utils'; -import { Callbacks } from '../reporter/Callbacks'; -import { Reporter } from '../reporter/Reporter'; -import { Actions } from '../ui/Actions'; -import { Ui } from '../ui/Ui'; -import { RunActions, RunState, runSuite } from './TestRun'; -import { countTests, filterOnly } from './Utils'; +import {Suite} from '@ephox/bedrock-common'; +import {HarnessResponse} from '../core/ServerTypes'; +import {UrlParams} from '../core/UrlParams'; +import {noop} from '../core/Utils'; +import {Callbacks} from '../reporter/Callbacks'; +import {Reporter} from '../reporter/Reporter'; +import {Actions} from '../ui/Actions'; +import {Ui} from '../ui/Ui'; +import {RunActions, RunState, runSuite} from './TestRun'; +import {countTests, filterOnly} from './Utils'; export interface Runner { readonly init: () => Promise; @@ -30,8 +29,10 @@ export const Runner = (rootSuite: Suite, params: UrlParams, callbacks: Callbacks }; const runNextChunk = (offset: number) => { - const sum = reporter.summary(); - actions.reloadPage(offset, sum.failed, sum.skipped); + reporter.waitForResults().then(() => { + const sum = reporter.summary(); + actions.reloadPage(offset, sum.failed, sum.skipped); + }); }; const retryTest = withSum(actions.retryTest); @@ -51,14 +52,23 @@ export const Runner = (rootSuite: Suite, params: UrlParams, callbacks: Callbacks reporter.done(); // make it easy to restart at this test stopTest(); - } else if (params.retry < retries) { - retryTest(params.retry + 1); } else { - loadNextTest(); + if (params.retry < retries) { + // post all results except the failure, and retry + reporter.retry().then(() => { + retryTest(params.retry + 1); + }); + } else { + // post the failure to the server and move on + reporter.waitForResults().then(() => { + // wait for 2 seconds for the purposes of showing the failure on video + setTimeout(loadNextTest, 2000); + }); + } } }; - const init = (): Promise => { + const init = async (): Promise => { // Filter the tests to ensure we have an accurate total test count filterOnly(rootSuite); numTests = countTests(rootSuite); @@ -66,15 +76,20 @@ export const Runner = (rootSuite: Suite, params: UrlParams, callbacks: Callbacks // Render the initial UI ui.render(params.offset, numTests, actions.restartTests, retryTest, loadNextTest); - // delay this ajax call until after the reporter status elements are in the page - const keepAliveTimer = setInterval(() => { - callbacks.sendKeepAlive(params.session).catch(() => { - // if the server shuts down stop trying to send messages - clearInterval(keepAliveTimer); - }); - }, KEEP_ALIVE_INTERVAL); - - return callbacks.sendInit(params.session).then(() => callbacks.loadHarness()); + return Promise.all([callbacks.sendInit(params.session), callbacks.loadHarness()]).then(([_, harness]) => { + // we don't need a keep-alive timer in auto mode, + if (harness.mode === 'manual') { + // delay this ajax call until after the reporter status elements are in the page + const keepAliveTimer = setInterval(() => { + callbacks.sendKeepAlive(params.session).catch(() => { + // if the server shuts down stop trying to send messages + clearInterval(keepAliveTimer); + }); + }, KEEP_ALIVE_INTERVAL); + } + + return harness; + }); }; const run = (chunk: number, retries: number, timeout: number, stopOnFailure: boolean): Promise => { @@ -116,4 +131,4 @@ export const Runner = (rootSuite: Suite, params: UrlParams, callbacks: Callbacks init, run }; -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/runner/TestRun.ts b/modules/runner/src/main/ts/runner/TestRun.ts index 6f0e1dce..96fbaeb1 100644 --- a/modules/runner/src/main/ts/runner/TestRun.ts +++ b/modules/runner/src/main/ts/runner/TestRun.ts @@ -1,5 +1,4 @@ import { Failure, LoggedError, RunnableState, Suite, Test } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import * as Context from '../core/Context'; import { InternalError, isInternalError, SkipError } from '../errors/Errors'; import { Reporter, TestReporter } from '../reporter/Reporter'; @@ -26,7 +25,7 @@ export interface RunActions { readonly runNextChunk: (offset: number) => void; } -const runTestWithRetry = (test: Test, state: RunState, report: TestReporter, retryCount: number): Promise => { +const runTestWithRetry = (test: Test, state: RunState, testReport: TestReporter, retryCount: number): Promise => { if (test.isSkipped()) { return Promise.reject(new SkipError()); } else { @@ -38,9 +37,12 @@ const runTestWithRetry = (test: Test, state: RunState, report: TestReporter, ret // Ensure we run the afterEach hooks no matter if the test failed .then(runAfterHooks(false), runAfterHooks(true)) .catch((e: LoggedError | InternalError) => { + // This is unique to `this.retries()` within a test, not the general page reload to retry system if (retryCount < test.retries() && !isInternalError(e)) { test.setResult(RunnableState.NotRun); - return report.retry().then(() => runTestWithRetry(test, state, report, retryCount + 1)); + testReport.retry(); + // don't fail the page + return runTestWithRetry(test, state, testReport, retryCount + 1); } else { return Promise.reject(e); } @@ -52,17 +54,23 @@ export const runTest = (test: Test, state: RunState, actions: RunActions, report const fail = (report: TestReporter, e: LoggedError) => { test.setResult(RunnableState.Failed, e); console.error(e); - return report.fail(e).then(actions.onFailure).then(() => Promise.reject()); + report.fail(e); + // this is where the page reloads if the global retry system is active + actions.onFailure(); + // Test failures must be an empty reject, otherwise the error management thinks it's a bedrock error + return Promise.reject(); }; - const skip = (report: TestReporter) => { + const skip = (testReport: TestReporter) => { test.setResult(RunnableState.Skipped); - return report.skip(test.title).then(actions.onSkip); + testReport.skip(test.title); + actions.onSkip(); }; - const pass = (report: TestReporter) => { + const pass = (testReport: TestReporter) => { test.setResult(RunnableState.Passed); - return report.pass().then(actions.onPass); + testReport.pass(); + actions.onPass(); }; state.testCount++; @@ -71,18 +79,19 @@ export const runTest = (test: Test, state: RunState, actions: RunActions, report } else if (state.testCount > state.offset + state.chunk) { actions.runNextChunk(state.offset + state.chunk); // Reject so no other tests are run + // Test failures must be an empty reject, otherwise the error management thinks it's a bedrock error return Promise.reject(); } else { - const report = reporter.test(test.file || 'Unknown', test.fullTitle(), state.totalTests); + const testReport = reporter.test(test.file || 'Unknown', test.fullTitle(), state.totalTests); actions.onStart(test); - return report.start() - .then(() => runTestWithRetry(test, state, report, 0)) - .then(() => pass(report), (e: LoggedError | InternalError) => { + testReport.start(); + return runTestWithRetry(test, state, testReport, 0) + .then(() => pass(testReport), (e: LoggedError | InternalError) => { if (e instanceof SkipError) { - return skip(report); + return skip(testReport); } else { - return fail(report, Failure.prepFailure(e)); + return fail(testReport, Failure.prepFailure(e)); } }); } @@ -111,4 +120,4 @@ export const runSuite = (suite: Suite, state: RunState, actions: RunActions, rep export const runSuites = (suites: Suite[], state: RunState, actions: RunActions, reporter: RunReporter): Promise => { return loop(suites, (suite) => runSuite(suite, state, actions, reporter)); -}; \ No newline at end of file +}; diff --git a/modules/runner/src/main/ts/runner/Utils.ts b/modules/runner/src/main/ts/runner/Utils.ts index be4dcfcd..84bff409 100644 --- a/modules/runner/src/main/ts/runner/Utils.ts +++ b/modules/runner/src/main/ts/runner/Utils.ts @@ -1,5 +1,4 @@ import { Suite, Test } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; export const countTests = (suite: Suite): number => suite.tests.length + suite.suites.reduce((acc, suite) => acc + countTests(suite), 0); @@ -46,4 +45,4 @@ export const filterOnly = (suite: Suite): void => { suite.suites.forEach(filterOnly); suite.suites = onlySuites; } -}; \ No newline at end of file +}; diff --git a/modules/runner/src/test/ts/TestUtils.ts b/modules/runner/src/test/ts/TestUtils.ts index df527ea8..41d95294 100644 --- a/modules/runner/src/test/ts/TestUtils.ts +++ b/modules/runner/src/test/ts/TestUtils.ts @@ -7,4 +7,6 @@ export const range = (count: number, fn: (idx: number) => T): T[] => { r.push(fn(i)); } return r; -}; \ No newline at end of file +}; + +export const wait = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); diff --git a/modules/runner/src/test/ts/reporter/ReporterTest.ts b/modules/runner/src/test/ts/reporter/ReporterTest.ts index 7069226a..c8e389fe 100644 --- a/modules/runner/src/test/ts/reporter/ReporterTest.ts +++ b/modules/runner/src/test/ts/reporter/ReporterTest.ts @@ -1,15 +1,15 @@ import { Failure, LoggedError } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import { assert } from 'chai'; import * as fc from 'fast-check'; import { beforeEach, describe, it } from 'mocha'; import { UrlParams } from '../../../main/ts/core/UrlParams'; import { Callbacks, TestErrorData } from '../../../main/ts/reporter/Callbacks'; import { Reporter } from '../../../main/ts/reporter/Reporter'; -import { noop } from '../TestUtils'; +import {noop, wait} from '../TestUtils'; interface StartTestData { readonly session: string; + readonly currentCount: number; readonly totalTests: number; readonly file: string; readonly name: string; @@ -51,12 +51,12 @@ describe('Reporter.test', () => { loadHarness: () => Promise.resolve({ retries: 0, chunk: 100, stopOnFailure: true, mode: 'manual', timeout: 10000 }), sendKeepAlive: () => Promise.resolve(), sendInit: () => Promise.resolve(), - sendTestStart: (session, totalTests, file, name) => { - startTestData.push({ session, totalTests, file, name }); + sendTestStart: (session, currentCount, totalTests, file, name) => { + startTestData.push({ session, currentCount, totalTests, file, name }); return Promise.resolve(); }, - sendTestResult: (session, file, name, passed, time, error, skipped) => { - endTestData.push({ session, file, name, passed, time, error, skipped }); + sendTestResults: (session, results) => { + results.forEach(r => endTestData.push({session, ...r})); return Promise.resolve(); }, sendDone: (session, error) => { @@ -66,8 +66,8 @@ describe('Reporter.test', () => { } }; - const reset = () => { - offset = Math.floor(Math.random() * 1000); + const reset = (newOffset: number = Math.floor(Math.random() * 1000)) => { + offset = newOffset; reporter = Reporter({ ...params, offset }, callbacks, ui); startTestData = []; endTestData = []; @@ -75,22 +75,24 @@ describe('Reporter.test', () => { doneError = undefined; }; - beforeEach(reset); + beforeEach(() => reset()); it('should report the session id, number tests, file and name on start', () => { return fc.assert(fc.asyncProperty(fc.hexaString(), fc.asciiString(), fc.integer(offset), (fileName, testName, testCount) => { - reset(); + reset(0); const test = reporter.test(fileName + 'Test.ts', testName, testCount); - return test.start().then(() => { - assert.equal(startTestData.length, 1); + test.start(); + return reporter.waitForResults().then(() => { + assert.equal(startTestData.length, 1, 'Checking there is start test data'); assert.deepEqual(startTestData[0], { + currentCount: offset + 1, session: sessionId, totalTests: testCount, file: fileName + 'Test.ts', name: testName - }); + }, 'Checking start test data contents'); - assert.equal(endTestData.length, 0); + assert.equal(endTestData.length, 0, 'Checking there is no end test data'); assert.deepEqual(reporter.summary(), { offset, passed: offset, @@ -103,51 +105,23 @@ describe('Reporter.test', () => { })); }); - it('should report the session id, file, name, passed state and time on a test success', () => { - return fc.assert(fc.asyncProperty(fc.hexaString(), fc.asciiString(), fc.integer(offset), (fileName, testName, testCount) => { - reset(); - const test = reporter.test(fileName + 'Test.ts', testName, testCount); - return test.start() - .then(test.pass) - .then(() => { - assert.equal(endTestData.length, 1); - const data = endTestData[0]; - assert.equal(data.session, sessionId); - assert.equal(data.file, fileName + 'Test.ts'); - assert.equal(data.name, testName); - assert.isTrue(data.passed); - assert.isNull(data.skipped); - assert.isNull(data.error); - assert.isString(data.time); - - assert.deepEqual(reporter.summary(), { - offset, - passed: offset + 1, - failed: 0, - skipped: 0 - }, 'Summary has one passed test'); - - assert.isFalse(doneCalled); - }); - })); - }); - it('should report the session id, file, name, passed state and time on a skipped test', () => { return fc.assert(fc.asyncProperty(fc.hexaString(), fc.asciiString(), fc.asciiString(), fc.integer(offset), (fileName, testName, skippedMessage, testCount) => { reset(); const test = reporter.test(fileName + 'Test.ts', testName, testCount); - return test.start() - .then(() => test.skip(skippedMessage)) + test.start(); + test.skip(skippedMessage); + return reporter.waitForResults() .then(() => { - assert.equal(endTestData.length, 1); + assert.equal(endTestData.length, 1, 'Checking there is end test data'); const data = endTestData[0]; - assert.equal(data.session, sessionId); - assert.equal(data.file, fileName + 'Test.ts'); - assert.equal(data.name, testName); - assert.isFalse(data.passed); - assert.equal(data.skipped, skippedMessage); - assert.isNull(data.error); - assert.isString(data.time); + assert.equal(data.session, sessionId, 'Checking session ID'); + assert.equal(data.file, fileName + 'Test.ts', 'Checking filename'); + assert.equal(data.name, testName, 'Checking testname'); + assert.isFalse(data.passed, 'Checking passed state'); + assert.equal(data.skipped, skippedMessage, 'Checking skipped message'); + assert.isNull(data.error, 'Checking no error'); + assert.isString(data.time, 'Checking time'); assert.deepEqual(reporter.summary(), { offset, @@ -167,18 +141,19 @@ describe('Reporter.test', () => { const test = reporter.test(fileName + 'Test.ts', testName, testCount); const error = LoggedError.loggedError(new Error('Failed'), [ 'Log Message' ]); - return test.start() - .then(() => test.fail(error)) + test.start(); + test.fail(error); + return reporter.waitForResults() .then(() => { - assert.equal(endTestData.length, 1); + assert.equal(endTestData.length, 1, 'Checking there is end test data'); const data = endTestData[ 0 ]; - assert.equal(data.session, sessionId); - assert.equal(data.file, fileName + 'Test.ts'); - assert.equal(data.name, testName); - assert.isFalse(data.passed); - assert.isNull(data.skipped); - assert.isString(data.time); - assert.equal(data.error?.text, 'Error: Failed\n\nLogs:\nLog Message'); + assert.equal(data.session, sessionId, 'Checking session ID'); + assert.equal(data.file, fileName + 'Test.ts', 'Checking filename'); + assert.equal(data.name, testName, 'Checking testname'); + assert.isFalse(data.passed, 'Checking passed state'); + assert.isNull(data.skipped, 'Checking skipped state'); + assert.isString(data.time, 'Checking time'); + assert.equal(data.error?.text, 'Error: Failed\n\nLogs:\nLog Message', 'Checking error text'); assert.deepEqual(reporter.summary(), { offset, @@ -192,15 +167,20 @@ describe('Reporter.test', () => { })); }); - it('should report done', () => { + it('should report done', async () => { reporter.done(); + // done waits about 100ms, so we have to wait 150 + await wait(150); assert.isTrue(doneCalled); assert.isUndefined(doneError); }); - it('should report done with an error', () => { + it('should report done with an error', async () => { reporter.done(Failure.prepFailure('Unexpected error occurred')); + // done waits about 100ms, so we have to wait 150 + await wait(150); + await Promise.resolve(); assert.isTrue(doneCalled); assert.include(doneError, 'Unexpected error occurred'); }); -}); \ No newline at end of file +}); diff --git a/modules/runner/src/test/ts/runner/RunnerTestUtils.ts b/modules/runner/src/test/ts/runner/RunnerTestUtils.ts index 4e0d74dd..5dd13332 100644 --- a/modules/runner/src/test/ts/runner/RunnerTestUtils.ts +++ b/modules/runner/src/test/ts/runner/RunnerTestUtils.ts @@ -1,5 +1,4 @@ import { Hook, HookType, LoggedError, Suite } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import { createHook } from '../../../main/ts/core/Hook'; import { Reporter, TestReporter } from '../../../main/ts/reporter/Reporter'; import { RunState } from '../../../main/ts/runner/TestRun'; @@ -53,6 +52,8 @@ export const MockReporter = (): MockReporter => { test, summary: () => ({ offset: 0, passed, failed: failures.length, skipped }), done: noop, + waitForResults: Promise.resolve, + retry: Promise.resolve, failures: () => failures }; }; @@ -63,4 +64,4 @@ export const createRunState = (offset: number, chunk: number, count = 0): RunSta chunk, timeout: TEST_TIMEOUT, testCount: count -}); \ No newline at end of file +}); diff --git a/modules/runner/src/test/ts/runner/TestRunTest.ts b/modules/runner/src/test/ts/runner/TestRunTest.ts index d6fb597e..8a75cdb6 100644 --- a/modules/runner/src/test/ts/runner/TestRunTest.ts +++ b/modules/runner/src/test/ts/runner/TestRunTest.ts @@ -1,5 +1,4 @@ import { Context, HookType, RunnableState, Suite, Test } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import { assert } from 'chai'; import * as fc from 'fast-check'; import { beforeEach, describe, it } from 'mocha'; @@ -355,4 +354,4 @@ describe('TestRun.runSuite', () => { assert.deepEqual(hooks, [ HookType.Before, HookType.BeforeEach, HookType.AfterEach, HookType.BeforeEach, HookType.AfterEach, HookType.After ]); }); }); -}); \ No newline at end of file +}); diff --git a/modules/runner/src/test/ts/runner/UtilsTest.ts b/modules/runner/src/test/ts/runner/UtilsTest.ts index e75e0b94..d912c210 100644 --- a/modules/runner/src/test/ts/runner/UtilsTest.ts +++ b/modules/runner/src/test/ts/runner/UtilsTest.ts @@ -1,5 +1,4 @@ import { Suite } from '@ephox/bedrock-common'; -import Promise from '@ephox/wrap-promise-polyfill'; import { assert } from 'chai'; import * as fc from 'fast-check'; import { describe, it } from 'mocha'; @@ -151,4 +150,4 @@ describe('Utils.filterOnly', () => { assert.lengthOf(root.suites, 1); })); }); -}); \ No newline at end of file +}); diff --git a/modules/sample/package.json b/modules/sample/package.json index af3b4aa7..8e5ac4a8 100644 --- a/modules/sample/package.json +++ b/modules/sample/package.json @@ -1,6 +1,6 @@ { "name": "@ephox/bedrock-sample", - "version": "14.1.4", + "version": "15.0.0-alpha.4", "author": "Tiny Technologies Inc", "license": "Apache-2.0", "scripts": { @@ -15,11 +15,10 @@ "test": "yarn test-samples-pass && yarn test-samples-only && yarn test-samples-pass-js" }, "dependencies": { - "@ephox/bedrock-client": "^14.1.1", - "@ephox/wrap-promise-polyfill": "^2.2.1" + "@ephox/bedrock-client": "^14.1.1" }, "devDependencies": { - "@ephox/bedrock-server": "^14.1.4" + "@ephox/bedrock-server": "^15.0.0-alpha.4" }, "files": [], "private": true, diff --git a/modules/sample/src/test/ts/client/fail/AsyncFailTest.ts b/modules/sample/src/test/ts/client/fail/AsyncFailTest.ts index 10f39502..af12629b 100644 --- a/modules/sample/src/test/ts/client/fail/AsyncFailTest.ts +++ b/modules/sample/src/test/ts/client/fail/AsyncFailTest.ts @@ -1,5 +1,4 @@ import { UnitTest } from '@ephox/bedrock-client'; -import Promise from '@ephox/wrap-promise-polyfill'; UnitTest.asyncTest('AsyncFail Test 1', (success, failure) => { setTimeout(() => { diff --git a/modules/sample/src/test/ts/client/pass/AsyncPassTest.ts b/modules/sample/src/test/ts/client/pass/AsyncPassTest.ts index 93dd6c86..ba3bdf26 100644 --- a/modules/sample/src/test/ts/client/pass/AsyncPassTest.ts +++ b/modules/sample/src/test/ts/client/pass/AsyncPassTest.ts @@ -1,5 +1,4 @@ import { UnitTest } from '@ephox/bedrock-client'; -import Promise from '@ephox/wrap-promise-polyfill'; UnitTest.asyncTest('AsyncPass Test 1', (success, _failure) => { new Promise(function (resolve, _reject) { diff --git a/modules/sample/src/test/ts/utils/Utils.ts b/modules/sample/src/test/ts/utils/Utils.ts index 07835f45..aa4730b2 100644 --- a/modules/sample/src/test/ts/utils/Utils.ts +++ b/modules/sample/src/test/ts/utils/Utils.ts @@ -1,4 +1,3 @@ -import Promise from '@ephox/wrap-promise-polyfill'; const post = (url: string, data: Record): Promise => { return new Promise((onSuccess, onFailure) => { @@ -39,4 +38,4 @@ export { sendKeyCombo, sendText, sendMouse -}; \ No newline at end of file +}; diff --git a/modules/server/package.json b/modules/server/package.json index c0370d18..748850f7 100644 --- a/modules/server/package.json +++ b/modules/server/package.json @@ -1,6 +1,6 @@ { "name": "@ephox/bedrock-server", - "version": "14.1.4", + "version": "15.0.0-alpha.4", "author": "Tiny Technologies Inc", "license": "Apache-2.0", "bin": { @@ -20,7 +20,7 @@ "@aws-sdk/client-device-farm": "^3.354.0", "@ephox/bedrock-client": "^14.1.1", "@ephox/bedrock-common": "^14.1.1", - "@ephox/bedrock-runner": "^14.1.1", + "@ephox/bedrock-runner": "^15.0.0-alpha.4", "@jsdevtools/coverage-istanbul-loader": "^3.0.5", "@lambdatest/node-tunnel": "^4.0.4", "@wdio/globals": "^8.14.1", diff --git a/modules/server/src/main/ts/bedrock/cli/Hud.ts b/modules/server/src/main/ts/bedrock/cli/Hud.ts index e0f5dc75..4e05f85d 100644 --- a/modules/server/src/main/ts/bedrock/cli/Hud.ts +++ b/modules/server/src/main/ts/bedrock/cli/Hud.ts @@ -14,12 +14,19 @@ interface ResultData { export interface Hud { readonly update: (data: ResultData) => Promise; + readonly warn: (...messages: any[]) => void; readonly complete: () => Promise; } export const create = (testfiles: string[], loglevel: 'simple' | 'advanced'): Hud => { let started = false; + const warn = (...messages: any[]): void => { + // disable the next cursor movement so the message isn't overwritten + started = false; + console.warn(...messages); + }; + const stream = process.stdout; const numFiles = testfiles.length > 0 ? testfiles.length : '?'; @@ -49,7 +56,8 @@ export const create = (testfiles: string[], loglevel: 'simple' | 'advanced'): Hu readline.clearLine(stream, 0); readline.cursorTo(stream, 0); } - stream.write('Current test: ' + (data.test !== undefined ? data.test.substring(0, 60) : 'Unknown') + '\n'); + const currentTestMessage = 'Current test: ' + (data.test !== undefined ? data.test : 'Unknown') + '\n'; + stream.write(currentTestMessage.substring(0, Env.IS_CI ? undefined : process.stdout.columns)); const totalFiles = data.totalFiles !== undefined ? data.totalFiles : numFiles; const totalTests = data.totalTests !== undefined ? data.totalTests : totalFiles; return writeProgress(data.id, data.done, data.numPassed, data.numSkipped, data.numFailed, totalTests); @@ -72,6 +80,7 @@ export const create = (testfiles: string[], loglevel: 'simple' | 'advanced'): Hu return { update: loglevel === 'advanced' && supportsAdvanced ? advUpdate : basicUpdate, + warn, complete }; }; diff --git a/modules/server/src/main/ts/bedrock/server/Apis.ts b/modules/server/src/main/ts/bedrock/server/Apis.ts index c534afa6..05c6c650 100644 --- a/modules/server/src/main/ts/bedrock/server/Apis.ts +++ b/modules/server/src/main/ts/bedrock/server/Apis.ts @@ -23,11 +23,13 @@ interface StartData { readonly session: string; readonly name: string; readonly file: string; + readonly number: number; readonly totalTests: number; } -interface ResultData extends Controller.TestResult { +export interface ResultsData { readonly session: string; + readonly results: Controller.TestResult[]; } interface DoneData { @@ -42,7 +44,7 @@ const pollRate = 200; const maxInvalidAttempts = 300; // TODO: Do not use files here. -export const create = (master: DriverMaster | null, maybeDriver: Attempt, projectdir: string, basedir: string, stickyFirstSession: boolean, singleTimeout: number, overallTimeout: number, testfiles: string[], loglevel: 'simple' | 'advanced', resetMousePosition: boolean): Apis => { +export const create = (master: DriverMaster | null, maybeDriver: Attempt, projectdir: string, basedir: string, stickyFirstSession: boolean, overallTimeout: number, testfiles: string[], loglevel: 'simple' | 'advanced', resetMousePosition: boolean): Apis => { let pageHasLoaded = false; let needsMousePositionReset = true; @@ -119,7 +121,7 @@ export const create = (master: DriverMaster | null, maybeDriver: Attempt Promise.all([ resetMousePositionAction(true), keepAliveAction() ])), Routes.effect('POST', '/tests/start', (data: StartData) => { - c.recordTestStart(data.session, data.name, data.file, data.totalTests); + c.recordTestStart(data.session, data.name, data.file, data.number, data.totalTests); return resetMousePositionAction(); }), - Routes.effect('POST', '/tests/result', (data: ResultData) => { - c.recordTestResult(data.session, data.name, data.file, data.passed, data.time, data.error, data.skipped); + Routes.effect('POST', '/tests/results', (data: ResultsData) => { + c.recordTestResults(data.session, data.results); return Promise.resolve(); }), Routes.effect('POST', '/tests/done', (data: DoneData) => { diff --git a/modules/server/src/main/ts/bedrock/server/Controller.ts b/modules/server/src/main/ts/bedrock/server/Controller.ts index 2d310c9e..cbd7ddf1 100644 --- a/modules/server/src/main/ts/bedrock/server/Controller.ts +++ b/modules/server/src/main/ts/bedrock/server/Controller.ts @@ -1,7 +1,6 @@ import { ErrorData } from '@ephox/bedrock-common'; import * as Hud from '../cli/Hud'; import * as Type from '../util/Type'; -import * as Env from '../util/Env'; export interface TestErrorData { readonly data: ErrorData; @@ -24,13 +23,9 @@ export interface TestResults { readonly now: number; } -interface InflightTest { +interface PreviousTest { readonly name: string; readonly file: string; - readonly start: number; -} - -interface PreviousTest extends InflightTest { readonly end: number; } @@ -40,25 +35,23 @@ interface TestSession { readonly lookup: Record>; alive: number; updated: number; - inflight: InflightTest | null; previous: PreviousTest | null; done: boolean; error?: string; totalTests: number; + currentTest: number; } export interface Controller { readonly enableHud: () => void; readonly recordAlive: (sessionId: string) => void; - readonly recordTestStart: (id: string, name: string, file: string, totalTests: number) => void; - readonly recordTestResult: (id: string, name: string, file: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string) => void; + readonly recordTestStart: (id: string, name: string, file: string, currentCount: number, totalTests: number) => void; + readonly recordTestResults: (id: string, results: TestResult[]) => void; readonly recordDone: (id: string, error?: string) => void; readonly awaitDone: () => Promise; } -// allow a little extra time for a test timeout so the runner can handle it gracefully -const timeoutGrace = 2000; -export const create = (stickyFirstSession: boolean, singleTimeout: number, overallTimeout: number, testfiles: string[], loglevel: 'simple' | 'advanced'): Controller => { +export const create = (stickyFirstSession: boolean, overallTimeout: number, testfiles: string[], loglevel: 'simple' | 'advanced'): Controller => { const hud = Hud.create(testfiles, loglevel); const sessions: Record = {}; let stickyId: string | null = null; @@ -93,10 +86,10 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa updated: now, results: [], lookup: {}, - inflight: null, previous: null, done: false, - totalTests: testfiles.length + totalTests: testfiles.length, + currentTest: 0 }; sessions[sessionId] = session; } @@ -108,21 +101,15 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa outputToHud = true; }; - const shouldUpdateHud = (session: TestSession): boolean => { - if (!outputToHud) return false; - if (stickyFirstSession && (timeoutError || session.id !== stickyId)) return false; - if (!Env.IS_CI || session.done || !session.results.at(-1)?.passed) return true; - // Only update the HUD at 10% intervals on remote: - return session.results.length % Math.round(session.totalTests * 0.1) === 0; - }; - const updateHud = (session: TestSession) => { - if (!shouldUpdateHud(session)) return; + if (!outputToHud) return; + if (stickyFirstSession && (timeoutError || session.id !== stickyId)) return; const id = session.id; const numFailed = session.results.reduce((sum, res) => sum + (res.passed || res.skipped ? 0 : 1), 0); const numSkipped = session.results.reduce((sum, res) => sum + (res.skipped ? 1 : 0), 0); const numPassed = session.results.length - numFailed - numSkipped; - const test = session.inflight !== null ? session.inflight.name : (session.previous !== null ? session.previous.name : ''); + + const test = session.previous !== null ? session.previous.name : ''; const done = session.done; hud.update({id, test, numPassed, numSkipped, numFailed, done, totalTests: session.totalTests}); }; @@ -131,24 +118,29 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa getSession(sessionId); }; - const recordTestStart = (id: string, name: string, file: string, totalTests: number) => { + const recordTestStart = (id: string, name: string, file: string, currentCount: number, totalTests: number) => { const session = getSession(id); - const start = Date.now(); - session.inflight = {name, file, start}; - session.updated = Date.now(); + const now = Date.now(); + session.updated = now; session.totalTests = totalTests; + session.currentTest = currentCount; session.done = false; - if (!session.results.length || !Env.IS_CI) { - // Update HUD on test starts when in CI only on the very first update i.e. `progress: 0/0`, otherwise skip them. - updateHud(session); - } + // a bit of a lie, but we only ever get 1 start now + session.previous = { + name, + file, + end: now + }; + updateHud(session); }; - const recordTestResult = (id: string, name: string, file: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string) => { - const now = Date.now(); - const session = getSession(id); - const record = { name, file, passed, time, error, skipped }; + const recordTestResult = (session: TestSession, record: TestResult) => { + const {name, file} = record; if (session.lookup[file] !== undefined && session.lookup[file][name] !== undefined) { + const existing = session.results[session.lookup[file][name]]; + if (!existing.error) { + hud.warn(`WARNING: overwriting test that didn't fail!`, file, '::', name); + } // rerunning a test session.results[session.lookup[file][name]] = record; } else { @@ -157,23 +149,35 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa session.lookup[file][name] = session.results.length; session.results.push(record); } - // this check is just in case the test start arrives before the result of the previous - if (session.inflight !== null && session.inflight.file === file && session.inflight.name === name) { + }; + + const recordTestResults = (id: string, results: TestResult[]) => { + const now = Date.now(); + const session = getSession(id); + session.updated = now; + session.done = false; + if (results.length > 0) { + const {name, file} = results.slice(-1)[0]; session.previous = { - ...session.inflight, + name, + file, end: now }; - session.inflight = null; + results.forEach( + (record) => + recordTestResult(session, record) + ); + updateHud(session); } - session.updated = now; - session.done = false; - updateHud(session); }; const recordDone = (id: string, error?: string) => { const session = getSession(id); session.done = true; session.error = error; + if (!error) { + session.currentTest = session.totalTests; + } session.updated = Date.now(); updateHud(session); }; @@ -214,20 +218,10 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa } clearInterval(poller); } else { - if (session.inflight !== null && (now - session.inflight.start) > (singleTimeout + timeoutGrace)) { - // one test took too long - const elapsed = formatTime(now - session.inflight.start); - const message = 'Test: ' + testName(session.inflight) + ' ran too long (' + elapsed + '). Limit for an individual test is set to: ' + formatTime(singleTimeout); - reject({message, results, start, now}); - clearInterval(poller); - timeoutError = true; - } else if (allElapsed > overallTimeout) { + if (allElapsed > overallTimeout) { // combined tests took too long let lastTest; - if (session.inflight !== null) { - const runningTime = now - session.inflight.start; - lastTest = 'Current test: ' + testName(session.inflight) + ' running ' + formatTime(runningTime) + '.'; - } else if (session.previous !== null) { + if (session.previous !== null) { const sincePrevious = now - session.previous.end; lastTest = 'Previous test: ' + testName(session.previous) + ' finished ' + formatTime(sincePrevious) + ' ago.'; } else { @@ -261,7 +255,7 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa enableHud, recordAlive, recordTestStart, - recordTestResult, + recordTestResults, recordDone, awaitDone }; diff --git a/modules/server/src/main/ts/bedrock/server/EffectUtils.ts b/modules/server/src/main/ts/bedrock/server/EffectUtils.ts index 370aec90..ebd807b0 100644 --- a/modules/server/src/main/ts/bedrock/server/EffectUtils.ts +++ b/modules/server/src/main/ts/bedrock/server/EffectUtils.ts @@ -39,8 +39,10 @@ const performActionOnFrame = async (driver: Browser, selector: string, action await driver.switchToFrame(null); return result; } catch (err: any) { - await driver.switchToFrame(null); - return Promise.reject(err); + return driver.status().then(status => { + console.log('webdriver failed with status', status); + return Promise.reject(err); + }); } }; @@ -63,4 +65,4 @@ export const performActionOnTarget = (driver: Browser, data: { selector: stri const selector = data.selector; const performer = selector.indexOf('=>') > -1 ? performActionOnFrame : performActionOnMain; return performer(driver, selector, action); -}; \ No newline at end of file +}; diff --git a/modules/server/src/main/ts/bedrock/server/Routes.ts b/modules/server/src/main/ts/bedrock/server/Routes.ts index 61fe0f3c..5b518a7b 100644 --- a/modules/server/src/main/ts/bedrock/server/Routes.ts +++ b/modules/server/src/main/ts/bedrock/server/Routes.ts @@ -30,7 +30,8 @@ const doResponse = (request: IncomingMessage, response: ServerResponse, status: } else { response.writeHead(status, { 'Content-Type': contentType, - 'Cache-Control': 'public, max-age=0' + 'Cache-Control': 'public, max-age=0', + 'Keep-Alive': 'timeout=120' // Avoid 502 errors }); response.end(data); } diff --git a/modules/server/src/main/ts/bedrock/server/Serve.ts b/modules/server/src/main/ts/bedrock/server/Serve.ts index 706cdc49..a1845fec 100644 --- a/modules/server/src/main/ts/bedrock/server/Serve.ts +++ b/modules/server/src/main/ts/bedrock/server/Serve.ts @@ -23,7 +23,6 @@ export interface ServeSettings { readonly overallTimeout: number; readonly projectdir: string; readonly runner: Routes.Runner; - readonly singleTimeout: number; readonly skipResetMousePosition: boolean; readonly stickyFirstSession: boolean; readonly testfiles: string[]; @@ -66,12 +65,11 @@ export const startCustom = async (settings: ServeSettings, createServer: (port: const maybeDriver = pref('driver'); const master = pref('master'); const stickyFirstSession = settings.stickyFirstSession; - const singleTimeout = pref('singleTimeout'); const overallTimeout = pref('overallTimeout'); const resetMousePosition = !pref('skipResetMousePosition'); const runner = pref('runner'); - const api = Apis.create(master, maybeDriver, projectdir, basedir, stickyFirstSession, singleTimeout, overallTimeout, testfiles, settings.loglevel, resetMousePosition); + const api = Apis.create(master, maybeDriver, projectdir, basedir, stickyFirstSession, overallTimeout, testfiles, settings.loglevel, resetMousePosition); const routers = runner.routers.concat( api.routers, @@ -107,10 +105,12 @@ export const startCustom = async (settings: ServeSettings, createServer: (port: export const start = (settings: ServeSettings): Promise => { return startCustom(settings, (port, listener) => { const server = http.createServer(listener); + server.requestTimeout = 120000; return { start: () => { return new Promise((resolve) => { server.listen(port, resolve); + server.keepAliveTimeout = 120000; }); }, stop: () => new Promise((resolve, reject) => { diff --git a/modules/server/src/resources/html/bedrock.html b/modules/server/src/resources/html/bedrock.html index 42b1c17c..2f84b661 100644 --- a/modules/server/src/resources/html/bedrock.html +++ b/modules/server/src/resources/html/bedrock.html @@ -4,6 +4,7 @@ +

Bedrock: Test Harness

diff --git a/yarn.lock b/yarn.lock index bd6bbb2f..abf1f2b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -987,11 +987,6 @@ resolved "https://registry.npmjs.org/@ephox/dispute/-/dispute-1.0.4.tgz" integrity sha512-PzklC1q7Flovi/pnxTFsY1pPKaEAm0whC+lFXM2Sh4Fc+lUt7A/UMRoV9bg9DyeTbD/mWEr/hvdyJp6hI14uKg== -"@ephox/wrap-promise-polyfill@^2.2.0", "@ephox/wrap-promise-polyfill@^2.2.1": - version "2.2.1" - resolved "https://registry.npmjs.org/@ephox/wrap-promise-polyfill/-/wrap-promise-polyfill-2.2.1.tgz" - integrity sha512-hg2IKjR3E8awv6ZPUZtHVLa+ZQkaN1ksVacw40Jq2gg6uMU9GkBTcSOl+9PswBzva2Jg0lxX0r0ipMAWemDwsA== - "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" resolved "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz" @@ -4433,12 +4428,7 @@ deepmerge-ts@^5.0.0, deepmerge-ts@^5.1.0: resolved "https://registry.yarnpkg.com/deepmerge-ts/-/deepmerge-ts-5.1.0.tgz#c55206cc4c7be2ded89b9c816cf3608884525d7a" integrity sha512-eS8dRJOckyo9maw9Tu5O5RUi/4inFLrnoLkBe3cPfDMx3WZioXtmOew4TXQaxq7Rhl4xjDtR7c6x8nNTxOvbFw== -deepmerge@^4.2.2: - version "4.3.1" - resolved "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz" - integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== - -deepmerge@^4.3.1: +deepmerge@^4.2.2, deepmerge@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==