TINY-11177: Vastly improve remote testing

CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+## Added
+- New server-side APIs to accept a batch of results instead of a single result #TINY-11177
+
+## Changed
+- Reverted TINY-10708 which was a server-side fix
+- Client no longer waits for log requests to complete between tests, which should speed up remote testing #TINY-11177
+- Console HUD no longer updates for individual tests #TINY-11177
+- Client now posts test status only in batches every 30 seconds, this is the only time the console HUD will update #TINY-11177
+
+## Removed
+- Single result server-side API #TINY-11177
+- Server-side monitoring of single test timeouts. This is still monitored client side. #TINY-11177
+- The Promise polyfill is no longer allowed on modern NodeJS frameworks so it has been removed. #TINY-11177
+
 ## 14.1.4 - 2024-03-27
 
 ### Fixed

Jenkinsfile

@@ -21,19 +21,35 @@ timestamps {
 
     stage("test") {
       exec('yarn test')
-
-      bedrockBrowsers(
-        prepareTests: {
-          yarnInstall()
-          exec('yarn build')
-        },
-        testDirs: [ 'modules/sample/src/test/ts/**/pass' ],
-        custom: '--config modules/sample/tsconfig.json --customRoutes modules/sample/routes.json --polyfills Promise Symbol'
-      )
     }
+  }
+
+  // Testing
+  stage("bedrock testing") {
+    bedrockRemoteBrowsers(
+      platforms: [
+        [ browser: 'chrome', provider: 'aws', buckets: 2 ],
+        [ browser: 'firefox', provider: 'aws', buckets: 2 ],
+        [ browser: 'edge', provider: 'lambdatest', buckets: 1 ],
+        [ browser: 'chrome', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1 ],
+        [ browser: 'firefox', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1 ],
+        [ browser: 'safari', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1 ],
+      ],
+      prepareTests: {
+        yarnInstall()
+        sh 'yarn build'
+      },
+      testDirs: [ 'modules/sample/src/test/ts/**/pass' ],
+      custom: '--config modules/sample/tsconfig.json --customRoutes modules/sample/routes.json --polyfills Promise Symbol'
+    )
+  }
 
-    if (isReleaseBranch()) {
-      stage("publish") {
+  // Publish
+  if (isReleaseBranch()) {
+    stage("publish") {
+      tinyPods.node() {
+        yarnInstall()
+        sh 'yarn build'
         tinyNpm.withNpmPublishCredentials {
           // We need to tell git to ignore the changes to .npmrc when publishing
           exec('git update-index --assume-unchanged .npmrc')

lerna.json

@@ -1,7 +1,7 @@
 {
   "npmClient": "yarn",
   "useWorkspaces": true,
-  "version": "14.1.4",
+  "version": "15.0.0-alpha.3",
   "publish": {
     "push": false
   }

modules/runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ephox/bedrock-runner",
-  "version": "14.1.1",
+  "version": "15.0.0-alpha.3",
   "author": "Tiny Technologies Inc",
   "license": "Apache-2.0",
   "scripts": {
@@ -11,7 +11,6 @@
   },
   "dependencies": {
     "@ephox/bedrock-common": "^14.1.1",
-    "@ephox/wrap-promise-polyfill": "^2.2.0",
     "jquery": "^3.4.1",
     "querystringify": "^2.1.1"
   },

modules/runner/src/main/ts/api/Main.ts

@@ -1,5 +1,4 @@
 import { Failure, Global } from '@ephox/bedrock-common';
-import Promise from '@ephox/wrap-promise-polyfill';
 import * as Globals from '../core/Globals';
 import * as TestLoader from '../core/TestLoader';
 import { UrlParams } from '../core/UrlParams';

modules/runner/src/main/ts/core/TestLoader.ts

@@ -1,4 +1,3 @@
-import Promise from '@ephox/wrap-promise-polyfill';
 import { ErrorCatcher } from '../errors/ErrorCatcher';
 
 export const load = (scriptUrl: string): Promise<void> =>
@@ -38,4 +37,4 @@ export const load = (scriptUrl: string): Promise<void> =>
 
     // Add the script to the dom to load it
     document.body.appendChild(script);
-  });
+  });

modules/runner/src/main/ts/core/Utils.ts

@@ -1,5 +1,4 @@
 import { Suite, Test } from '@ephox/bedrock-common';
-import Promise from '@ephox/wrap-promise-polyfill';
 import sourceMappedStackTrace from 'sourcemapped-stacktrace';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -62,4 +61,4 @@ export const setStack = (error: Error, stack: string | undefined): void => {
   } catch (err) {
     // Do nothing
   }
-};
+};

modules/runner/src/main/ts/reporter/Callbacks.ts

@@ -1,7 +1,16 @@
 import { ErrorData, Global } from '@ephox/bedrock-common';
-import Promise from '@ephox/wrap-promise-polyfill';
+
 import { HarnessResponse } from '../core/ServerTypes';
 
+export interface TestReport {
+  file: string;
+  name: string;
+  passed: boolean;
+  time: string;
+  skipped: string | null;
+  error: TestErrorData | null
+}
+
 export interface TestErrorData {
   readonly data: ErrorData;
   readonly text: string;
@@ -11,25 +20,34 @@ export interface Callbacks {
   readonly loadHarness: () => Promise<HarnessResponse>
   readonly sendKeepAlive: (session: string) => Promise<void>;
   readonly sendInit: (session: string) => Promise<void>;
-  readonly sendTestStart: (session: string, totalTests: number, file: string, name: string) => Promise<void>;
-  readonly sendTestResult: (session: string, file: string, name: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string | null) => Promise<void>;
+  readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string) => Promise<void>;
+  readonly sendTestResults: (session: string, results: TestReport[]) => Promise<void>;
   readonly sendDone: (session: string, error?: string) => Promise<void>;
 }
 
 declare const $: JQueryStatic;
 
-const sendJson = <T>(url: string, data: any): Promise<T> => {
+function generateErrorMessage(xhr: JQuery.jqXHR<any>, onError: (reason?: any) => void, url: string, requestDetails: string, statusText: 'timeout' | 'error' | 'abort' | 'parsererror', e: string) {
+  if (xhr.readyState === 0) {
+    onError(`Unable to open connection to ${url}, ${requestDetails}. Status text "${statusText}", error thrown "${e}"`);
+  } else {
+    onError(`Response status ${xhr.status} connecting to ${url}, ${requestDetails}. Status text "${statusText}", error thrown "${e}"`);
+  }
+}
+
+const sendJson = <T>(url: string, jsonData: any): Promise<T> => {
   return new Promise((onSuccess, onError) => {
+    const data = JSON.stringify(jsonData);
     $.ajax({
       method: 'post',
       url,
       contentType: 'application/json; charset=UTF-8',
       dataType: 'json',
       success: onSuccess,
       error: (xhr, statusText, e) => {
-        onError(e);
+        generateErrorMessage(xhr, onError, url, `sending ${data}`, statusText, e);
       },
-      data: JSON.stringify(data),
+      data,
     });
   });
 };
@@ -41,7 +59,7 @@ const getJson = <T>(url: string): Promise<T> => {
       dataType: 'json',
       success: onSuccess,
       error: (xhr, statusText, e) => {
-        onError(e);
+        generateErrorMessage(xhr, onError, url, 'as a get request', statusText, e);
       }
     });
   }));
@@ -64,24 +82,20 @@ export const Callbacks = (): Callbacks => {
     });
   };
 
-  const sendTestStart = (session: string, totalTests: number, file: string, name: string): Promise<void> => {
+  const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string): Promise<void> => {
     return sendJson('/tests/start', {
+      number,
       totalTests,
       session,
       file,
       name,
     });
   };
 
-  const sendTestResult = (session: string, file: string, name: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string | null): Promise<void> => {
-    return sendJson('/tests/result', {
+  const sendTestResults = (session: string, results: TestReport[]): Promise<void> => {
+    return sendJson('/tests/results', {
       session,
-      file,
-      name,
-      passed,
-      skipped,
-      time,
-      error,
+      results,
     });
   };
 
@@ -101,7 +115,7 @@ export const Callbacks = (): Callbacks => {
     sendInit,
     sendKeepAlive,
     sendTestStart,
-    sendTestResult,
+    sendTestResults,
     sendDone
   };
-};
+};