TINY-10708: Suppress bedrock logs in remote testing (#142)

* TINY-10708: Reduce logs when on remote to 10% intervals.

And only update HUD when `ResultData` has changed

* TINY-10708: Added util/Env.ts

* TINY-10708: Use `Env.IS_CI` instead of passing along the `remote` flag

* TINY-10708: On CI, only log the first test start, otherwise skip them

They have mostly repeated data from the previous test result log.

* TINY-10708: Removed accidental `total - numSkipped`

* TINY-10708: Updated changelog

* Remove unnecessary async keyword

Co-authored-by: ltrouton <[email protected]>

* Update CHANGELOG.md

Co-authored-by: Andrew Herron <[email protected]>

* TINY-10708: Always update HUD on test failures

* TINY-10708: bedrock-sample package json scripts now use chrome-headless instead of phantomjs

---------

Co-authored-by: ltrouton <[email protected]>
Co-authored-by: Andrew Herron <[email protected]>

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 ### Improved
+
+- Reduced progress logs when on CI to 10% intervals #TINY-10708
 - Now supports using `pnpm-workspace.yaml` to fetch workspaces #TINY-10688
 
 ## 14.1.2 - 2024-01-31

modules/sample/package.json

@@ -6,12 +6,12 @@
   "scripts": {
     "bedrock": "node ../server/bin/bedrock.js",
     "bedrock-auto": "node ../server/bin/bedrock-auto.js",
-    "test-samples-pass": "bedrock-auto -b phantomjs --config tsconfig.json --customRoutes routes.json --polyfills Promise Symbol -d src/test/ts/**/pass",
-    "test-samples-only": "bedrock-auto -b phantomjs --config tsconfig.json --polyfills Promise Symbol -d src/test/ts/**/only",
-    "test-samples-pass-js": "bedrock-auto -b phantomjs --polyfills Promise Symbol -d src/test/js/**/pass",
+    "test-samples-pass": "bedrock-auto -b chrome-headless --config tsconfig.json --customRoutes routes.json --polyfills Promise Symbol -d src/test/ts/**/pass",
+    "test-samples-only": "bedrock-auto -b chrome-headless --config tsconfig.json --polyfills Promise Symbol -d src/test/ts/**/only",
+    "test-samples-pass-js": "bedrock-auto -b chrome-headless --polyfills Promise Symbol -d src/test/js/**/pass",
     "test-samples-pass-manual": "bedrock --config tsconfig.json --customRoutes routes.json --polyfills Promise Symbol -d src/test/ts/**/pass",
     "test-samples-pass-manual-js": "bedrock --polyfills Promise Symbol -d src/test/js/**/pass",
-    "test-samples-fail": "bedrock-auto -b phantomjs --config tsconfig.json -d src/test/ts/**/fail",
+    "test-samples-fail": "bedrock-auto -b chrome-headless --config tsconfig.json -d src/test/ts/**/fail",
     "test": "yarn test-samples-pass && yarn test-samples-only && yarn test-samples-pass-js"
   },
   "dependencies": {

modules/server/src/main/ts/BedrockAuto.ts

@@ -111,7 +111,7 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
 
       return Lifecycle.done(result, webdriver, shutdown(shutdownServices), settings.gruntDone, settings.delayExit);
     } catch (e) {
-      return Lifecycle.error(e, webdriver, shutdown(shutdownServices), settings.gruntDone, settings.delayExit);
+      return Lifecycle.error(e as any, webdriver, shutdown(shutdownServices), settings.gruntDone, settings.delayExit);
     }
   }).catch(async (err) => {
     // Chalk does not use a formatter. Using node's built-in to expand Objects, etc.

modules/server/src/main/ts/bedrock/cli/Hud.ts

@@ -1,4 +1,5 @@
 import * as readline from 'readline';
+import * as Env from '../util/Env';
 
 interface ResultData {
   readonly done: boolean;
@@ -25,24 +26,29 @@ export const create = (testfiles: string[], loglevel: 'simple' | 'advanced'): Hu
 
   const writeProgress = (id: string, stopped: boolean, numPassed: number, numSkipped: number, numFailed: number, total: number | '?') => {
     const numRun = numPassed + numFailed + numSkipped;
+    total = total === '?' ? -Infinity : total;
     const status = stopped ? (numRun < total ? 'STOPPED' : 'COMPLETE') : 'RUNNING';
     stream.write(
       'Session: ' + id + ', Status: ' + status + ', Progress: ' + numRun + '/' + total +
       ', Failed: ' + numFailed + ', Skipped: ' + numSkipped + ' ... ' + '\n'
     );
-    readline.clearLine(stream, 1);
+    if (!Env.IS_CI) {
+      readline.clearLine(stream, 1);
+    }
     return Promise.resolve();
   };
 
   const advUpdate = (data: ResultData) => {
-    if (started) {
+    if (started && !Env.IS_CI) {
       // Note, this writes over the above line, which is why we only do this after the first update.
       readline.moveCursor(stream, 0, -2);
     } else {
       started = true;
     }
-    readline.clearLine(stream, 0);
-    readline.cursorTo(stream, 0);
+    if (!Env.IS_CI) {
+      readline.clearLine(stream, 0);
+      readline.cursorTo(stream, 0);
+    }
     stream.write('Current test: ' + (data.test !== undefined ? data.test.substring(0, 60) : 'Unknown') + '\n');
     const totalFiles = data.totalFiles !== undefined ? data.totalFiles : numFiles;
     const totalTests = data.totalTests !== undefined ? data.totalTests : totalFiles;

modules/server/src/main/ts/bedrock/server/Controller.ts

@@ -1,6 +1,7 @@
 import { ErrorData } from '@ephox/bedrock-common';
 import * as Hud from '../cli/Hud';
 import * as Type from '../util/Type';
+import * as Env from '../util/Env';
 
 export interface TestErrorData {
   readonly data: ErrorData;
@@ -107,9 +108,16 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa
     outputToHud = true;
   };
 
+  const shouldUpdateHud = (session: TestSession): boolean => {
+    if (!outputToHud) return false;
+    if (stickyFirstSession && (timeoutError || session.id !== stickyId)) return false;
+    if (!Env.IS_CI || session.done || !session.results.at(-1)?.passed) return true;
+    // Only update the HUD at 10% intervals on remote:
+    return session.results.length % Math.round(session.totalTests * 0.1) === 0;
+  };
+
   const updateHud = (session: TestSession) => {
-    if (!outputToHud) return;
-    if (stickyFirstSession && (timeoutError || session.id !== stickyId)) return;
+    if (!shouldUpdateHud(session)) return;
     const id = session.id;
     const numFailed = session.results.reduce((sum, res) => sum + (res.passed || res.skipped ? 0 : 1), 0);
     const numSkipped = session.results.reduce((sum, res) => sum + (res.skipped ? 1 : 0), 0);
@@ -130,7 +138,10 @@ export const create = (stickyFirstSession: boolean, singleTimeout: number, overa
     session.updated = Date.now();
     session.totalTests = totalTests;
     session.done = false;
-    updateHud(session);
+    if (!session.results.length || !Env.IS_CI) {
+      // Update HUD on test starts when in CI only on the very first update i.e. `progress: 0/0`, otherwise skip them.
+      updateHud(session);
+    }
   };
 
   const recordTestResult = (id: string, name: string, file: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string) => {

modules/server/src/main/ts/bedrock/util/Env.ts

@@ -0,0 +1 @@
+export const IS_CI = 'CI' in process.env && process.stdout.isTTY;