Add Async Runner for News Site Next Testing

resources/suite-runner.mjs

@@ -1,7 +1,6 @@
-import { TestRunner } from "./test-runner.mjs";
+import { TEST_RUNNER_LOOKUP } from "./test-runner.mjs";
 import { WarmupSuite } from "./benchmark-runner.mjs";
 
-// FIXME: Create AsyncSuiteRunner subclass.
 // FIXME: Create RemoteSuiteRunner subclass.
 export class SuiteRunner {
     #frame;
@@ -53,7 +52,8 @@ export class SuiteRunner {
             if (this.#client?.willRunTest)
                 await this.#client.willRunTest(this.#suite, test);
 
-            const testRunner = new TestRunner(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults);
+            const testRunnerClass = TEST_RUNNER_LOOKUP[this.#suite.type ?? "default"];
+            const testRunner = new testRunnerClass(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults);
             await testRunner.runTest();
         }
         performance.mark(suiteEndLabel);
@@ -100,5 +100,6 @@ class RemoteSuiteRunner extends SuiteRunner {}
 export const SUITE_RUNNER_LOOKUP = {
     __proto__: null,
     default: SuiteRunner,
+    async: SuiteRunner,
     remote: RemoteSuiteRunner,
 };

resources/test-invoker.mjs

@@ -24,7 +24,24 @@ export class TimerTestInvoker extends TestInvoker {
     }
 }
 
-export class RAFTestInvoker extends TestInvoker {
+class AsyncTimerTestInvoker extends TestInvoker {
+    start() {
+        return new Promise((resolve) => {
+            setTimeout(async () => {
+                await this._syncCallback();
+                setTimeout(() => {
+                    this._asyncCallback();
+                    requestAnimationFrame(async () => {
+                        await this._reportCallback();
+                        resolve();
+                    });
+                }, 0);
+            }, this._params.waitBeforeSync);
+        });
+    }
+}
+
+class BaseRAFTestInvoker extends TestInvoker {
     start() {
         return new Promise((resolve) => {
             if (this._params.waitBeforeSync)
@@ -33,7 +50,9 @@ export class RAFTestInvoker extends TestInvoker {
                 this._scheduleCallbacks(resolve);
         });
     }
+}
 
+class RAFTestInvoker extends BaseRAFTestInvoker {
     _scheduleCallbacks(resolve) {
         requestAnimationFrame(() => this._syncCallback());
         requestAnimationFrame(() => {
@@ -48,8 +67,31 @@ export class RAFTestInvoker extends TestInvoker {
     }
 }
 
+class AsyncRAFTestInvoker extends BaseRAFTestInvoker {
+    _scheduleCallbacks(resolve) {
+        requestAnimationFrame(async () => {
+            await this._syncCallback();
+            requestAnimationFrame(() => {
+                setTimeout(() => {
+                    this._asyncCallback();
+                    setTimeout(async () => {
+                        await this._reportCallback();
+                        resolve();
+                    }, 0);
+                }, 0);
+            });
+        });
+    }
+}
+
 export const TEST_INVOKER_LOOKUP = {
     __proto__: null,
     timer: TimerTestInvoker,
     raf: RAFTestInvoker,
 };
+
+export const ASYNC_TEST_INVOKER_LOOKUP = {
+    __proto__: null,
+    timer: AsyncTimerTestInvoker,
+    raf: AsyncRAFTestInvoker,
+};

resources/test-runner.mjs

@@ -1,4 +1,4 @@
-import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
+import { TEST_INVOKER_LOOKUP, ASYNC_TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
 
 export class TestRunner {
     #frame;
@@ -18,6 +18,30 @@ export class TestRunner {
         this.#frame = frame;
     }
 
+    get frame() {
+        return this.#frame;
+    }
+
+    get page() {
+        return this.#page;
+    }
+
+    get params() {
+        return this.#params;
+    }
+
+    get suite() {
+        return this.#suite;
+    }
+
+    get test() {
+        return this.#test;
+    }
+
+    get callback() {
+        return this.#callback;
+    }
+
     async runTest() {
         // Prepare all mark labels outside the measuring loop.
         const suiteName = this.#suite.name;
@@ -77,3 +101,74 @@ export class TestRunner {
         return invoker.start();
     }
 }
+
+export class AsyncTestRunner extends TestRunner {
+    async runTest() {
+        // Prepare all mark labels outside the measuring loop.
+        const suiteName = this.suite.name;
+        const testName = this.test.name;
+        const syncStartLabel = `${suiteName}.${testName}-start`;
+        const syncEndLabel = `${suiteName}.${testName}-sync-end`;
+        const asyncStartLabel = `${suiteName}.${testName}-async-start`;
+        const asyncEndLabel = `${suiteName}.${testName}-async-end`;
+
+        let syncTime;
+        let asyncStartTime;
+        let asyncTime;
+
+        const runSync = async () => {
+            if (this.params.warmupBeforeSync) {
+                performance.mark("warmup-start");
+                const startTime = performance.now();
+                // Infinite loop for the specified ms.
+                while (performance.now() - startTime < this.params.warmupBeforeSync)
+                    continue;
+                performance.mark("warmup-end");
+            }
+            performance.mark(syncStartLabel);
+            const syncStartTime = performance.now();
+            await this.test.run(this.page);
+            const syncEndTime = performance.now();
+            performance.mark(syncEndLabel);
+
+            syncTime = syncEndTime - syncStartTime;
+
+            performance.mark(asyncStartLabel);
+            asyncStartTime = performance.now();
+        };
+        const measureAsync = () => {
+            const bodyReference = this.frame ? this.frame.contentDocument.body : document.body;
+            const windowReference = this.frame ? this.frame.contentWindow : window;
+            // Some browsers don't immediately update the layout for paint.
+            // Force the layout here to ensure we're measuring the layout time.
+            const height = bodyReference.getBoundingClientRect().height;
+            windowReference._unusedHeightValue = height; // Prevent dead code elimination.
+
+            const asyncEndTime = performance.now();
+            performance.mark(asyncEndLabel);
+
+            asyncTime = asyncEndTime - asyncStartTime;
+
+            if (this.params.warmupBeforeSync)
+                performance.measure("warmup", "warmup-start", "warmup-end");
+            performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel);
+            performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
+        };
+
+        const report = () => this.callback(this.test, syncTime, asyncTime);
+        const invokerClass = ASYNC_TEST_INVOKER_LOOKUP[this.params.measurementMethod];
+        const invoker = new invokerClass(runSync, measureAsync, report, this.params);
+
+        return invoker.start();
+    }
+}
+
+// FIXME: implement remote steps
+class RemoteTestRunner extends TestRunner {}
+
+export const TEST_RUNNER_LOOKUP = {
+    __proto__: null,
+    default: TestRunner,
+    async: AsyncTestRunner,
+    remote: RemoteTestRunner,
+};

resources/tests.mjs

@@ -894,6 +894,49 @@ Suites.push({
     ],
 });
 
+Suites.push({
+    name: "NewsSite-Next-Async",
+    url: "resources/newssite/news-next/dist/index.html#/home",
+    tags: ["experimental", "newssite", "language"],
+    type: "async",
+    disabled: true,
+    async prepare(page) {
+        await page.waitForElement("#navbar-dropdown-toggle");
+    },
+    tests: [
+        new BenchmarkTestStep("NavigateToUS", async (page) => {
+            for (let i = 0; i < 25; i++) {
+                page.querySelector("#navbar-dropdown-toggle").click();
+                page.layout();
+                page.querySelector("#navbar-dropdown-toggle").click();
+                page.layout();
+            }
+            page.querySelector("#navbar-navlist-us-link").click();
+            page.layout();
+        }),
+        new BenchmarkTestStep("NavigateToWorld", async (page) => {
+            for (let i = 0; i < 25; i++) {
+                page.querySelector("#navbar-dropdown-toggle").click();
+                page.layout();
+                page.querySelector("#navbar-dropdown-toggle").click();
+                page.layout();
+            }
+            page.querySelector("#navbar-navlist-world-link").click();
+            page.layout();
+        }),
+        new BenchmarkTestStep("NavigateToPolitics", async (page) => {
+            for (let i = 0; i < 25; i++) {
+                page.querySelector("#navbar-dropdown-toggle").click();
+                page.layout();
+                page.querySelector("#navbar-dropdown-toggle").click();
+                page.layout();
+            }
+            page.querySelector("#navbar-navlist-politics-link").click();
+            page.layout();
+        }),
+    ],
+});
+
 Suites.push({
     name: "NewsSite-Nuxt",
     url: "resources/newssite/news-nuxt/dist/index.html",