feat(e2e): accept failed iterations and add retry option (#67)

* feat: reject Promise instead of throwing * feat: exclude failed iterations from computations * feat: add maxRetries option
bamlab · Jan 31, 2023 · 153fcd3 · 153fcd3
1 parent 6949805
commit 153fcd3
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 23 deletions.
diff --git a/packages/e2e-performance/src/bin.ts b/packages/e2e-performance/src/bin.ts
@@ -20,6 +20,14 @@ program
       .default(10)
       .argParser((arg) => parseInt(arg, 10))
   )
+  .addOption(
+    new Option(
+      "--maxRetries <maxRetries>",
+      "Maximum number of retries allowed over all iterations."
+    )
+      .default(3)
+      .argParser((arg) => parseInt(arg, 10))
+  )
   .addOption(
     new Option(
       "--duration <duration>",
@@ -49,6 +57,7 @@ program
 const runTest = async ({
   duration,
   iterationCount,
+  maxRetries,
   beforeEachCommand,
   beforeAllCommand,
   bundleId,
@@ -58,6 +67,7 @@ const runTest = async ({
 }: {
   duration?: number;
   iterationCount?: number;
+  maxRetries?: number;
   beforeAllCommand?: string;
   beforeEachCommand?: string;
   testCommand: string;
@@ -83,7 +93,8 @@ const runTest = async ({
       },
       duration,
     },
-    iterationCount
+    iterationCount,
+    maxRetries
   );
 
   writeResults({

diff --git a/packages/e2e-performance/src/executeAsync.ts b/packages/e2e-performance/src/executeAsync.ts
@@ -19,7 +19,7 @@ export const executeAsync = (command: string) => {
     ...(isMacOs ? command.split(" ") : ["-e", "-c", command]),
   ]);
 
-  return new Promise((resolve) => {
+  return new Promise((resolve, reject) => {
     child.stdout.on("data", (data: ReadableStream<string>) => {
       console.log(data.toString());
     });
@@ -32,10 +32,12 @@ export const executeAsync = (command: string) => {
 
     child.on("close", (code: number | null) => {
       if (code !== 0) {
-        throw new Error(
+        Logger.error(
           `Error when running "${command}": exited with code ${code}`
         );
-      } else resolve(code);
+        reject(code);
+      }
+      resolve(0);
     });
   });
 };
diff --git a/packages/e2e-performance/src/index.ts b/packages/e2e-performance/src/index.ts
@@ -24,32 +24,67 @@ class PerformanceTester {
   }
 
   private async executeTestCase(): Promise<TestCaseIterationResult> {
-    const { beforeTest, run, afterTest, duration } = this.testCase;
+    try {
+      const { beforeTest, run, afterTest, duration } = this.testCase;
 
-    if (beforeTest) await beforeTest();
+      if (beforeTest) await beforeTest();
 
-    const performanceMeasurer = new PerformanceMeasurer(this.bundleId);
-    // We don't await here to not block the thread
-    // but it's not ideal, we could spawn a worker for the measurer
-    performanceMeasurer.start();
-    await run();
-    const measures = await performanceMeasurer.stop(duration);
+      const performanceMeasurer = new PerformanceMeasurer(this.bundleId);
+      // We don't await here to not block the thread
+      // but it's not ideal, we could spawn a worker for the measurer
+      performanceMeasurer.start();
+      await run();
+      const measures = await performanceMeasurer.stop(duration);
 
-    if (afterTest) await afterTest();
+      if (afterTest) await afterTest();
 
-    return measures;
+      return measures;
+    } catch (error) {
+      throw new Error("Error while running test");
+    }
   }
 
-  async iterate(iterationCount: number): Promise<TestCaseIterationResult[]> {
+  async iterate(
+    iterationCount: number,
+    maxRetries: number
+  ): Promise<TestCaseIterationResult[]> {
+    let retriesCount = 0;
+    let currentIterationIndex = 0;
     const measures: TestCaseIterationResult[] = [];
 
-    for (let i = 0; i < iterationCount; i++) {
-      Logger.info(`Running iteration ${i + 1}/${iterationCount}`);
-      const measure = await this.executeTestCase();
-      Logger.success(
-        `Finished iteration ${i + 1}/${iterationCount} in ${measure.time}ms`
+    while (currentIterationIndex < iterationCount) {
+      Logger.info(
+        `Running iteration ${currentIterationIndex + 1}/${iterationCount}`
       );
-      measures.push(measure);
+      try {
+        const measure = await this.executeTestCase();
+        Logger.success(
+          `Finished iteration ${
+            currentIterationIndex + 1
+          }/${iterationCount} in ${measure.time}ms (${retriesCount} ${
+            retriesCount > 1 ? "retries" : "retry"
+          } so far)`
+        );
+        measures.push(measure);
+        currentIterationIndex++;
+      } catch (error) {
+        Logger.error(
+          `Iteration ${
+            currentIterationIndex + 1
+          }/${iterationCount} failed (ignoring measure): ${
+            error instanceof Error ? error.message : "unknown error"
+          }`
+        );
+
+        retriesCount++;
+        if (retriesCount > maxRetries) {
+          throw new Error("Max number of retries reached.");
+        }
+      }
+    }
+
+    if (measures.length === 0) {
+      throw new Error("No measure returned");
     }
 
     return measures;
@@ -96,10 +131,11 @@ npx @perf-profiler/web-reporter ${filePath}`
 export const measurePerformance = async (
   bundleId: string,
   testCase: TestCase,
-  iterationCount = 10
+  iterationCount = 10,
+  maxRetries = 3
 ) => {
   const tester = new PerformanceTester(bundleId, testCase);
-  const measures = await tester.iterate(iterationCount);
+  const measures = await tester.iterate(iterationCount, maxRetries);
 
   return {
     measures,