✨ Add properties to XML reporting based on status.

The status of particular test cases is now reported in the
JUnit-formatted XML report. It is given as a `<property>` child element
on the `<testcase>` element.

Refs CAP-2408.

bin-src/init.test.ts

@@ -11,7 +11,6 @@ import {
 
 vi.mock('jsonfile', async (importOriginal) => {
   return {
-    // @ts-expect-error TS does not think actual is an object, but it's fine.
     ...(await importOriginal()),
     writeFile: vi.fn(() => Promise.resolve()),
   };

node-src/index.test.ts

@@ -43,7 +43,7 @@ vi.mock('execa');
 // NOTE: we'd prefer to mock the require.resolve() of `@chromatic-com/playwright/..` but
 // vitest doesn't allow you to do that.
 const mockedBuildCommand = 'mocked build command';
-vi.mock(import('./lib/e2e'), async (importOriginal) => ({
+(vi as any).mock(import('./lib/e2e'), async (importOriginal) => ({
   ...(await importOriginal()),
   getE2EBuildCommand: () => mockedBuildCommand,
 }));

node-src/lib/logSerializers.ts

@@ -14,7 +14,6 @@ function responseSerializer({ status, statusText, headers, url, _raw }: Response
 // all of our environment variables! See https://github.com/chromaui/chromatic/issues/1993
 // Note it is added to both err.envPairs *and* err.options.envPairs :facepalm:
 function stripEnvironmentPairs(err: any) {
-  // @ts-expect-error Ignore the _ property
   const { envPairs, options: { envPairs: _, ...options } = {}, ...sanitizedError } = err;
   return { sanitizedErr: sanitizedError, ...(err.options && { options }) };
 }

node-src/tasks/report.ts

@@ -1,11 +1,28 @@
-import reportBuilder from 'junit-report-builder';
+import reportBuilder, { TestSuite } from 'junit-report-builder';
 import path from 'path';
 
 import { createTask, transitionTo } from '../lib/tasks';
 import { Context } from '../types';
 import wroteReport from '../ui/messages/info/wroteReport';
 import { initial, pending, success } from '../ui/tasks/report';
 
+interface TestMode {
+  name: string;
+}
+
+interface TestSpec {
+  name: string;
+  component: {
+    name: string;
+    displayName: string;
+  };
+}
+
+interface TestParameters {
+  viewport: number;
+  viewportIsDefault: boolean;
+}
+
 const ReportQuery = `
   query ReportQuery($buildNumber: Int!) {
     app {
@@ -69,8 +86,11 @@ interface ReportQueryResult {
   };
 }
 
-// TODO: refactor this function
-// eslint-disable-next-line complexity
+/**
+ * Generate a JUnit report XML file for a particular run.
+ *
+ * @param ctx The {@link Context} for which we're generating a report.
+ */
 export const generateReport = async (ctx: Context) => {
   const { client, log } = ctx;
   const { junitReport } = ctx.options;
@@ -97,7 +117,7 @@ export const generateReport = async (ctx: Context) => {
   );
   const buildTime = (build.completedAt || Date.now()) - build.createdAt;
 
-  const suite = reportBuilder
+  const suite: TestSuite = reportBuilder
     .testSuite()
     .name(`Chromatic build ${build.number}`)
     .time(Math.round(buildTime / 1000))
@@ -108,38 +128,56 @@ export const generateReport = async (ctx: Context) => {
     .property('storybookUrl', build.storybookUrl);
 
   for (const { status, result, spec, parameters, mode } of build.tests) {
-    const testSuffixName = mode.name || `[${parameters.viewport}px]`;
-    const suffix = parameters.viewportIsDefault ? '' : testSuffixName;
-    const testCase = suite
-      .testCase()
-      .className(spec.component.name.replaceAll(/[/|]/g, '.')) // transform story path to class path
-      .name(`${spec.name} ${suffix}`);
-
-    switch (status) {
-      case 'FAILED':
-        testCase.error('Server error while taking snapshot, please try again', status);
-        break;
-      case 'BROKEN':
-        testCase.error('Snapshot is broken due to an error in your Storybook', status);
-        break;
-      case 'DENIED':
-        testCase.failure('Snapshot was denied by a user', status);
-        break;
-      case 'PENDING':
-        testCase.failure('Snapshot contains visual changes and must be reviewed', status);
-        break;
-      default: {
-        if (['SKIPPED', 'PRESERVED'].includes(result)) {
-          testCase.skipped();
-        }
-      }
-    }
+    generateReportTestCase(suite, status, result, spec, parameters, mode);
   }
 
   reportBuilder.writeTo(ctx.reportPath);
   log.info(wroteReport(ctx.reportPath, 'JUnit XML'));
 };
 
+/**
+ * Generate a single `<testcase>` within a JUnit report and test run with Chromatic.
+ *
+ * @param suite The {@link TestSuite} we're currently processing.
+ * @param status The status of the test, as a `string`.
+ * @param result The result of the test, as a `string`.
+ * @param spec The descriptive metadata about the test spec, as a {@link TestSpec}.
+ * @param parameters The parameters passed to run the test, as a {@link TestParameters}.
+ * @param mode The mode of the test run, as a {@link TestMode}.
+ */
+export const generateReportTestCase = (
+  suite: TestSuite,
+  status: string,
+  result: string,
+  spec: TestSpec,
+  parameters: TestParameters,
+  mode: TestMode
+) => {
+  const testSuffixName = mode.name || `[${parameters.viewport}px]`;
+  const suffix = parameters.viewportIsDefault ? '' : testSuffixName;
+  const testCase: any = suite.testCase().className(spec.component.name.replaceAll(/[/|]/g, '.')); // transform story path to class path
+  testCase.property('result', status).name(`${spec.name} ${suffix}`);
+
+  switch (status) {
+    case 'FAILED':
+      testCase.error('Server error while taking snapshot, please try again', status);
+      break;
+    case 'BROKEN':
+      testCase.error('Snapshot is broken due to an error in your Storybook', status);
+      break;
+    case 'DENIED':
+      testCase.failure('Snapshot was denied by a user', status);
+      break;
+    case 'PENDING':
+      testCase.failure('Snapshot contains visual changes and must be reviewed', status);
+      break;
+    default: {
+      if (['SKIPPED', 'PRESERVED'].includes(result)) {
+        testCase.skipped();
+      }
+    }
+  }
+};
 /**
  * Sets up the Listr task for generating a JUnit report.
  *

package.json

@@ -165,7 +165,7 @@
     "husky": "^7.0.0",
     "json5": "^2.2.3",
     "jsonfile": "^6.0.1",
-    "junit-report-builder": "2.1.0",
+    "junit-report-builder": "3.1.0",
     "listr": "0.14.3",
     "listr-update-renderer": "^0.5.0",
     "meow": "^9.0.0",

yarn.lock

@@ -7589,7 +7589,7 @@ __metadata:
     husky: "npm:^7.0.0"
     json5: "npm:^2.2.3"
     jsonfile: "npm:^6.0.1"
-    junit-report-builder: "npm:2.1.0"
+    junit-report-builder: "npm:3.1.0"
     listr: "npm:0.14.3"
     listr-update-renderer: "npm:^0.5.0"
     meow: "npm:^9.0.0"
@@ -8366,10 +8366,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"date-format@npm:0.0.2":
-  version: 0.0.2
-  resolution: "date-format@npm:0.0.2"
-  checksum: 10c0/ef6117bd0ca7b646c022909b15396a8492e8e3ef5bfcd560420faac0a0c45292a13a2f541da56f78dd79035ffb04eeb7a219edfbb6c7b98ac3b091666dc69e55
+"date-format@npm:4.0.3":
+  version: 4.0.3
+  resolution: "date-format@npm:4.0.3"
+  checksum: 10c0/be940a4c0f35875cd9df1b160531e9bbb447c4d8d2817b4b110f79c109018b4a1c0d4676ef756694bcf2e7f103f0893ff53c814896ec42e3b7390bb16b299f6f
   languageName: node
   linkType: hard
 
@@ -12568,15 +12568,15 @@ __metadata:
   languageName: node
   linkType: hard
 
-"junit-report-builder@npm:2.1.0":
-  version: 2.1.0
-  resolution: "junit-report-builder@npm:2.1.0"
+"junit-report-builder@npm:3.1.0":
+  version: 3.1.0
+  resolution: "junit-report-builder@npm:3.1.0"
   dependencies:
-    date-format: "npm:0.0.2"
-    lodash: "npm:^4.17.15"
-    make-dir: "npm:^1.3.0"
-    xmlbuilder: "npm:^10.0.0"
-  checksum: 10c0/f5e761ac74071d6f21303801125117eb63d21571919bc287d4a37aefe6f4f1805950ef304c1b6b7007ec955d4e8ddf3f6182619c8abb4f76a16d2d110423aecf
+    date-format: "npm:4.0.3"
+    lodash: "npm:^4.17.21"
+    make-dir: "npm:^3.1.0"
+    xmlbuilder: "npm:^15.1.1"
+  checksum: 10c0/fbb5f89143dcadc95cd886550bdc6602d3210c43b86841cbd088cc7bb7e4cb2629b2775d966579394d320c5de56221a07254dcef2002963b015dcff44e4ec64d
   languageName: node
   linkType: hard
 
@@ -13183,15 +13183,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"make-dir@npm:^1.3.0":
-  version: 1.3.0
-  resolution: "make-dir@npm:1.3.0"
-  dependencies:
-    pify: "npm:^3.0.0"
-  checksum: 10c0/5eb94f47d7ef41d89d1b8eef6539b8950d5bd99eeba093a942bfd327faa37d2d62227526b88b73633243a2ec7972d21eb0f4e5d62ae4e02a79e389f4a7bb3022
-  languageName: node
-  linkType: hard
-
 "make-dir@npm:^2.0.0, make-dir@npm:^2.1.0":
   version: 2.1.0
   resolution: "make-dir@npm:2.1.0"
@@ -13202,7 +13193,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"make-dir@npm:^3.0.0, make-dir@npm:^3.0.2":
+"make-dir@npm:^3.0.0, make-dir@npm:^3.0.2, make-dir@npm:^3.1.0":
   version: 3.1.0
   resolution: "make-dir@npm:3.1.0"
   dependencies:
@@ -20350,10 +20341,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"xmlbuilder@npm:^10.0.0":
-  version: 10.1.1
-  resolution: "xmlbuilder@npm:10.1.1"
-  checksum: 10c0/26c465e8bd16b4e882d39c2e2a29bb277434d254717aa05df117dd0009041d92855426714b2d1a6a5f76983640349f4edb80073b6ae374e0e6c3d13029ea8237
+"xmlbuilder@npm:^15.1.1":
+  version: 15.1.1
+  resolution: "xmlbuilder@npm:15.1.1"
+  checksum: 10c0/665266a8916498ff8d82b3d46d3993913477a254b98149ff7cff060d9b7cc0db7cf5a3dae99aed92355254a808c0e2e3ec74ad1b04aa1061bdb8dfbea26c18b8
   languageName: node
   linkType: hard