test: compare local benchmark to latest and exit 1

perf/efps/index.ts

@@ -2,6 +2,8 @@
 // eslint-disable-next-line import/no-unassigned-import
 import 'dotenv/config'
 
+import fs from 'node:fs'
+import os from 'node:os'
 import path from 'node:path'
 import process from 'node:process'
 import {fileURLToPath} from 'node:url'
@@ -11,13 +13,13 @@ import chalk from 'chalk'
 import Table from 'cli-table3'
 import Ora from 'ora'
 
-// eslint-disable-next-line import/no-extraneous-dependencies
 import {exec} from './helpers/exec'
 import {runTest} from './runTest'
 import article from './tests/article/article'
 import recipe from './tests/recipe/recipe'
 import singleString from './tests/singleString/singleString'
 import synthetic from './tests/synthetic/synthetic'
+import {type EfpsResult} from './types'
 
 const headless = true
 const tests = [singleString, recipe, article, synthetic]
@@ -63,16 +65,34 @@ await exec({
   cwd: monorepoRoot,
 })
 
+// Prepare the latest version of the 'sanity' package
+const tmpDir = path.join(os.tmpdir(), `sanity-latest-${Date.now()}`)
+await fs.promises.mkdir(tmpDir, {recursive: true})
+spinner.start('')
+await exec({
+  command: 'npm install sanity@latest --no-save',
+  cwd: tmpDir,
+  spinner,
+  text: ['Downloading latest sanity package…', 'Downloaded latest sanity package'],
+})
+const sanityPackagePath = path.join(tmpDir, 'node_modules', 'sanity')
+
 await exec({
   text: ['Ensuring playwright is installed…', 'Playwright is installed'],
   command: 'npx playwright install',
   spinner,
 })
 
 const table = new Table({
-  head: [chalk.bold('benchmark'), 'eFPS p50', 'eFPS p75', 'eFPS p90'].map((cell) =>
-    chalk.cyan(cell),
-  ),
+  head: [
+    chalk.bold('benchmark'),
+    'latest p50',
+    'local p50 (Δ%)',
+    'latest p75',
+    'local p75 (Δ%)',
+    'latest p90',
+    'local p90 (Δ%)',
+  ].map((cell) => chalk.cyan(cell)),
 })
 
 const formatFps = (fps: number) => {
@@ -82,26 +102,108 @@ const formatFps = (fps: number) => {
   return chalk.yellow(rounded)
 }
 
+const formatPercentage = (value: number): string => {
+  const rounded = value.toFixed(1)
+  const sign = value >= 0 ? '+' : ''
+  if (value >= 0) return chalk.green(`${sign}${rounded}%`)
+  return chalk.red(`${rounded}%`)
+}
+
+// Initialize the regression flag
+let hasSignificantRegression = false
+
+interface TestResult {
+  testName: string
+  version: 'local' | 'latest'
+  results: EfpsResult[]
+}
+
+const allResults: TestResult[] = []
+
 for (let i = 0; i < tests.length; i++) {
   const test = tests[i]
-  const results = await runTest({
-    prefix: `Running '${test.name}' [${i + 1}/${tests.length}]…`,
+
+  // Run with local 'sanity' package
+  const localResults = await runTest({
+    prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with local 'sanity'…`,
+    test,
+    resultsDir,
+    spinner,
+    client,
+    headless,
+    projectId,
+  })
+
+  allResults.push({
+    testName: test.name,
+    version: 'local',
+    results: localResults,
+  })
+
+  // Run with latest 'sanity' package
+  const latestResults = await runTest({
+    prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with 'sanity@latest'…`,
     test,
     resultsDir,
     spinner,
     client,
     headless,
     projectId,
+    sanityPackagePath,
+  })
+
+  allResults.push({
+    testName: test.name,
+    version: 'latest',
+    results: latestResults,
   })
+}
 
-  for (const result of results) {
-    table.push({
-      [[chalk.bold(test.name), result.label ? `(${result.label})` : ''].join(' ')]: [
-        formatFps(result.p50),
-        formatFps(result.p75),
-        formatFps(result.p90),
-      ],
-    })
+for (const test of tests) {
+  const localResult = allResults.find((r) => r.testName === test.name && r.version === 'local')
+  const latestResult = allResults.find((r) => r.testName === test.name && r.version === 'latest')
+
+  if (localResult && latestResult) {
+    const localResultsMap = new Map<string | undefined, EfpsResult>()
+    for (const res of localResult.results) {
+      localResultsMap.set(res.label, res)
+    }
+    const latestResultsMap = new Map<string | undefined, EfpsResult>()
+    for (const res of latestResult.results) {
+      latestResultsMap.set(res.label, res)
+    }
+
+    for (const [label, latest] of latestResultsMap) {
+      const local = localResultsMap.get(label)
+      if (local) {
+        // Compute percentage differences
+        const p50Diff = ((local.p50 - latest.p50) / latest.p50) * 100
+        const p75Diff = ((local.p75 - latest.p75) / latest.p75) * 100
+        const p90Diff = ((local.p90 - latest.p90) / latest.p90) * 100
+
+        // Check for significant regression
+        // eslint-disable-next-line max-depth
+        if (p50Diff < -50 || p75Diff < -50 || p90Diff < -50) {
+          hasSignificantRegression = true
+        }
+
+        const rowLabel = [chalk.bold(test.name), label ? `(${label})` : ''].join(' ')
+
+        table.push([
+          rowLabel,
+          formatFps(latest.p50),
+          `${formatFps(local.p50)} (${formatPercentage(p50Diff)})`,
+          formatFps(latest.p75),
+          `${formatFps(local.p75)} (${formatPercentage(p75Diff)})`,
+          formatFps(latest.p90),
+          `${formatFps(local.p90)} (${formatPercentage(p90Diff)})`,
+        ])
+      } else {
+        spinner.fail(`Missing local result for test '${test.name}', label '${label}'`)
+      }
+    }
+  } else {
+    spinner.fail(`Missing results for test '${test.name}'`)
   }
 }
 
@@ -113,3 +215,9 @@ console.log(`
 │ The number of renders ("frames") that is assumed to be possible
 │ within a second. Derived from input latency. ${chalk.green('Higher')} is better.
 `)
+
+// Exit with code 1 if regression detected
+if (hasSignificantRegression) {
+  console.error(chalk.red('Performance regression detected exceeding 50% threshold.'))
+  process.exit(1)
+}

perf/efps/runTest.ts

@@ -24,6 +24,7 @@ interface RunTestOptions {
   projectId: string
   headless: boolean
   client: SanityClient
+  sanityPackagePath?: string // Add this line
 }
 
 export async function runTest({
@@ -34,26 +35,31 @@ export async function runTest({
   projectId,
   headless,
   client,
+  sanityPackagePath,
 }: RunTestOptions): Promise<EfpsResult[]> {
   const log = (text: string) => {
     spinner.text = `${prefix}\n  └ ${text}`
   }
 
   spinner.start(prefix)
 
-  const outDir = path.join(workspaceDir, 'builds', test.name)
-  const testResultsDir = path.join(resultsDir, test.name)
+  const versionLabel = sanityPackagePath ? 'latest' : 'local'
+  const outDir = path.join(workspaceDir, 'builds', test.name, versionLabel)
+  const testResultsDir = path.join(resultsDir, test.name, versionLabel)
 
   await fs.promises.mkdir(outDir, {recursive: true})
   log('Building…')
 
+  const alias: Record<string, string> = {'#config': fileURLToPath(test.configPath!)}
+  if (sanityPackagePath) {
+    alias.sanity = sanityPackagePath
+  }
+
   await vite.build({
     appType: 'spa',
     build: {outDir, sourcemap: true},
     plugins: [{...sourcemaps(), enforce: 'pre'}, react()],
-    resolve: {
-      alias: {'#config': fileURLToPath(test.configPath!)},
-    },
+    resolve: {alias},
     logLevel: 'silent',
   })
 
@@ -107,9 +113,9 @@ export async function runTest({
 
     log('Loading editor…')
     await page.goto(
-      `http://localhost:3300/intent/edit/id=${encodeURIComponent(document._id)};type=${encodeURIComponent(
-        documentToCreate._type,
-      )}`,
+      `http://localhost:3300/intent/edit/id=${encodeURIComponent(
+        document._id,
+      )};type=${encodeURIComponent(documentToCreate._type)}`,
     )
 
     await cdp.send('Profiler.enable')
@@ -138,7 +144,7 @@ export async function runTest({
       JSON.stringify(remappedProfile),
     )
 
-    spinner.succeed(`Ran benchmark '${test.name}'`)
+    spinner.succeed(`Ran benchmark '${test.name}' (${versionLabel})`)
 
     return results
   } finally {