diff --git a/.github/workflows/efps.yml b/.github/workflows/efps.yml new file mode 100644 index 00000000000..dbe3c389a63 --- /dev/null +++ b/.github/workflows/efps.yml @@ -0,0 +1,81 @@ +name: eFPS Test +on: + pull_request: +jobs: + install: + timeout-minutes: 30 + runs-on: ubuntu-latest + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - uses: pnpm/action-setup@v4 + name: Install pnpm + id: pnpm-install + with: + run_install: false + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Cache node modules + id: cache-node-modules + uses: actions/cache@v4 + env: + cache-name: cache-node-modules + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}- + v1-${{ runner.os }}-pnpm-store- + v1-${{ runner.os }}- + + - name: Install project dependencies + run: pnpm install + + - name: Store Playwright's Version + run: | + PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //') + echo "Playwright's Version: $PLAYWRIGHT_VERSION" + echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV + + - name: Cache Playwright Browsers for Playwright's Version + id: cache-playwright-browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }} + + - name: Install Playwright Browsers + if: steps.cache-playwright-browsers.outputs.cache-hit != 'true' + run: npx playwright install --with-deps + + - name: Run eFPS tests + env: + VITE_PERF_EFPS_PROJECT_ID: ${{ secrets.PERF_EFPS_PROJECT_ID }} + VITE_PERF_EFPS_DATASET: ${{ secrets.PERF_EFPS_DATASET }} + PERF_EFPS_SANITY_TOKEN: ${{ secrets.PERF_EFPS_SANITY_TOKEN }} + run: pnpm efps:test + + - name: PR comment with report + uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2 + if: ${{ github.event_name == 'pull_request' }} + with: + comment_tag: "efps-report" + filePath: ${{ github.workspace }}/perf/efps/results/benchmark-results.md + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: efps-report + path: perf/efps/results + retention-days: 30 diff --git a/package.json b/package.json index 923d3610000..4409c9635de 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "e2e:setup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/setup", "e2e:start": "pnpm --filter studio-e2e-testing preview", "etl": "node -r dotenv-flow/config -r esbuild-register scripts/etl", + "efps:test": "cd perf/efps && pnpm test", "example:blog-studio": "cd examples/blog-studio && pnpm start", "example:clean-studio": "cd examples/blog-studio && pnpm start", "example:ecommerce-studio": "cd examples/blog-studio && pnpm start", diff --git a/perf/efps/index.ts b/perf/efps/index.ts index c7cb95e7703..35f35aee4ce 100644 --- a/perf/efps/index.ts +++ b/perf/efps/index.ts @@ -1,7 +1,10 @@ +/* eslint-disable max-depth */ /* eslint-disable no-console */ // eslint-disable-next-line import/no-unassigned-import import 'dotenv/config' +import fs from 'node:fs' +import os from 'node:os' import path from 'node:path' import process from 'node:process' import {fileURLToPath} from 'node:url' @@ -11,13 +14,13 @@ import chalk from 'chalk' import Table from 'cli-table3' import Ora from 'ora' -// eslint-disable-next-line import/no-extraneous-dependencies import {exec} from './helpers/exec' import {runTest} from './runTest' import article from './tests/article/article' import recipe from './tests/recipe/recipe' import singleString from './tests/singleString/singleString' import synthetic from './tests/synthetic/synthetic' +import {type EfpsResult} from './types' const headless = true const tests = [singleString, recipe, article, synthetic] @@ -63,6 +66,18 @@ await exec({ cwd: monorepoRoot, }) +// Prepare the latest version of the 'sanity' package +const tmpDir = path.join(os.tmpdir(), `sanity-latest-${Date.now()}`) +await fs.promises.mkdir(tmpDir, {recursive: true}) +spinner.start('') +await exec({ + command: 'npm install sanity@latest --no-save', + cwd: tmpDir, + spinner, + text: ['Downloading latest sanity package…', 'Downloaded latest sanity package'], +}) +const sanityPackagePath = path.join(tmpDir, 'node_modules', 'sanity') + await exec({ text: ['Ensuring playwright is installed…', 'Playwright is installed'], command: 'npx playwright install', @@ -70,11 +85,13 @@ await exec({ }) const table = new Table({ - head: [chalk.bold('benchmark'), 'eFPS p50', 'eFPS p75', 'eFPS p90'].map((cell) => - chalk.cyan(cell), + head: [chalk.bold('benchmark'), 'Passed?', 'p50 eFPS (Δ%)', 'p75 eFPS (Δ%)', 'p90 eFPS (Δ%)'].map( + (cell) => chalk.cyan(cell), ), }) +const markdownRows: string[] = [] + const formatFps = (fps: number) => { const rounded = fps.toFixed(1) if (fps >= 60) return chalk.green(rounded) @@ -82,10 +99,62 @@ const formatFps = (fps: number) => { return chalk.yellow(rounded) } +const formatPercentage = (value: number): string => { + const rounded = value.toFixed(1) + const sign = value >= 0 ? '+' : '' + if (value > -50) return `${sign}${rounded}%` + return chalk.red(`${sign}${rounded}%`) +} + +// For markdown formatting without colors +const formatFpsPlain = (fps: number) => { + const rounded = fps.toFixed(1) + return rounded +} + +const formatPercentagePlain = (value: number): string => { + const rounded = value.toFixed(1) + const sign = value >= 0 ? '+' : '' + const emoji = value >= 0 ? '🟢' : '🔴' + return `${sign}${rounded}% ${emoji}` +} + +function getStatus( + p50Diff: number, + p75Diff: number, + p90Diff: number, +): 'error' | 'warning' | 'passed' { + if (p50Diff < -50 || p75Diff < -50 || p90Diff < -50) { + return 'error' + } else if (p50Diff < -20 || p75Diff < -20 || p90Diff < -20) { + return 'warning' + } + return 'passed' +} + +function getStatusEmoji(status: 'error' | 'warning' | 'passed'): string { + if (status === 'error') return '🔴' + if (status === 'warning') return '⚠️' + return '✅' +} + +// Initialize the overall status +let overallStatus: 'error' | 'warning' | 'passed' = 'passed' + +interface TestResult { + testName: string + version: 'local' | 'latest' + results: EfpsResult[] +} + +const allResults: TestResult[] = [] + for (let i = 0; i < tests.length; i++) { const test = tests[i] - const results = await runTest({ - prefix: `Running '${test.name}' [${i + 1}/${tests.length}]…`, + + // Run with local 'sanity' package + const localResults = await runTest({ + prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with local 'sanity'…`, test, resultsDir, spinner, @@ -94,14 +163,88 @@ for (let i = 0; i < tests.length; i++) { projectId, }) - for (const result of results) { - table.push({ - [[chalk.bold(test.name), result.label ? `(${result.label})` : ''].join(' ')]: [ - formatFps(result.p50), - formatFps(result.p75), - formatFps(result.p90), - ], - }) + allResults.push({ + testName: test.name, + version: 'local', + results: localResults, + }) + + // Run with latest 'sanity' package + const latestResults = await runTest({ + prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with 'sanity@latest'…`, + test, + resultsDir, + spinner, + client, + headless, + projectId, + sanityPackagePath, + }) + + allResults.push({ + testName: test.name, + version: 'latest', + results: latestResults, + }) +} + +for (const test of tests) { + const localResult = allResults.find((r) => r.testName === test.name && r.version === 'local') + const latestResult = allResults.find((r) => r.testName === test.name && r.version === 'latest') + + if (localResult && latestResult) { + const localResultsMap = new Map() + for (const res of localResult.results) { + localResultsMap.set(res.label, res) + } + const latestResultsMap = new Map() + for (const res of latestResult.results) { + latestResultsMap.set(res.label, res) + } + + for (const [label, latest] of latestResultsMap) { + const local = localResultsMap.get(label) + if (local) { + // Compute percentage differences + const p50Diff = ((local.p50 - latest.p50) / latest.p50) * 100 + const p75Diff = ((local.p75 - latest.p75) / latest.p75) * 100 + const p90Diff = ((local.p90 - latest.p90) / latest.p90) * 100 + + // Determine test status + const testStatus = getStatus(p50Diff, p75Diff, p90Diff) + + // Update overall status + if (testStatus === 'error') { + overallStatus = 'error' + } else if (testStatus === 'warning' && overallStatus === 'passed') { + overallStatus = 'warning' + } + + const rowLabel = [chalk.bold(test.name), label ? `(${label})` : ''].join(' ') + + table.push([ + rowLabel, + getStatusEmoji(testStatus), + `${formatFps(local.p50)} (${formatPercentage(p50Diff)})`, + `${formatFps(local.p75)} (${formatPercentage(p75Diff)})`, + `${formatFps(local.p90)} (${formatPercentage(p90Diff)})`, + ]) + + // Add to markdown rows + const markdownRow = [ + [test.name, label ? `(${label})` : ''].join(' '), + getStatusEmoji(testStatus), + `${formatFpsPlain(local.p50)} (${formatPercentagePlain(p50Diff)})`, + `${formatFpsPlain(local.p75)} (${formatPercentagePlain(p75Diff)})`, + `${formatFpsPlain(local.p90)} (${formatPercentagePlain(p90Diff)})`, + ] + markdownRows.push(`| ${markdownRow.join(' | ')} |`) + } else { + spinner.fail(`Missing local result for test '${test.name}', label '${label}'`) + } + } + } else { + spinner.fail(`Missing results for test '${test.name}'`) } } @@ -113,3 +256,37 @@ console.log(` │ The number of renders ("frames") that is assumed to be possible │ within a second. Derived from input latency. ${chalk.green('Higher')} is better. `) + +// Map overallStatus to status text +const statusText = + // eslint-disable-next-line no-nested-ternary + overallStatus === 'error' ? 'Error' : overallStatus === 'warning' ? 'Warning' : 'Passed' +const statusEmoji = getStatusEmoji(overallStatus) + +// Build the markdown content +const markdownContent = [ + '# Benchmark Results', + '', + `
`, + `${statusEmoji} Performance Benchmark Results — Status: **${statusText}** `, + '', + '| Benchmark | Passed? | p50 eFPS (Δ%) | p75 eFPS (Δ%) | p90 eFPS (Δ%) |', + '|-----------|---------|---------------|---------------|---------------|', + ...markdownRows, + '
', + '', + '> **eFPS — editor "Frames Per Second"**', + '> ', + '> The number of renders ("frames") that is assumed to be possible within a second. Derived from input latency. **Higher** is better.', + '', +].join('\n') + +// Write markdown file to root of results +const markdownOutputPath = path.join(workspaceDir, 'results', 'benchmark-results.md') +await fs.promises.writeFile(markdownOutputPath, markdownContent) + +// Exit with code 1 if regression detected +if (overallStatus === 'error') { + console.error(chalk.red('Performance regression detected exceeding 50% threshold.')) + process.exit(1) +} diff --git a/perf/efps/runTest.ts b/perf/efps/runTest.ts index 201487686df..0bf11f3b826 100644 --- a/perf/efps/runTest.ts +++ b/perf/efps/runTest.ts @@ -24,6 +24,7 @@ interface RunTestOptions { projectId: string headless: boolean client: SanityClient + sanityPackagePath?: string // Add this line } export async function runTest({ @@ -34,6 +35,7 @@ export async function runTest({ projectId, headless, client, + sanityPackagePath, }: RunTestOptions): Promise { const log = (text: string) => { spinner.text = `${prefix}\n └ ${text}` @@ -41,19 +43,23 @@ export async function runTest({ spinner.start(prefix) - const outDir = path.join(workspaceDir, 'builds', test.name) - const testResultsDir = path.join(resultsDir, test.name) + const versionLabel = sanityPackagePath ? 'latest' : 'local' + const outDir = path.join(workspaceDir, 'builds', test.name, versionLabel) + const testResultsDir = path.join(resultsDir, test.name, versionLabel) await fs.promises.mkdir(outDir, {recursive: true}) log('Building…') + const alias: Record = {'#config': fileURLToPath(test.configPath!)} + if (sanityPackagePath) { + alias.sanity = sanityPackagePath + } + await vite.build({ appType: 'spa', build: {outDir, sourcemap: true}, plugins: [{...sourcemaps(), enforce: 'pre'}, react()], - resolve: { - alias: {'#config': fileURLToPath(test.configPath!)}, - }, + resolve: {alias}, logLevel: 'silent', }) @@ -107,9 +113,9 @@ export async function runTest({ log('Loading editor…') await page.goto( - `http://localhost:3300/intent/edit/id=${encodeURIComponent(document._id)};type=${encodeURIComponent( - documentToCreate._type, - )}`, + `http://localhost:3300/intent/edit/id=${encodeURIComponent( + document._id, + )};type=${encodeURIComponent(documentToCreate._type)}`, ) await cdp.send('Profiler.enable') @@ -138,7 +144,7 @@ export async function runTest({ JSON.stringify(remappedProfile), ) - spinner.succeed(`Ran benchmark '${test.name}'`) + spinner.succeed(`Ran benchmark '${test.name}' (${versionLabel})`) return results } finally {