test: compare local benchmark to latest and exit 1

perf/efps/index.ts

@@ -1,7 +1,10 @@
+/* eslint-disable max-depth */
 /* eslint-disable no-console */
 // eslint-disable-next-line import/no-unassigned-import
 import 'dotenv/config'
 
+import fs from 'node:fs'
+import os from 'node:os'
 import path from 'node:path'
 import process from 'node:process'
 import {fileURLToPath} from 'node:url'
@@ -11,13 +14,13 @@ import chalk from 'chalk'
 import Table from 'cli-table3'
 import Ora from 'ora'
 
-// eslint-disable-next-line import/no-extraneous-dependencies
 import {exec} from './helpers/exec'
 import {runTest} from './runTest'
 import article from './tests/article/article'
 import recipe from './tests/recipe/recipe'
 import singleString from './tests/singleString/singleString'
 import synthetic from './tests/synthetic/synthetic'
+import {type EfpsResult} from './types'
 
 const headless = true
 const tests = [singleString, recipe, article, synthetic]
@@ -63,29 +66,87 @@ await exec({
   cwd: monorepoRoot,
 })
 
+// Prepare the latest version of the 'sanity' package
+const tmpDir = path.join(os.tmpdir(), `sanity-latest-${Date.now()}`)
+await fs.promises.mkdir(tmpDir, {recursive: true})
+spinner.start('')
+await exec({
+  command: 'npm install sanity@latest --no-save',
+  cwd: tmpDir,
+  spinner,
+  text: ['Downloading latest sanity package…', 'Downloaded latest sanity package'],
+})
+const sanityPackagePath = path.join(tmpDir, 'node_modules', 'sanity')
+
 await exec({
   text: ['Ensuring playwright is installed…', 'Playwright is installed'],
   command: 'npx playwright install',
   spinner,
 })
 
 const table = new Table({
-  head: [chalk.bold('benchmark'), 'eFPS p50', 'eFPS p75', 'eFPS p90'].map((cell) =>
-    chalk.cyan(cell),
+  head: [chalk.bold('benchmark'), 'Passed?', 'p50 eFPS (Δ%)', 'p75 eFPS (Δ%)', 'p90 eFPS (Δ%)'].map(
+    (cell) => chalk.cyan(cell),
   ),
 })
 
+const markdownRows: string[] = []
+
 const formatFps = (fps: number) => {
   const rounded = fps.toFixed(1)
   if (fps >= 60) return chalk.green(rounded)
   if (fps < 20) return chalk.red(rounded)
   return chalk.yellow(rounded)
 }
 
+const formatPercentage = (value: number): string => {
+  const rounded = value.toFixed(1)
+  const sign = value >= 0 ? '+' : ''
+  if (value > -50) return `${sign}${rounded}%`
+  return chalk.red(`${sign}${rounded}%`)
+}
+
+// For markdown formatting without colors
+const formatFpsPlain = (fps: number) => {
+  const rounded = fps.toFixed(1)
+  return rounded
+}
+
+const formatPercentagePlain = (value: number): string => {
+  const rounded = value.toFixed(1)
+  const sign = value >= 0 ? '+' : ''
+  return `${sign}${rounded}%`
+}
+
+function getStatus(p50Diff: number, p75Diff: number, p90Diff: number): 'error' | 'passed' {
+  if (p50Diff < -50 || p75Diff < -50 || p90Diff < -50) {
+    return 'error'
+  }
+  return 'passed'
+}
+
+function getStatusEmoji(status: 'error' | 'passed'): string {
+  if (status === 'error') return '🔴'
+  return '✅'
+}
+
+// Initialize the overall status
+let overallStatus: 'error' | 'passed' = 'passed'
+
+interface TestResult {
+  testName: string
+  version: 'local' | 'latest'
+  results: EfpsResult[]
+}
+
+const allResults: TestResult[] = []
+
 for (let i = 0; i < tests.length; i++) {
   const test = tests[i]
-  const results = await runTest({
-    prefix: `Running '${test.name}' [${i + 1}/${tests.length}]…`,
+
+  // Run with local 'sanity' package
+  const localResults = await runTest({
+    prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with local 'sanity'…`,
     test,
     resultsDir,
     spinner,
@@ -94,14 +155,86 @@ for (let i = 0; i < tests.length; i++) {
     projectId,
   })
 
-  for (const result of results) {
-    table.push({
-      [[chalk.bold(test.name), result.label ? `(${result.label})` : ''].join(' ')]: [
-        formatFps(result.p50),
-        formatFps(result.p75),
-        formatFps(result.p90),
-      ],
-    })
+  allResults.push({
+    testName: test.name,
+    version: 'local',
+    results: localResults,
+  })
+
+  // Run with latest 'sanity' package
+  const latestResults = await runTest({
+    prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with 'sanity@latest'…`,
+    test,
+    resultsDir,
+    spinner,
+    client,
+    headless,
+    projectId,
+    sanityPackagePath,
+  })
+
+  allResults.push({
+    testName: test.name,
+    version: 'latest',
+    results: latestResults,
+  })
+}
+
+for (const test of tests) {
+  const localResult = allResults.find((r) => r.testName === test.name && r.version === 'local')
+  const latestResult = allResults.find((r) => r.testName === test.name && r.version === 'latest')
+
+  if (localResult && latestResult) {
+    const localResultsMap = new Map<string | undefined, EfpsResult>()
+    for (const res of localResult.results) {
+      localResultsMap.set(res.label, res)
+    }
+    const latestResultsMap = new Map<string | undefined, EfpsResult>()
+    for (const res of latestResult.results) {
+      latestResultsMap.set(res.label, res)
+    }
+
+    for (const [label, latest] of latestResultsMap) {
+      const local = localResultsMap.get(label)
+      if (local) {
+        // Compute percentage differences
+        const p50Diff = ((local.p50 - latest.p50) / latest.p50) * 100
+        const p75Diff = ((local.p75 - latest.p75) / latest.p75) * 100
+        const p90Diff = ((local.p90 - latest.p90) / latest.p90) * 100
+
+        // Determine test status
+        const testStatus = getStatus(p50Diff, p75Diff, p90Diff)
+
+        // Update overall status
+        if (testStatus === 'error') {
+          overallStatus = 'error'
+        }
+
+        const rowLabel = [chalk.bold(test.name), label ? `(${label})` : ''].join(' ')
+
+        table.push([
+          rowLabel,
+          getStatusEmoji(testStatus),
+          `${formatFps(local.p50)} (${formatPercentage(p50Diff)})`,
+          `${formatFps(local.p75)} (${formatPercentage(p75Diff)})`,
+          `${formatFps(local.p90)} (${formatPercentage(p90Diff)})`,
+        ])
+
+        // Add to markdown rows
+        const markdownRow = [
+          [test.name, label ? `(${label})` : ''].join(' '),
+          getStatusEmoji(testStatus),
+          `${formatFpsPlain(local.p50)} (${formatPercentagePlain(p50Diff)})`,
+          `${formatFpsPlain(local.p75)} (${formatPercentagePlain(p75Diff)})`,
+          `${formatFpsPlain(local.p90)} (${formatPercentagePlain(p90Diff)})`,
+        ]
+        markdownRows.push(`| ${markdownRow.join(' | ')} |`)
+      } else {
+        spinner.fail(`Missing local result for test '${test.name}', label '${label}'`)
+      }
+    }
+  } else {
+    spinner.fail(`Missing results for test '${test.name}'`)
   }
 }
 
@@ -113,3 +246,35 @@ console.log(`
 │ The number of renders ("frames") that is assumed to be possible
 │ within a second. Derived from input latency. ${chalk.green('Higher')} is better.
 `)
+
+// Map overallStatus to status text
+const statusText = overallStatus === 'error' ? 'Error' : 'Passed'
+const statusEmoji = getStatusEmoji(overallStatus)
+
+// Build the markdown content
+const markdownContent = [
+  '# Benchmark Results',
+  '',
+  `<details>`,
+  `<summary>${statusEmoji} Performance Benchmark Results — Status: **${statusText}** </summary>`,
+  '',
+  '| Benchmark | Passed? | p50 eFPS (Δ%) | p75 eFPS (Δ%) | p90 eFPS (Δ%) |',
+  '|-----------|---------|---------------|---------------|---------------|',
+  ...markdownRows,
+  '</details>',
+  '',
+  '> **eFPS — editor "Frames Per Second"**',
+  '> ',
+  '> The number of renders ("frames") that is assumed to be possible within a second. Derived from input latency. **Higher** is better.',
+  '',
+].join('\n')
+
+// Write markdown file to root of results
+const markdownOutputPath = path.join(workspaceDir, 'results', 'benchmark-results.md')
+await fs.promises.writeFile(markdownOutputPath, markdownContent)
+
+// Exit with code 1 if regression detected
+if (overallStatus === 'error') {
+  console.error(chalk.red('Performance regression detected exceeding 50% threshold.'))
+  process.exit(1)
+}

perf/efps/runTest.ts

@@ -24,6 +24,7 @@ interface RunTestOptions {
   projectId: string
   headless: boolean
   client: SanityClient
+  sanityPackagePath?: string // Add this line
 }
 
 export async function runTest({
@@ -34,26 +35,31 @@ export async function runTest({
   projectId,
   headless,
   client,
+  sanityPackagePath,
 }: RunTestOptions): Promise<EfpsResult[]> {
   const log = (text: string) => {
     spinner.text = `${prefix}\n  └ ${text}`
   }
 
   spinner.start(prefix)
 
-  const outDir = path.join(workspaceDir, 'builds', test.name)
-  const testResultsDir = path.join(resultsDir, test.name)
+  const versionLabel = sanityPackagePath ? 'latest' : 'local'
+  const outDir = path.join(workspaceDir, 'builds', test.name, versionLabel)
+  const testResultsDir = path.join(resultsDir, test.name, versionLabel)
 
   await fs.promises.mkdir(outDir, {recursive: true})
   log('Building…')
 
+  const alias: Record<string, string> = {'#config': fileURLToPath(test.configPath!)}
+  if (sanityPackagePath) {
+    alias.sanity = sanityPackagePath
+  }
+
   await vite.build({
     appType: 'spa',
     build: {outDir, sourcemap: true},
     plugins: [{...sourcemaps(), enforce: 'pre'}, react()],
-    resolve: {
-      alias: {'#config': fileURLToPath(test.configPath!)},
-    },
+    resolve: {alias},
     logLevel: 'silent',
   })
 
@@ -107,9 +113,9 @@ export async function runTest({
 
     log('Loading editor…')
     await page.goto(
-      `http://localhost:3300/intent/edit/id=${encodeURIComponent(document._id)};type=${encodeURIComponent(
-        documentToCreate._type,
-      )}`,
+      `http://localhost:3300/intent/edit/id=${encodeURIComponent(
+        document._id,
+      )};type=${encodeURIComponent(documentToCreate._type)}`,
     )
 
     await cdp.send('Profiler.enable')
@@ -138,7 +144,7 @@ export async function runTest({
       JSON.stringify(remappedProfile),
     )
 
-    spinner.succeed(`Ran benchmark '${test.name}'`)
+    spinner.succeed(`Ran benchmark '${test.name}' (${versionLabel})`)
 
     return results
   } finally {