Merge pull request #439 from PassiveLogic/fix/fix-benchmark-setup

fix: Benchmark setup fixes and filtering option

Author: kateinoigakukun

Benchmarks/README.md

@@ -27,4 +27,11 @@ node run.js --adaptive --output=stable-results.json
 
 # Run benchmarks and compare with previous results
 node run.js --baseline=previous-results.json
+
+# Run only a subset of benchmarks
+# Substring match
+node run.js --filter=Call
+# Regex (with flags)
+node run.js --filter=/^Property access\//
+node run.js --filter=/string/i
 ```

Benchmarks/Sources/Benchmarks.swift

@@ -14,7 +14,7 @@ class Benchmark {
             body()
             return .undefined
         }
-        benchmarkRunner("\(title)/\(name)", jsBody)
+        try! benchmarkRunner("\(title)/\(name)", jsBody)
     }
 }
 
@@ -24,13 +24,13 @@ class Benchmark {
 
     call.testSuite("JavaScript function call through Wasm import") {
         for _ in 0..<20_000_000 {
-            benchmarkHelperNoop()
+            try! benchmarkHelperNoop()
         }
     }
 
     call.testSuite("JavaScript function call through Wasm import with int") {
         for _ in 0..<10_000_000 {
-            benchmarkHelperNoopWithNumber(42)
+            try! benchmarkHelperNoopWithNumber(42)
         }
     }
 

Benchmarks/run.js

@@ -21,6 +21,32 @@ function updateProgress(current, total, label = '', width) {
     process.stdout.write(`${label} [${bar}] ${current}/${total}`);
 }
 
+/**
+ * Create a name filter function from CLI argument
+ * - Supports substring match (default)
+ * - Supports /regex/flags syntax
+ * @param {string|undefined} arg
+ * @returns {(name: string) => boolean}
+ */
+function createNameFilter(arg) {
+    if (!arg) {
+        return () => true;
+    }
+    if (arg.startsWith('/') && arg.lastIndexOf('/') > 0) {
+        const lastSlash = arg.lastIndexOf('/');
+        const pattern = arg.slice(1, lastSlash);
+        const flags = arg.slice(lastSlash + 1);
+        try {
+            const re = new RegExp(pattern, flags);
+            return (name) => re.test(name);
+        } catch (e) {
+            console.error('Invalid regular expression for --filter:', e.message);
+            process.exit(1);
+        }
+    }
+    return (name) => name.includes(arg);
+}
+
 /**
  * Calculate coefficient of variation (relative standard deviation)
  * @param {Array<number>} values - Array of measurement values
@@ -238,24 +264,28 @@ function saveJsonResults(filePath, data) {
  * @param {Object} results - Results object to store benchmark data
  * @returns {Promise<void>}
  */
-async function singleRun(results) {
+async function singleRun(results, nameFilter) {
     const options = await defaultNodeSetup({})
+    const benchmarkRunner = (name, body) => {
+        if (nameFilter && !nameFilter(name)) {
+            return;
+        }
+        const startTime = performance.now();
+        body();
+        const endTime = performance.now();
+        const duration = endTime - startTime;
+        if (!results[name]) {
+            results[name] = []
+        }
+        results[name].push(duration)
+    }
     const { exports } = await instantiate({
         ...options,
-        imports: {
+        getImports: () => ({
             benchmarkHelperNoop: () => { },
             benchmarkHelperNoopWithNumber: (n) => { },
-            benchmarkRunner: (name, body) => {
-                const startTime = performance.now();
-                body();
-                const endTime = performance.now();
-                const duration = endTime - startTime;
-                if (!results[name]) {
-                    results[name] = []
-                }
-                results[name].push(duration)
-            }
-        }
+            benchmarkRunner: benchmarkRunner
+        })
     });
     exports.run();
 }
@@ -266,7 +296,7 @@ async function singleRun(results) {
  * @param {Object} options - Adaptive sampling options
  * @returns {Promise<void>}
  */
-async function runUntilStable(results, options, width) {
+async function runUntilStable(results, options, width, nameFilter, filterArg) {
     const {
         minRuns = 5,
         maxRuns = 50,
@@ -285,9 +315,14 @@ async function runUntilStable(results, options, width) {
         // Update progress with estimated completion
         updateProgress(runs, maxRuns, "Benchmark Progress:", width);
 
-        await singleRun(results);
+        await singleRun(results, nameFilter);
         runs++;
 
+        if (runs === 1 && Object.keys(results).length === 0) {
+            console.error(`\nNo benchmarks matched filter: ${filterArg}`);
+            process.exit(1);
+        }
+
         // Check if we've reached minimum runs
         if (runs < minRuns) continue;
 
@@ -349,6 +384,7 @@ Options:
   --min-runs=NUMBER     Minimum runs for adaptive sampling (default: 5)
   --max-runs=NUMBER     Maximum runs for adaptive sampling (default: 50)
   --target-cv=NUMBER    Target coefficient of variation % (default: 5)
+  --filter=PATTERN      Filter benchmarks by name (substring or /regex/flags)
   --help                Show this help message
 `);
 }
@@ -363,7 +399,8 @@ async function main() {
             adaptive: { type: 'boolean', default: false },
             'min-runs': { type: 'string', default: '5' },
             'max-runs': { type: 'string', default: '50' },
-            'target-cv': { type: 'string', default: '5' }
+            'target-cv': { type: 'string', default: '5' },
+            filter: { type: 'string' }
         }
     });
 
@@ -374,6 +411,8 @@ async function main() {
 
     const results = {};
     const width = 30;
+    const filterArg = args.values.filter;
+    const nameFilter = createNameFilter(filterArg);
 
     if (args.values.adaptive) {
         // Adaptive sampling mode
@@ -388,7 +427,7 @@ async function main() {
             console.log(`Results will be saved to: ${args.values.output}`);
         }
 
-        await runUntilStable(results, options, width);
+        await runUntilStable(results, options, width, nameFilter, filterArg);
     } else {
         // Fixed number of runs mode
         const runs = parseInt(args.values.runs, 10);
@@ -410,7 +449,12 @@ async function main() {
         console.log("\nOverall Progress:");
         for (let i = 0; i < runs; i++) {
             updateProgress(i, runs, "Benchmark Runs:", width);
-            await singleRun(results);
+            await singleRun(results, nameFilter);
+            if (i === 0 && Object.keys(results).length === 0) {
+                process.stdout.write("\n");
+                console.error(`No benchmarks matched filter: ${filterArg}`);
+                process.exit(1);
+            }
         }
         updateProgress(runs, runs, "Benchmark Runs:", width);
         console.log("\n");