diff --git a/bench/README.md b/bench/README.md index a241f60..5fc3c8d 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,131 +1,283 @@ # Bupkis Performance Benchmarks -This directory contains comprehensive performance benchmarks for the Bupkis assertion library, providing detailed performance analysis and monitoring capabilities using [tinybench](https://npm.im/tinybench). +This directory contains comprehensive performance benchmarks for the Bupkis assertion library using [modestbench](https://github.com/boneskull/modestbench), a modern TypeScript-first benchmarking framework. ## Overview -The benchmark system monitors performance across multiple dimensions: +These benchmarks test all assertion implementations in Bupkis, programmatically generated to ensure comprehensive coverage as assertions are added or modified. This implementation serves to "dogfood" modestbench with a real-world project. -1. **Traditional Categorical Benchmarks** - Group assertions by functionality (type, collection, comparison, pattern) -2. **Implementation Class Benchmarks** - Group assertions by their implementation classes for performance comparison -3. **Comprehensive Analysis** - Overall performance overview with implementation distribution +## Benchmark Suites + +### 1. Sync Schema Assertions (`sync-schema.bench.js`) + +**Tests**: 44 pure schema-based sync assertions +**Tags**: `sync`, `schema` +**Description**: Assertions that use Zod schemas for validation (not function-based). + +These are pure schema-based assertions without callback functions. Generally faster than function-based equivalents due to optimized schema execution. + +**Examples**: + +- Type checking (`'to be a string'`, `'to be a number'`) +- Array/object validation (`'to be empty'`, `'to be non-empty'`) +- Built-in type checking (`'to be a Date'`, `'to be a RegExp'`) + +**Run**: + +```bash +npm run bench:schema +``` + +--- + +### 2. Sync Function Pure Assertions (`sync-function-pure.bench.js`) + +**Tests**: 7 pure function-based sync assertions +**Tags**: `sync`, `function`, `pure` +**Description**: Assertions that return `boolean` or `AssertionFailure` objects directly. + +These are typically the fastest assertion type due to minimal overhead. -## Files Overview +**Examples**: -- **`index.ts`** - Main benchmark suite with comprehensive assertion tests -- **`config.ts`** - Configuration utilities, test data generators, and performance thresholds -- **`suites.ts`** - Traditional categorical benchmark suites organized by assertion functionality -- **`comprehensive-suites.ts`** - **NEW** Implementation class-based benchmarks for ALL assertions -- **`runner.ts`** - CLI runner for executing different benchmark suites and modes +- Set operations (`'to have union'`, `'to have intersection'`) +- Function error validation -## Available Scripts +**Run**: + +```bash +npm run bench:pure +``` + +--- + +### 3. Sync Function Schema Assertions (`sync-function-schema.bench.js`) + +**Tests**: 59 schema-returning function-based sync assertions +**Tags**: `sync`, `function`, `schema-returning` +**Description**: Assertions that use callback functions returning Zod schemas or `AssertionParseRequest` objects. + +More complex than pure functions but still function-based implementations. + +**Examples**: + +- Collection operations +- Type checking assertions +- Comparison assertions + +**Run**: + +```bash +npm run bench:function-schema +``` + +--- + +### 4. Async Function Assertions (`async-function.bench.js`) + +**Tests**: 8 async function-based assertions +**Tags**: `async`, `function` +**Description**: Promise-based assertions with callback functions for validation. + +Includes reject/resolve patterns with parameter validation. + +**Examples**: + +- `'to reject'` +- `'to reject with'` +- `'to resolve'` +- `'to resolve to'` + +**Run**: + +```bash +npm run bench:async +``` + +--- + +### 5. ValueToSchema Utility (`value-to-schema.bench.js`) + +**Tests**: 8 benchmarks (4 categories Γ— 2 option sets) +**Tags**: `utility`, `value-to-schema` +**Description**: Performance tests for the `valueToSchema()` utility function. + +**Categories Tested**: + +- **primitives**: Basic types (string, number, boolean, etc.) +- **objects**: Plain objects with various properties +- **arrays**: Arrays with different element types +- **builtinObjects**: Built-in JS objects (Date, RegExp, etc.) + +**Option Sets**: + +- **default**: Standard behavior (`{}`) +- **literal-primitives**: Use literal schemas (`{ literalPrimitives: true }`) + +Each category tests 50 generated samples per benchmark iteration. + +**Run**: + +```bash +npm run bench:value +``` + +--- + +## Running Benchmarks + +### Run All Benchmarks ```bash -# Run the main benchmark suite npm run bench +``` + +### Run Specific Suites -# Run benchmarks in watch mode for development -npm run bench:dev +```bash +# Pure schema assertions only +npm run bench:schema -# Run the CLI benchmark runner -npm run bench:runner +# Pure function assertions only +npm run bench:pure -# Quick performance check across all implementation types -npm run bench:runner -- --mode quick --suite comprehensive +# Schema-returning function assertions only +npm run bench:function-schema -# Specific implementation class benchmarks -npm run bench:runner -- --suite sync-function # 66 function-based sync assertions (all) -npm run bench:runner -- --suite sync-function-pure # 7 pure function sync assertions (fastest) -npm run bench:runner -- --suite sync-function-schema # 59 schema-returning function sync assertions -npm run bench:runner -- --suite sync-schema # 44 schema-based sync assertions -npm run bench:runner -- --suite async-function # 8 function-based async assertions +# Async assertions only +npm run bench:async -# Traditional categorical suites -npm run bench:runner -- --suite type collection comparison pattern +# ValueToSchema utility only +npm run bench:value -# Different benchmark modes -npm run bench:runner -- --mode quick # Fast iteration (development) -npm run bench:runner -- --mode default # Balanced accuracy -npm run bench:runner -- --mode comprehensive # High accuracy (CI/production) +# All sync suites +npm run bench:sync ``` -## Benchmark Suites +### Run with Different Reporters + +```bash +# Human-readable output (default) +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --reporters human + +# JSON output for programmatic analysis +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --reporters json + +# CSV output for spreadsheet analysis +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --reporters csv + +# Multiple reporters +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --reporters human,json,csv +``` -### πŸ†• Implementation Class Suites +### Filter by Tags -These suites benchmark **ALL assertions** grouped by their implementation classes, providing insights into the performance characteristics of different assertion architectures: +```bash +# Run only sync benchmarks +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --tags sync -#### **`comprehensive`** - Complete Analysis +# Run only function-based benchmarks +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --tags function -- Assertion Implementation Distribution: - - Sync Function-based: 66 assertions (pure: 7, schema: 59) - - Sync Schema-based: 44 assertions - - Async Function-based: 8 assertions - - Async Schema-based: 0 assertions - - **Total: 118 assertions** +# Run utility benchmarks +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --tags utility +``` -#### **`sync-function`** - All Function-based Sync Assertions +### Adjust Iterations -- Tests all assertions that use callback functions for validation (66 total) -- Includes both pure and schema-returning function implementations -- Example task names: `"{unknown} 'to be an instance of' / 'to be a' / 'to be an' {constructible}" [sync-function]` +```bash +# Quick run for development +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --iterations 50 -#### **`sync-function-pure`** - Pure Function Sync Assertions (NEW) +# Comprehensive run for accuracy +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --iterations 500 -- Tests 7 assertions that return boolean or AssertionFailure objects directly -- Generally the fastest assertion type due to minimal overhead -- Examples: Set operations (`'to have union'`, `'to have intersection'`) and function error validation -- Performance threshold: 1200 ops/sec +# Adjust time limit per benchmark +node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --time 2000 +``` -#### **`sync-function-schema`** - Schema-returning Function Sync Assertions (NEW) +## Architecture -- Tests 59 assertions that return Zod schemas or AssertionParseRequest objects -- More complex than pure functions but still function-based implementations -- Examples: Collection operations, type checking, comparison assertions -- Performance threshold: 800 ops/sec +### Programmatic Generation -#### **`sync-schema`** - Schema-based Sync Assertions +Benchmarks are **programmatically generated** at module load time, ensuring: -- Tests 44 assertions that use Zod schemas for validation -- Generally faster than function-based equivalents due to optimized schema execution -- Example: `"{unknown} 'to be a string'" [sync-schema]` +- All assertions are tested automatically +- No manual updates needed when assertions change +- Type-safe and maintainable code +- Consistent testing approach + +### Directory Structure + +```text +bench/ +β”œβ”€β”€ shared/ +β”‚ β”œβ”€β”€ assertion-data.js # Test data generation utilities +β”‚ β”œβ”€β”€ benchmark-generator.js # Benchmark factory functions +β”‚ └── config.js # Configuration presets +β”œβ”€β”€ sync-function-pure.bench.js +β”œβ”€β”€ sync-function-schema.bench.js +β”œβ”€β”€ sync-schema.bench.js +β”œβ”€β”€ async-function.bench.js +β”œβ”€β”€ value-to-schema.bench.js +└── README.md (this file) +``` -#### **`async-function`** - Function-based Async Assertions +### Shared Utilities -- Tests 8 Promise-based assertions with callback functions -- Includes reject/resolve patterns with parameter validation +#### `shared/assertion-data.js` -#### **`async-schema`** - Schema-based Async Assertions +- Combines all fast-check generators from `test-data/` +- `getTestDataForAssertion(assertion)` - Gets test data for any assertion +- `getPrimaryPhrase(assertion)` - Extracts display name +- `isThrowingAssertion(assertion)` - Identifies error-testing assertions -- Currently no async schema-based assertions in the codebase +#### `shared/benchmark-generator.js` -### Traditional Categorical Suites +- `createSyncBenchmark()` - Factory for sync assertions +- `createAsyncBenchmark()` - Factory for async assertions +- Handles error suppression for expected failures -#### **Type Assertions (`type`)** +#### `shared/config.js` -- Basic type checking assertions (string, number, boolean, etc.) -- Focuses on the fundamental assertion building blocks +- Benchmark configuration presets (quick, default, comprehensive) +- Per-suite configuration overrides +- Tag taxonomy for filtering -### Collection Assertions (`collection`) +### Test Data Generation -- Array and object operations (contains, length, keys, etc.) -- Tests performance with different collection sizes +Test data is **pre-generated** using fast-check arbitraries: -### Comparison Assertions (`comparison`) +1. Each assertion has corresponding generators in `test-data/` +2. Data is sampled once at module load time +3. Same data is reused across benchmark iterations for consistency +4. Ensures reproducible results -- Equality, inequality, and relational comparisons -- Number and string comparisons +### Error Handling -### Pattern Assertions (`pattern`) +Benchmarks suppress errors for assertions that **intentionally test error conditions**: -- Regular expression matching -- String pattern operations (startsWith, endsWith, includes) +- Assertions containing "throw", "reject", "fail" in their phrase +- These assertions are expected to throw/reject +- Other errors are logged as warnings -## Performance Thresholds +## Performance Notes -The benchmark system includes configurable performance thresholds by implementation type: +### Expected Performance Ranges -### Implementation-based Thresholds +Based on existing benchmarks: + +| Suite Type | Expected ops/sec | Notes | +| -------------------- | ---------------- | -------------------------------- | +| Sync Schema | >1500 | Pure schema validation (fastest) | +| Sync Function Pure | >1200 | Minimal overhead | +| Sync Function Schema | >800 | More complex schema generation | +| Async Function | >15000 | Promise overhead included | +| ValueToSchema | >5000 | Depends on input complexity | + +### Performance Thresholds + +The benchmark system tracks performance thresholds by implementation type: - **sync-function**: 1000 ops/sec (general function-based sync assertions) - **sync-function-pure**: 1200 ops/sec (pure function assertions - highest threshold) @@ -134,88 +286,174 @@ The benchmark system includes configurable performance thresholds by implementat - **async-function**: 15000 ops/sec (async function-based assertions) - **async-schema**: 15000 ops/sec (async schema-based assertions) -### Legacy Categorical Thresholds +### Optimization Tips + +1. **Warm-up**: ModestBench includes warmup iterations by default +2. **Iterations**: Adjust with `--iterations` for accuracy vs speed +3. **Time Limit**: Use `--time` to control max time per benchmark +4. **Concurrent**: Consider `--concurrent` for independent benchmarks (use cautiously) + +## Historical Tracking + +ModestBench automatically stores results in `.modestbench-history/`: + +```bash +# List recent runs +node --import tsx node_modules/.bin/modestbench history list + +# Show detailed results +node --import tsx node_modules/.bin/modestbench history show + +# Compare two runs +node --import tsx node_modules/.bin/modestbench history compare + +# Export data +node --import tsx node_modules/.bin/modestbench history export --format csv --output results.csv + +# Clean old data +node --import tsx node_modules/.bin/modestbench history clean --older-than 30d +``` + +## CI/CD Integration + +Example GitHub Actions workflow: + +```yaml +name: Benchmarks +on: [push, pull_request] + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 20 + + - run: npm ci + - run: npm run build + + - name: Run Benchmarks + run: npm run bench + + - name: Export Results + run: node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --reporters json,csv --output ./results -- **Basic assertions**: 1.0ms threshold -- **Collection operations**: 2.0ms threshold -- **Comparison operations**: 1.5ms threshold -- **Complex operations**: 5.0ms threshold -- **Regex operations**: 3.0ms threshold + - name: Upload Results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: ./results/ +``` -Benchmarks that exceed these thresholds will generate performance warnings during execution with `--check` flag. +For continuous integration, benchmarks can be used to catch performance regressions across releases. -## Suite Overlap Resolution +## Troubleshooting -The benchmark runner automatically handles overlapping suite selections to prevent duplicate execution: +### Benchmarks Not Found -- **Parent-Child Relationships**: If you select both `sync-function` and its child suites (`sync-function-pure`, `sync-function-schema`), the runner will automatically remove the child suites and only execute the parent suite -- **Deduplication Feedback**: The runner provides clear messages about which suites were removed due to overlap -- **Efficient Execution**: This ensures each assertion is benchmarked exactly once per run +**Problem**: `No benchmark files found` -Examples: +**Solution**: Ensure you're running from the bupkis root directory: ```bash -# This will only run sync-function (child suites are automatically removed) -npm run bench:runner -- --suite sync-function --suite sync-function-pure +cd /path/to/bupkis +npm run bench +``` + +### Import Errors -# This will run both child suites independently -npm run bench:runner -- --suite sync-function-pure --suite sync-function-schema +**Problem**: `Cannot find module` errors + +**Solution**: Ensure all dependencies are installed: + +```bash +npm install +npm run build ``` -## Configuration Modes +### ModestBench Symlink Bug -### Quick Mode +**Known Issue**: ModestBench has a bug with symlinked installations ([#4](https://github.com/boneskull/modestbench/issues/4)) -- 50 iterations, 500ms time limit -- 5 warmup iterations, 50ms warmup time -- Ideal for rapid development feedback +**Workaround**: Use `node --import tsx node_modules/.bin/modestbench` instead of `npx modestbench` -### Default Mode +This is already reflected in the npm scripts. -- 100 iterations, 1000ms time limit -- 10 warmup iterations, 100ms warmup time -- Balanced accuracy and speed +### Test Data Generation Fails -### Comprehensive Mode +**Problem**: `No generator found for assertion` -- 200 iterations, 2000ms time limit -- 20 warmup iterations, 200ms warmup time -- Maximum accuracy for CI/CD and release validation +**Solution**: Verify all generator maps are exported from `test-data/index.ts`: -## Test Data Generators +```bash +npm run build +``` + +### Performance Degradation + +**Problem**: Benchmarks are slower than expected -The `TEST_DATA` object provides consistent test data: +**Solution**: -- `simpleObject()` - Basic 3-property object -- `nestedObject()` - Complex user object with nested properties -- `largeArray(size)` - Configurable large array for stress testing -- `deepObject()` - Deeply nested object (4 levels) -- `stringArray()` - Array of fruit names for string operations -- `mixedArray()` - Array with mixed data types +1. Check system load +2. Close other applications +3. Run comprehensive mode for accurate results: -## Usage in CI/CD + ```bash + node --import tsx node_modules/.bin/modestbench run bench/**/*.bench.js --iterations 500 --time 2000 + ``` -For continuous integration, use comprehensive mode to catch performance regressions: +### Memory Issues + +**Problem**: Out of memory errors + +**Solution**: Reduce sample size or run suites individually: ```bash -npm run bench:runner -- --mode comprehensive +npm run bench:pure +npm run bench:schema +# etc. ``` -The benchmarks will exit with a non-zero code if any performance thresholds are exceeded. +## Development + +### Adding New Assertions + +When new assertions are added to Bupkis: + +1. **Add generators** to `test-data/` (required) +2. **Export from index** in appropriate generator map +3. **Rebuild**: `npm run build` +4. **Run benchmarks**: Assertions are automatically included + +No changes needed to benchmark files! + +### Modifying Benchmark Configuration + +Edit `shared/config.js` to adjust: + +- Iteration counts per suite +- Time limits +- Warmup settings +- Tag taxonomy -## Adding New Benchmarks +### Creating New Suites -1. **Add to existing suite**: Modify the appropriate function in `suites.ts` -2. **Create new suite**: Add a new create function in `suites.ts` and update `runner.ts` -3. **Update main benchmark**: Add cases to `index.ts` for the comprehensive overview +1. Create new `.bench.js` file in `bench/` +2. Import utilities from `shared/` +3. Follow existing pattern for programmatic generation +4. Export modestbench-compatible structure +5. Add npm script to `package.json` -## Performance Monitoring +## Resources -The benchmark system is designed to help monitor: +- [ModestBench Documentation](https://github.com/boneskull/modestbench#readme) +- [TinyBench (underlying library)](https://github.com/tinylibs/tinybench) +- [Fast-Check (test data generation)](https://github.com/dubzzz/fast-check) +- [Bupkis Documentation](../README.md) -- **Performance regressions** between releases -- **Scaling behavior** with different data sizes -- **Assertion category performance** to identify bottlenecks -- **Development impact** of code changes +--- -Use the benchmark results to ensure Bupkis maintains excellent performance characteristics across all assertion types. +**Questions?** Open an issue or check the main Bupkis documentation. diff --git a/bench/assertion-classifier.ts b/bench/assertion-classifier.ts index 065e70d..5e4edf1 100644 --- a/bench/assertion-classifier.ts +++ b/bench/assertion-classifier.ts @@ -1,126 +1,84 @@ /** - * Assertion classification utilities for benchmark suite partitioning. - * - * This module provides utilities to classify sync-function assertions by their - * return types, enabling targeted performance analysis: - * - * - Pure assertions: return boolean or AssertionFailure - * - Schema-based assertions: return Zod schema or AssertionParseRequest + * Utility for classifying sync function assertions by their return types. */ -import { BupkisAssertionFunctionSync } from '../src/assertion/assertion-sync.js'; -import { type AnyAssertion, SyncAssertions } from '../src/assertion/index.js'; +import type { AnyAssertion } from '../src/assertion/index.js'; -/** - * Classification result for sync-function assertions - */ -export interface AssertionClassification { - /** Assertions that return boolean or AssertionFailure */ +import { + BupkisAssertionFunctionSync, + BupkisAssertionSchemaSync, +} from '../src/assertion/assertion-sync.js'; +import { SyncAssertions } from '../src/assertion/index.js'; +import { isBoolean, isZodType } from '../src/guards.js'; +import { + isAssertionFailure, + isAssertionParseRequest, +} from '../src/internal-schema.js'; + +type AssertionClassification = 'pure' | 'schema'; + +interface SyncFunctionAssertionClassification { pure: BupkisAssertionFunctionSync[]; - /** Assertions that return Zod schema or AssertionParseRequest */ schema: BupkisAssertionFunctionSync[]; - /** Total number of sync-function assertions found */ - total: number; } -/** - * Type guard to check if assertion is a sync function-based implementation - */ -export const isSyncFunctionAssertion = ( - assertion: T, -): assertion is BupkisAssertionFunctionSync & T => - assertion instanceof BupkisAssertionFunctionSync; +export const isSyncFunctionAssertion = ( + assertion: AnyAssertion, +): assertion is BupkisAssertionFunctionSync => { + return ( + assertion instanceof BupkisAssertionFunctionSync && + !(assertion instanceof BupkisAssertionSchemaSync) + ); +}; -/** - * Classifies a sync-function assertion by analyzing its implementation to - * determine whether it returns pure values (boolean/AssertionFailure) or - * schema-based values (Zod schema/AssertionParseRequest). - * - * NOTE: This function performs static analysis by examining the implementation - * function source code since we cannot execute the functions without proper - * arguments. The classification is based on return type patterns. - */ export const classifyAssertion = ( assertion: BupkisAssertionFunctionSync, -): 'pure' | 'schema' => { - const impl = assertion.impl as (...args: any[]) => any; - const source = impl.toString(); +): AssertionClassification => { + try { + // These are dummy args to provide to the assertion implementation which + // will most certainly cause it to fail. We can then inspect the result to + // determine if it is a pure or schema assertion. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + const dummyArgs = assertion.slots.map((_slot: unknown, i: number) => { + if (i === 0) { + return null; + } + return ''; + }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call + const result = assertion.impl(...dummyArgs); - // Look for patterns that indicate schema-based return types - const schemaPatterns = [ - // Direct Zod schema returns - /return\s+\w*[Ss]chema/, - // Zod method calls - /\.(?:gt|gte|lt|lte|min|max|length|regex|email|url|uuid|includes|startsWith|endsWith)\(/, - // Schema creation patterns - /z\.(?:string|number|boolean|object|array|literal|enum|union|intersection|record|map|set)\(/, - // AssertionParseRequest patterns - /AssertionParseRequest/, - // Schema variable assignments followed by return - /(?:const|let|var)\s+\w*[Ss]chema\s*=.*?return\s+\w*[Ss]chema/s, - ]; - - // Look for patterns that indicate pure return types - const purePatterns = [ - // Boolean returns - /return\s+(?:true|false|\w+\s*[=!]==?\s*|\w+\s*[<>]=?\s*|\w+\s*instanceof\s+)/, - // Comparison operations - /return\s+[^;{]+[<>!=]=?[^;{]+/, - // Method calls that typically return boolean - /return\s+\w+\.(?:test|match|includes|startsWith|endsWith|every|some|hasOwnProperty)\(/, - // AssertionFailure patterns - /AssertionFailure/, - // Error throwing (implicitly boolean/void) - /throw\s+new\s+\w*Error/, - ]; - - // Check for schema patterns first - const hasSchemaPattern = schemaPatterns.some((pattern) => - pattern.test(source), - ); - if (hasSchemaPattern) { + if (isBoolean(result) || isAssertionFailure(result)) { + return 'pure'; + } else if ( + isZodType(result) || + isAssertionParseRequest(result) || + (result && typeof result === 'object' && '_def' in result) + ) { + return 'schema'; + } else { + return 'schema'; + } + } catch { return 'schema'; } - - // Check for pure patterns - const hasPurePattern = purePatterns.some((pattern) => pattern.test(source)); - if (hasPurePattern) { - return 'pure'; - } - - // Default classification based on common patterns - // If the function is very short and simple, likely pure - if (source.length < 200 && /return\s+[^;{]+;?\s*}?\s*$/.test(source)) { - return 'pure'; - } - - // Default to schema if uncertain (safer for new schema-based assertions) - return 'schema'; }; -/** - * Gets all sync-function assertions and classifies them by return type. - * - * @returns Classification results with total count and categorized assertions - */ -export const getSyncFunctionAssertions = (): AssertionClassification => { - const syncFunctionAssertions = SyncAssertions.filter(isSyncFunctionAssertion); - - const pure: BupkisAssertionFunctionSync[] = []; - const schema: BupkisAssertionFunctionSync[] = []; +export const getSyncFunctionAssertions = + (): SyncFunctionAssertionClassification => { + const pure: BupkisAssertionFunctionSync[] = []; + const schema: BupkisAssertionFunctionSync[] = []; - for (const assertion of syncFunctionAssertions) { - const classification = classifyAssertion(assertion); - if (classification === 'pure') { - pure.push(assertion); - } else { - schema.push(assertion); + for (const assertion of SyncAssertions) { + if (isSyncFunctionAssertion(assertion)) { + const classification = classifyAssertion(assertion); + if (classification === 'pure') { + pure.push(assertion); + } else { + schema.push(assertion); + } + } } - } - return { - pure, - schema, - total: syncFunctionAssertions.length, + return { pure, schema }; }; -}; diff --git a/bench/async-function.bench.ts b/bench/async-function.bench.ts new file mode 100644 index 0000000..493e74e --- /dev/null +++ b/bench/async-function.bench.ts @@ -0,0 +1,46 @@ +/** + * Async Function Assertions Benchmark Suite + * + * Tests function-based async assertions that use callback functions for Promise + * validation. + * + * Includes reject/resolve patterns with parameter validation. This suite covers + * 8 assertions. + */ + +import type { BenchmarkDefinition } from './shared/benchmark-generator.js'; +import type { BenchmarkConfig } from './shared/config.js'; + +import { BupkisAssertionFunctionAsync } from '../src/assertion/assertion-async.js'; +import { AsyncAssertions } from '../src/assertion/index.js'; +import { getTestDataForAssertion } from './shared/assertion-data.js'; +import { createAsyncBenchmark } from './shared/benchmark-generator.js'; +import { SUITE_CONFIGS } from './shared/config.js'; + +const asyncFunctionAssertions = AsyncAssertions.filter( + (assertion): assertion is BupkisAssertionFunctionAsync => + assertion instanceof BupkisAssertionFunctionAsync, +); + +const benchmarks: Record = {}; + +for (const assertion of asyncFunctionAssertions) { + const name = `${assertion}`; + const testData = getTestDataForAssertion(assertion); + + benchmarks[name] = createAsyncBenchmark( + assertion, + testData, + ['async', 'function'], + {}, + ); +} + +export default { + suites: { + 'Async Function Assertions': { + benchmarks, + config: SUITE_CONFIGS['async-function'] as Partial, + }, + }, +}; diff --git a/bench/config.ts b/bench/config.ts deleted file mode 100644 index 0e9cc42..0000000 --- a/bench/config.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Benchmark configuration and utilities for Bupkis. - * - * This module provides common configuration and helper functions for all - * benchmark suites. - */ - -import type { BenchOptions } from 'tinybench'; - -/** - * Default benchmark configuration optimized for assertion testing. - */ -export const DEFAULT_BENCH_CONFIG: BenchOptions = { - iterations: 100, - time: 1000, - warmupIterations: 10, - warmupTime: 100, -}; - -/** - * Configuration for quick/development benchmarks. - */ -export const QUICK_BENCH_CONFIG: BenchOptions = { - iterations: 50, - time: 500, - warmupIterations: 5, - warmupTime: 50, -}; - -/** - * Configuration for comprehensive/CI benchmarks. - */ -export const COMPREHENSIVE_BENCH_CONFIG: BenchOptions = { - iterations: 200, - time: 2000, - warmupIterations: 20, - warmupTime: 200, -}; - -/** - * Configuration optimized for CI environments with limited resources. Focuses - * on relative performance and consistency over absolute numbers. - */ -export const CI_BENCH_CONFIG: BenchOptions = { - iterations: 30, - time: 1000, - warmupIterations: 5, - warmupTime: 100, -}; - -/** - * Performance thresholds for different implementation types (in ops/sec). Based - * on the new implementation-pattern-based suite organization. - */ -export const PERFORMANCE_THRESHOLDS = { - 'async-function': 15000, // Async function-based assertions (promise validation with callbacks) - 'async-schema': 15000, // Async schema-based assertions (promise validation with schemas) - 'sync-function': 1000, // Sync function-based assertions (validation with callback functions) - 'sync-function-pure': 1200, // Pure sync function assertions (return AssertionFailure/boolean) - 'sync-function-schema': 800, // Schema-based sync function assertions (return Zod schema/AssertionParseRequest) - 'sync-schema': 1500, // Sync schema-based assertions (validation with Zod schemas) - 'value-to-schema': 5000, // ValueToSchema function benchmarks -} as const; - -/** - * ANSI color codes for terminal output formatting. - */ -export const colors = { - bright: '\x1b[1m', - brightCyan: '\x1b[1m\x1b[36m', - brightGreen: '\x1b[1m\x1b[32m', - brightWhite: '\x1b[1m\x1b[37m', - dim: '\x1b[2m', - green: '\x1b[32m', - reset: '\x1b[0m', - white: '\x1b[37m', - yellow: '\x1b[33m', -} as const; diff --git a/bench/runner.ts b/bench/runner.ts deleted file mode 100644 index 027304e..0000000 --- a/bench/runner.ts +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env node - -/** - * Comprehensive benchmark runner for Bupkis. - * - * This script provides a CLI interface for running different benchmark suites - * and modes, useful for CI/CD and development. - */ - -import type { Task } from 'tinybench'; - -import { colors, PERFORMANCE_THRESHOLDS } from './config.js'; -import { - type BenchMode, - createAsyncFunctionAssertionsBench, - createAsyncSchemaAssertionsBench, - createSyncFunctionAssertionsBench, - createSyncFunctionPureAssertionsBench, - createSyncFunctionSchemaAssertionsBench, - createSyncSchemaAssertionsBench, - runBenchmarkSuite, -} from './suites.js'; -import { createValueToSchemaBench } from './value-to-schema-suite.js'; - -/** - * Helper function to format benchmark results for consistent output. - */ -export interface BenchmarkResult { - average: number; - max: number; - min: number; - name: string; - opsPerSec: number; - standardDeviation: number; -} - -/** - * Extracts and formats benchmark results from tinybench tasks. - */ -export const formatResults = (tasks: Task[]): BenchmarkResult[] => { - return tasks - .filter((task) => task.result && task.result.latency) - .map((task) => { - const result = task.result!; - - return { - average: result.latency.mean, - max: result.latency.max, - min: result.latency.min, - name: task.name, - opsPerSec: Math.round(1000 / result.latency.mean), - standardDeviation: result.latency.sd, - }; - }); -}; - -/** - * Checks results against performance thresholds and returns warnings. Uses - * suite-based thresholds based on implementation patterns. - */ -export const checkPerformance = ( - results: BenchmarkResult[], - thresholds: Record = PERFORMANCE_THRESHOLDS, -): { passed: boolean; warnings: string[] } => { - const warnings: string[] = []; - - for (const result of results) { - // Extract suite name from task name (format: "assertion [suite-name]") - const suiteMatch = result.name.match(/\[(.+?)\]$/); - const suiteName = suiteMatch?.[1]; - - if (!suiteName || !(suiteName in thresholds)) { - // Skip if we can't determine the suite or don't have a threshold for it - continue; - } - - const threshold = thresholds[suiteName]; - - if (threshold === undefined) { - // Skip if we don't have a threshold for this suite - continue; - } - - // Check if ops/sec is below the threshold (performance issue) - if (result.opsPerSec < threshold) { - // Parse the assertion name to separate the assertion string from the group - const groupMatch = result.name.match(/^(.+?)(\s+\[.+?\])$/); - if (groupMatch) { - const [, assertionString, group] = groupMatch; - warnings.push( - `${colors.dim}${group}${colors.reset} ${colors.brightGreen}${assertionString}${colors.reset}: ${colors.yellow}${result.opsPerSec}${colors.reset}/${colors.brightWhite}${threshold}${colors.reset} ops/sec${colors.reset}`, - ); - } else { - // Fallback for unexpected name format - warnings.push( - `${colors.brightGreen}${result.name}${colors.reset}: ${colors.yellow}${result.opsPerSec} ops/sec${colors.reset} ${colors.brightWhite}(below threshold: ${colors.yellow}${threshold} ops/sec${colors.brightWhite})${colors.reset}`, - ); - } - } - } - - return { - passed: warnings.length === 0, - warnings, - }; -}; - -interface RunnerOptions { - checkPerformance: boolean; - mode: BenchMode; - suites: string[]; - table: boolean; -} - -/** - * Available benchmark suites. - */ -const AVAILABLE_SUITES = { - all: 'Run all benchmark suites', - 'async-function': - 'Async function-based assertions (promise validation with callbacks)', - 'async-schema': - 'Async schema-based assertions (promise validation with schemas)', - 'sync-function': - 'Sync function-based assertions (validation with callback functions)', - 'sync-function-pure': - 'pure sync function assertions (return AssertionFailure/boolean)', - 'sync-function-schema': - 'schema-based sync function assertions (return Zod schema/AssertionParseRequest)', - 'sync-schema': 'Sync schema-based assertions (validation with Zod schemas)', - 'value-to-schema': 'ValueToSchema function performance benchmarks', -} as const; - -/** - * Parse command line arguments. - */ -const parseArgs = (): RunnerOptions => { - const args = process.argv.slice(2); - - let mode: BenchMode = 'default'; - let suites: string[] = ['all']; - let checkPerformance = false; - let table = false; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - - if (arg === '--mode' && i + 1 < args.length) { - const modeArg = args[i + 1] as BenchMode; - if ( - modeArg === 'ci' || - modeArg === 'quick' || - modeArg === 'default' || - modeArg === 'comprehensive' - ) { - mode = modeArg; - } else { - console.error( - `❌ Error: Invalid mode '${modeArg}'. Available modes: ci, quick, default, comprehensive`, - ); - process.exit(1); - } - i++; // Skip next arg since we consumed it - } else if (arg === '--suite' && i + 1 < args.length) { - if (suites.includes('all')) { - suites = []; // Clear 'all' if specific suites are specified - } - const suite = args[i + 1]; - if (suite) { - // Validate suite name - if (!Object.hasOwnProperty.call(AVAILABLE_SUITES, suite)) { - console.error( - `❌ Error: Invalid suite '${suite}'. Available suites: ${Object.keys(AVAILABLE_SUITES).join(', ')}`, - ); - process.exit(1); - } - suites.push(suite); - } - i++; // Skip next arg since we consumed it - } else if (arg === '--check') { - checkPerformance = true; - } else if (arg === '--table') { - table = true; - } else if (arg === '--help') { - printHelp(); - process.exit(0); - } - } - - return { checkPerformance, mode, suites, table }; -}; - -/** - * Print usage information. - */ -const printHelp = (): void => { - console.log(` -Bupkis Benchmark Runner - -Usage: npm run bench:runner [options] - -Options: - --mode Benchmark mode: quick, default, comprehensive, ci (default: default) - --suite Specific suite to run (can be used multiple times) - --check Check benchmark results against performance thresholds - --table Output results in table format - --help Show this help message - -Available suites: -${Object.entries(AVAILABLE_SUITES) - .map(([key, desc]) => ` ${key.padEnd(15)} ${desc}`) - .join('\n')} - -Examples: - npm run bench:runner - npm run bench:runner -- --mode quick - npm run bench:runner -- --suite sync-function --suite async-function - npm run bench:runner -- --mode comprehensive --check - `); -}; - -/** - * Resolves suite overlaps by removing child suites when parent suites are - * present. This prevents duplicate execution when hierarchical suites are - * selected. - * - * @param suites - Array of suite names to resolve - * @returns Resolved suite names with overlaps removed - */ -const resolveSuiteOverlaps = (suites: string[]): string[] => { - const resolvedSuites = [...suites]; - - // If sync-function is present, remove its child suites to prevent duplication - if (resolvedSuites.includes('sync-function')) { - const childSuites = ['sync-function-pure', 'sync-function-schema']; - const removedSuites = childSuites.filter((suite) => - resolvedSuites.includes(suite), - ); - - if (removedSuites.length > 0) { - console.log( - `ℹ️ Deduplication: Removed ${removedSuites.join(', ')} (overridden by sync-function)`, - ); - } - - return resolvedSuites.filter((suite) => !childSuites.includes(suite)); - } - - return resolvedSuites; -}; - -/** - * Run the specified benchmark suites. - */ -const runBenchmarks = async (options: RunnerOptions): Promise => { - console.log('πŸš€ Bupkis Performance Benchmark Runner\n'); - - if (options.checkPerformance) { - console.log('πŸ” Performance checking enabled'); - } - - // Resolve suite overlaps to prevent duplicate execution - const resolvedSuites = resolveSuiteOverlaps(options.suites); - const resolvedOptions = { ...options, suites: resolvedSuites }; - - // Show which suites will be executed - if (resolvedSuites.length === 1 && resolvedSuites[0] === 'all') { - console.log('πŸ“Š Running all benchmark suites\n'); - } else { - console.log( - `πŸ“Š Running ${resolvedSuites.length} suite${resolvedSuites.length === 1 ? '' : 's'}: ${resolvedSuites.join(', ')}\n`, - ); - } - - const startTime = Date.now(); - const benchResults: BenchmarkResult[] = []; - const tables: Array< - [ - name: string, - results: Array>, - ] - > = []; - - try { - // Implementation-based assertion benchmarks grouped by execution strategy - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('sync-function') - ) { - const bench = await runBenchmarkSuite( - 'Sync Function-based Assertions', - createSyncFunctionAssertionsBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['Sync Function-based Assertions', bench.table()]); - } - } - - // New granular sync-function suites - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('sync-function-pure') - ) { - const bench = await runBenchmarkSuite( - 'Sync Function-based Pure Assertions', - createSyncFunctionPureAssertionsBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['Sync Function-based Pure Assertions', bench.table()]); - } - } - - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('sync-function-schema') - ) { - const bench = await runBenchmarkSuite( - 'Sync Function-based Schema Assertions', - createSyncFunctionSchemaAssertionsBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['Sync Function-based Schema Assertions', bench.table()]); - } - } - - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('sync-schema') - ) { - const bench = await runBenchmarkSuite( - 'Sync Schema-based Assertions', - createSyncSchemaAssertionsBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['Sync Schema-based Assertions', bench.table()]); - } - } - - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('async-function') - ) { - const bench = await runBenchmarkSuite( - 'Async Function-based Assertions', - createAsyncFunctionAssertionsBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['Async Function-based Assertions', bench.table()]); - } - } - - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('async-schema') - ) { - const bench = await runBenchmarkSuite( - 'Async Schema-based Assertions', - createAsyncSchemaAssertionsBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['Async Schema-based Assertions', bench.table()]); - } - } - - if ( - resolvedOptions.suites.includes('all') || - resolvedOptions.suites.includes('value-to-schema') - ) { - const bench = await runBenchmarkSuite( - 'ValueToSchema Function Benchmarks', - createValueToSchemaBench, - options.mode, - ); - benchResults.push(...formatResults(bench.tasks)); - if (options.table) { - tables.push(['ValueToSchema Function Benchmarks', bench.table()]); - } - } - - const endTime = Date.now(); - const duration = (endTime - startTime) / 1000; - - console.log(`\nβœ… All benchmarks completed in ${duration.toFixed(2)}s`); - - if (options.table && tables.length) { - for (const [name, table] of tables) { - console.log(`\nπŸ“Š ${name} Results:`); - console.table(table); - } - } - - // Performance checking - if (options.checkPerformance && benchResults.length > 0) { - console.log('\nπŸ” Checking performance against thresholds...'); - const performanceCheck = checkPerformance(benchResults); - - if (performanceCheck.passed) { - console.log('βœ… All benchmarks meet performance thresholds'); - } else { - console.log('\n⚠️ Performance warnings:'); - for (const warning of performanceCheck.warnings) { - console.log(` ${warning}`); - } - - // Exit with error code in CI mode if performance check fails - if (options.mode === 'ci') { - console.log('\n❌ Performance thresholds not met in CI mode'); - process.exit(1); - } - } - } - } catch (error) { - console.error('\n❌ Benchmark failed:', error); - process.exit(1); - } -}; - -// Main execution -const main = async (): Promise => { - const options = parseArgs(); - await runBenchmarks(options); -}; - -// Run if this file is executed directly -if (import.meta.url === `file://${process.argv[1]}`) { - void main(); -} - -// Export for testing -export { AVAILABLE_SUITES, resolveSuiteOverlaps }; diff --git a/bench/shared/assertion-data.ts b/bench/shared/assertion-data.ts new file mode 100644 index 0000000..46d81c0 --- /dev/null +++ b/bench/shared/assertion-data.ts @@ -0,0 +1,116 @@ +/** + * Test data generation utilities for modestbench benchmarks. + * + * This module combines all fast-check generators from test-data/ and provides + * utilities to get test data for any assertion. + */ + +import fc from 'fast-check'; + +import type { AnyAssertion } from '../../src/assertion/index.js'; + +import { + AsyncParametricGenerators, + SyncBasicGenerators, + SyncCollectionGenerators, + SyncDateGenerators, + SyncEsotericGenerators, + SyncParametricGenerators, +} from '../../test-data/index.js'; + +type AssertionGenerators = + | fc.Arbitrary + | readonly fc.Arbitrary[]; + +const assertionArbitraries = new Map(); + +for (const [assertion, generators] of SyncBasicGenerators) { + assertionArbitraries.set(assertion, generators); +} +for (const [assertion, generators] of SyncCollectionGenerators) { + assertionArbitraries.set(assertion, generators); +} +for (const [assertion, generators] of SyncDateGenerators) { + assertionArbitraries.set(assertion, generators); +} +for (const [assertion, generators] of SyncEsotericGenerators) { + assertionArbitraries.set(assertion, generators); +} +for (const [assertion, generators] of SyncParametricGenerators) { + assertionArbitraries.set(assertion, generators); +} +for (const [assertion, generators] of AsyncParametricGenerators) { + assertionArbitraries.set(assertion, generators); +} + +const isGeneratorArray = ( + generators: AssertionGenerators, +): generators is readonly fc.Arbitrary[] => Array.isArray(generators); + +export const getTestDataForAssertion = ( + assertion: AnyAssertion, +): readonly [subject: unknown, phrase: string, ...args: unknown[]] => { + const generators = assertionArbitraries.get(assertion); + + if (!generators) { + throw new Error(`No generator found for assertion ${assertion}`); + } + + if (isGeneratorArray(generators)) { + const sample = fc.sample(fc.tuple(...generators), 1)[0]; + if (!sample) { + throw new Error(`Failed to sample generators for ${assertion}`); + } + return sample as unknown as readonly [ + subject: unknown, + phrase: string, + ...args: unknown[], + ]; + } else { + const sample = fc.sample(generators, 1)[0]; + if (!sample) { + throw new Error(`Failed to sample generators for ${assertion}`); + } + return sample; + } +}; + +const getPrimaryPhrase = (assertion: AnyAssertion): string => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const parts = assertion.parts; + + for (const part of parts) { + if (typeof part === 'string') { + return part; + } + + if (Array.isArray(part) && part.length > 0 && typeof part[0] === 'string') { + return part[0]; + } + } + + throw new Error( + `Could not determine primary phrase for assertion ${assertion}`, + ); +}; + +export const isThrowingAssertion = (assertion: AnyAssertion): boolean => { + const phrase = getPrimaryPhrase(assertion); + if (!phrase) { + return false; + } + + const throwingPatterns = [ + 'to throw', + 'throws', + 'to reject', + 'rejects', + 'to be rejected', + 'to fail', + 'fails', + ]; + + return throwingPatterns.some((pattern) => + phrase.toLowerCase().includes(pattern), + ); +}; diff --git a/bench/shared/benchmark-generator.ts b/bench/shared/benchmark-generator.ts new file mode 100644 index 0000000..a6f4dd6 --- /dev/null +++ b/bench/shared/benchmark-generator.ts @@ -0,0 +1,60 @@ +/** + * Factory functions for creating modestbench-compatible benchmark objects. + * + * Provides helpers to generate benchmark definitions for sync and async + * assertions with proper error handling. + */ + +import type { AnyAssertion } from '../../src/assertion/index.js'; +import type { BenchmarkConfig } from './config.js'; + +import { expect, expectAsync } from '../../src/index.js'; +import { isThrowingAssertion } from './assertion-data.js'; + +export interface BenchmarkDefinition { + config: Partial; + fn: () => Promise | void; + tags: string[]; +} + +export const createSyncBenchmark = ( + assertion: AnyAssertion, + testData: readonly [subject: unknown, phrase: string, ...args: unknown[]], + tags: string[] = [], + config: Partial = {}, +): BenchmarkDefinition => { + return { + config, + fn() { + try { + expect(...testData); + } catch (error) { + if (!isThrowingAssertion(assertion)) { + console.warn(`Unexpected error in ${assertion}:`, error); + } + } + }, + tags, + }; +}; + +export const createAsyncBenchmark = ( + assertion: AnyAssertion, + testData: readonly [subject: unknown, phrase: string, ...args: unknown[]], + tags: string[] = [], + config: Partial = {}, +): BenchmarkDefinition => { + return { + config, + async fn() { + try { + await expectAsync(...testData); + } catch (error) { + if (!isThrowingAssertion(assertion)) { + console.warn(`Unexpected error in ${assertion}:`, error); + } + } + }, + tags, + }; +}; diff --git a/bench/shared/config.ts b/bench/shared/config.ts new file mode 100644 index 0000000..43aa602 --- /dev/null +++ b/bench/shared/config.ts @@ -0,0 +1,34 @@ +/** + * Configuration presets for modestbench benchmarks. + * + * Defines benchmark configuration options and tag taxonomy. + */ + +export interface BenchmarkConfig { + iterations: number; + time: number; + warmupIterations?: number; + warmupTime?: number; +} +export const SUITE_CONFIGS: Record> = { + 'async-function': { + iterations: 50, + time: 1000, + }, + 'sync-function-pure': { + iterations: 200, + time: 1000, + }, + 'sync-function-schema': { + iterations: 100, + time: 1000, + }, + 'sync-schema': { + iterations: 150, + time: 1000, + }, + 'value-to-schema': { + iterations: 100, + time: 1000, + }, +}; diff --git a/bench/suites.ts b/bench/suites.ts deleted file mode 100644 index 10184da..0000000 --- a/bench/suites.ts +++ /dev/null @@ -1,488 +0,0 @@ -/** - * Comprehensive benchmark suites for bupkis assertion performance testing. - * - * This module provides benchmarks for ALL assertions grouped by their - * implementation classes, allowing for targeted performance analysis of - * function-based vs schema assertions and sync vs async execution. - */ - -import fc from 'fast-check'; -import { Bench } from 'tinybench'; - -import { - BupkisAssertionFunctionAsync, - BupkisAssertionSchemaAsync, -} from '../src/assertion/assertion-async.js'; -import { - BupkisAssertionFunctionSync, - BupkisAssertionSchemaSync, -} from '../src/assertion/assertion-sync.js'; -import { - type AnyAssertion, - AsyncAssertions, - SyncAssertions, -} from '../src/assertion/index.js'; -import { expect, expectAsync } from '../src/index.js'; -import { - AsyncParametricGenerators, - SyncBasicGenerators, - SyncCollectionGenerators, - SyncDateGenerators, - SyncEsotericGenerators, - SyncParametricGenerators, -} from '../test-data/index.js'; -import { type GeneratorParams } from '../test/property/property-test-config.js'; -import { getSyncFunctionAssertions } from './assertion-classifier.js'; -import { - CI_BENCH_CONFIG, - colors, - COMPREHENSIVE_BENCH_CONFIG, - DEFAULT_BENCH_CONFIG, - QUICK_BENCH_CONFIG, -} from './config.js'; - -/** - * Configuration for benchmark creation - */ -export interface BenchmarkConfig { - assertions: readonly AnyAssertion[]; - filter: (assertion: AnyAssertion) => boolean; - label: string; - name: string; - taskRunner: ( - assertion: AnyAssertion, - testData: unknown[], - ) => Promise | void; -} - -/** - * Factory function to create event handlers with timeout management - */ -export const createEventHandlers = (benchmarkName: string) => { - const taskTimeouts = new Map(); - - const startHandler = () => { - // this function intentionally left blank - }; - - const cycleHandler = (evt: any) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - const task = evt.task as { name: string; result?: { hz?: number } }; - - // Clear timeout for this task - const timeout = taskTimeouts.get(task.name); - if (timeout) { - clearTimeout(timeout); - taskTimeouts.delete(task.name); - } - - // Parse the task name to extract assertion title and collection - const match = task.name.match(/^(.+?)\s+\[(.+?)\]$/); - if (match) { - const [, assertionTitle, collection] = match; - const opsPerSec = task.result?.hz?.toFixed(0) ?? '?'; - console.log( - `βœ“ ${colors.dim}[${collection}]${colors.reset} ${colors.brightGreen}${assertionTitle}${colors.reset}: ${colors.yellow}${opsPerSec} ops/sec${colors.reset}`, - ); - } else { - // Fallback for unexpected format - console.log( - `βœ“ ${task.name}: ${colors.yellow}${task.result?.hz?.toFixed(0) ?? '?'} ops/sec${colors.reset}`, - ); - } - }; - - const completeHandler = () => { - console.log(`🏁 ${benchmarkName} benchmark complete!\n`); - // Clear any remaining timeouts - for (const timeout of taskTimeouts.values()) { - clearTimeout(timeout); - } - taskTimeouts.clear(); - }; - - const createTaskTimeout = (taskName: string) => { - const timeout = setTimeout(() => { - const error = new Error( - `Benchmark timeout: Task "${taskName}" took longer than 10 seconds to complete. This may indicate a hanging assertion or infinite loop.`, - ); - console.error(`\n⚠️ \x1b[31mTimeout Error:\x1b[0m ${error.message}`); - throw error; - }, 10000); - timeout.unref(); // Don't keep the process alive - taskTimeouts.set(taskName, timeout); - }; - - return { - completeHandler, - createTaskTimeout, - cycleHandler, - startHandler, - taskTimeouts, - }; -}; - -/** - * Factory function to create a benchmark with standardized setup - */ -export const createBenchmark = (config: BenchmarkConfig): Bench => { - const bench = new Bench(DEFAULT_BENCH_CONFIG); - const filteredAssertions = config.assertions.filter(config.filter); - - if (!filteredAssertions.length) { - console.log( - `ℹ️ No assertions matched the filter criteria. Skipping suite ${colors.yellow}${config.label}${colors.reset}`, - ); - return bench; - } - const handlers = createEventHandlers(config.name); - - // Set up event listeners - bench.addEventListener('start', handlers.startHandler); - bench.addEventListener('cycle', handlers.cycleHandler); - bench.addEventListener('complete', () => { - handlers.completeHandler(); - // Clean up event listeners - bench.removeEventListener('start', handlers.startHandler); - bench.removeEventListener('cycle', handlers.cycleHandler); - bench.removeEventListener('complete', handlers.completeHandler); - }); - - console.log( - `⏱️ Benchmarking ${colors.brightCyan}${filteredAssertions.length}${colors.reset} ${colors.yellow}${config.label}${colors.reset}`, - ); - - // Add benchmarks for each assertion - for (const assertion of filteredAssertions) { - const phrase = getPrimaryPhrase(assertion); - - if (phrase) { - const testData = getTestDataForAssertion(assertion); - const taskName = `${assertion} [${config.name}]`; - - bench.add(taskName, () => config.taskRunner(assertion, [...testData])); - } - } - - return bench; -}; - -/** - * Combined assertion arbitraries from all generator maps - */ -const assertionArbitraries = new Map(); -// Combine all generator maps -for (const [assertion, generators] of SyncBasicGenerators) { - assertionArbitraries.set(assertion, generators); -} -for (const [assertion, generators] of SyncCollectionGenerators) { - assertionArbitraries.set(assertion, generators); -} -for (const [assertion, generators] of SyncDateGenerators) { - assertionArbitraries.set(assertion, generators); -} -for (const [assertion, generators] of SyncEsotericGenerators) { - assertionArbitraries.set(assertion, generators); -} -for (const [assertion, generators] of SyncParametricGenerators) { - assertionArbitraries.set(assertion, generators); -} -for (const [assertion, generators] of AsyncParametricGenerators) { - assertionArbitraries.set(assertion, generators); -} - -/** - * Type guard to check if assertion is a sync function-based implementation - */ -const isSyncFunctionAssertion = ( - assertion: T, -): assertion is BupkisAssertionFunctionSync & T => - assertion instanceof BupkisAssertionFunctionSync; - -/** - * Type guard to check if assertion is a sync schema-based implementation - */ -const isSyncSchemaAssertion = ( - assertion: T, -): assertion is BupkisAssertionSchemaSync & T => - assertion instanceof BupkisAssertionSchemaSync; - -/** - * Type guard to check if assertion is an async function-based implementation - */ -const isAsyncFunctionAssertion = ( - assertion: T, -): assertion is BupkisAssertionFunctionAsync & T => - assertion instanceof BupkisAssertionFunctionAsync; - -/** - * Type guard to check if assertion is an async schema-based implementation - */ -const isAsyncSchemaAssertion = ( - assertion: T, -): assertion is BupkisAssertionSchemaAsync & T => - assertion instanceof BupkisAssertionSchemaAsync; - -/** - * Extract the primary phrase from assertion parts - */ -const getPrimaryPhrase = (assertion: AnyAssertion): null | string => { - const parts = assertion.parts as unknown[]; - - // Try each part until we find a string or string array - for (const part of parts) { - if (typeof part === 'string') { - return part; - } - if (Array.isArray(part) && part.length > 0 && typeof part[0] === 'string') { - return part[0]; - } - } - - throw new Error( - `Could not determine primary phrase for assertion ${assertion}`, - ); -}; - -/** - * Type guard to check if generators is an array of arbitraries - */ -const isGeneratorArray = ( - generators: any, -): generators is readonly [ - subject: fc.Arbitrary, - phrase: fc.Arbitrary, - ...fc.Arbitrary[], -] => Array.isArray(generators); - -/** - * Get generator params for an assertion that can be sampled and spread to - * expect/expectAsync. - * - * Ideally, this would return the exact Parts type for each assertion, but due - * to fast-check's typing limitations and the complexity of assertion type - * inference, we return a well-typed tuple structure that matches the expected - * signature for expect/expectAsync. - * - * @param assertion - The assertion to get test data for - * @returns Test data tuple matching the assertion's expected arguments - */ -const getTestDataForAssertion = ( - assertion: AnyAssertion, -): readonly [subject: unknown, phrase: string, ...unknown[]] => { - const generators = assertionArbitraries.get(assertion); - - if (generators) { - if (isGeneratorArray(generators)) { - // Convert array format to tuple and sample - const sample = fc.sample(fc.tuple(...generators), 1)[0]; - if (!sample) { - throw new Error(`Failed to sample generators for ${assertion}`); - } - return sample; - } else { - // Sample tuple format directly - const sample = fc.sample(generators, 1)[0]; - if (!sample) { - throw new Error(`Failed to sample generators for ${assertion}`); - } - return sample; - } - } - throw new Error(`No generator found for assertion ${assertion}`); -}; - -/** - * Determines if an assertion is expected to throw/reject based on its phrase. - * These assertions test error conditions and throwing is their normal - * behavior. - */ -const isThrowingAssertion = (assertion: AnyAssertion): boolean => { - const phrase = getPrimaryPhrase(assertion); - if (!phrase) { - return false; - } - - // List of phrases that indicate the assertion is meant to test error conditions - const throwingPatterns = [ - 'to throw', - 'throws', - 'to reject', - 'rejects', - 'to be rejected', - 'to fail', - 'fails', - ]; - - return throwingPatterns.some((pattern) => - phrase.toLowerCase().includes(pattern), - ); -}; - -/** - * Warns about unexpected exceptions during benchmarking. Only warns for - * assertions that aren't expected to throw. - */ -const warnUnexpectedException = ( - assertion: AnyAssertion, - error: unknown, -): void => { - if (!isThrowingAssertion(assertion)) { - console.warn( - `⚠️ Unexpected exception in benchmark for ${assertion}:`, - error instanceof Error ? error.message : error, - ); - } -}; - -/** - * Create benchmarks for sync function-based assertions. Tests assertions that - * use callback functions for validation. - */ -export const createSyncFunctionAssertionsBench = (): Bench => - createBenchmark({ - assertions: SyncAssertions, - filter: isSyncFunctionAssertion, - label: 'sync function-based assertions', - name: 'sync-function', - taskRunner: (assertion, testData) => { - try { - expect(...testData); - } catch (error) { - warnUnexpectedException(assertion, error); - } - }, - }); - -/** - * Create benchmarks for sync schema-based assertions. Tests assertions that use - * Zod schemas for validation. - */ -export const createSyncSchemaAssertionsBench = (): Bench => - createBenchmark({ - assertions: SyncAssertions, - filter: isSyncSchemaAssertion, - label: 'sync schema-based assertions', - name: 'sync-schema', - taskRunner: (assertion, testData) => { - try { - expect(...testData); - } catch (error) { - warnUnexpectedException(assertion, error); - } - }, - }); - -/** - * Create benchmarks for sync function-based assertions that return pure values. - * Tests assertions that use callback functions returning boolean or - * AssertionFailure. - */ -export const createSyncFunctionPureAssertionsBench = (): Bench => { - const { pure } = getSyncFunctionAssertions(); - - return createBenchmark({ - assertions: pure, - filter: () => true, // Already filtered by getSyncFunctionAssertions - label: 'sync function-based pure assertions', - name: 'sync-function-pure', - taskRunner: (assertion, testData) => { - try { - expect(...testData); - } catch (error) { - warnUnexpectedException(assertion, error); - } - }, - }); -}; - -/** - * Create benchmarks for sync function-based assertions that return schemas. - * Tests assertions that use callback functions returning Zod schemas or - * AssertionParseRequest. - */ -export const createSyncFunctionSchemaAssertionsBench = (): Bench => { - const { schema } = getSyncFunctionAssertions(); - - return createBenchmark({ - assertions: schema, - filter: () => true, // Already filtered by getSyncFunctionAssertions - label: 'sync function-based schema assertions', - name: 'sync-function-schema', - taskRunner: (assertion, testData) => { - try { - expect(...testData); - } catch (error) { - warnUnexpectedException(assertion, error); - } - }, - }); -}; - -/** - * Create benchmarks for async function-based assertions. Tests assertions that - * use callback functions for Promise validation. - */ -export const createAsyncFunctionAssertionsBench = (): Bench => - createBenchmark({ - assertions: AsyncAssertions, - filter: isAsyncFunctionAssertion, - label: 'async function-based assertions', - name: 'async-function', - taskRunner: async (assertion, testData) => { - try { - await expectAsync(...testData); - } catch (error) { - warnUnexpectedException(assertion, error); - } - }, - }); - -/** - * Create benchmarks for async schema-based assertions. Tests assertions that - * use Zod schemas for Promise validation. - */ -export const createAsyncSchemaAssertionsBench = (): Bench => - createBenchmark({ - assertions: AsyncAssertions, - filter: isAsyncSchemaAssertion, - label: 'async schema-based assertions', - name: 'async-schema', - taskRunner: async (assertion, testData) => { - try { - await expectAsync(...testData); - } catch (error) { - warnUnexpectedException(assertion, error); - } - }, - }); - -/** - * Configuration options for benchmark modes. - */ -export const BENCH_MODES = { - ci: CI_BENCH_CONFIG, - comprehensive: COMPREHENSIVE_BENCH_CONFIG, - default: DEFAULT_BENCH_CONFIG, - quick: QUICK_BENCH_CONFIG, -} as const; - -export type BenchMode = keyof typeof BENCH_MODES; - -/** - * Run a specific benchmark suite. - */ -export const runBenchmarkSuite = async ( - name: string, - createBench: (config?: any) => Bench, - mode: BenchMode = 'default', -): Promise => { - console.log( - `πŸ”§ Running ${colors.yellow}${name}${colors.reset} benchmarks in ${colors.white}${mode}${colors.reset} mode…`, - ); - - const bench = createBench(BENCH_MODES[mode]); - await bench.run(); - - return bench; -}; diff --git a/bench/sync-function-pure.bench.ts b/bench/sync-function-pure.bench.ts new file mode 100644 index 0000000..6d19ef5 --- /dev/null +++ b/bench/sync-function-pure.bench.ts @@ -0,0 +1,42 @@ +/** + * Sync Function Pure Assertions Benchmark Suite + * + * Tests pure function-based sync assertions that return boolean or + * AssertionFailure objects directly (no schema generation). + * + * These are typically the fastest assertion type due to minimal overhead. + * Examples: Set operations, function error validation. + */ + +import type { BenchmarkDefinition } from './shared/benchmark-generator.js'; +import type { BenchmarkConfig } from './shared/config.js'; + +import { getSyncFunctionAssertions } from './assertion-classifier.js'; +import { getTestDataForAssertion } from './shared/assertion-data.js'; +import { createSyncBenchmark } from './shared/benchmark-generator.js'; +import { SUITE_CONFIGS } from './shared/config.js'; + +const { pure } = getSyncFunctionAssertions(); + +const benchmarks: Record = {}; + +for (const assertion of pure) { + const name = `${assertion}`; + const testData = getTestDataForAssertion(assertion); + + benchmarks[name] = createSyncBenchmark( + assertion, + testData, + ['sync', 'function', 'pure'], + {}, + ); +} + +export default { + suites: { + 'Sync Function Pure Assertions': { + benchmarks, + config: SUITE_CONFIGS['sync-function-pure'] as Partial, + }, + }, +}; diff --git a/bench/sync-function-schema.bench.ts b/bench/sync-function-schema.bench.ts new file mode 100644 index 0000000..305999e --- /dev/null +++ b/bench/sync-function-schema.bench.ts @@ -0,0 +1,44 @@ +/** + * Sync Function Schema Assertions Benchmark Suite + * + * Tests function-based sync assertions that return Zod schemas or + * AssertionParseRequest objects. + * + * More complex than pure functions but still function-based implementations. + * Examples: Collection operations, type checking, comparison assertions. + * + * This suite covers 59 assertions. + */ + +import type { BenchmarkDefinition } from './shared/benchmark-generator.js'; +import type { BenchmarkConfig } from './shared/config.js'; + +import { getSyncFunctionAssertions } from './assertion-classifier.js'; +import { getTestDataForAssertion } from './shared/assertion-data.js'; +import { createSyncBenchmark } from './shared/benchmark-generator.js'; +import { SUITE_CONFIGS } from './shared/config.js'; + +const { schema } = getSyncFunctionAssertions(); + +const benchmarks: Record = {}; + +for (const assertion of schema) { + const name = `${assertion}`; + const testData = getTestDataForAssertion(assertion); + + benchmarks[name] = createSyncBenchmark( + assertion, + testData, + ['sync', 'function', 'schema-returning'], + {}, + ); +} + +export default { + suites: { + 'Sync Function Schema Assertions': { + benchmarks, + config: SUITE_CONFIGS['sync-function-schema'] as Partial, + }, + }, +}; diff --git a/bench/sync-schema.bench.ts b/bench/sync-schema.bench.ts new file mode 100644 index 0000000..465727c --- /dev/null +++ b/bench/sync-schema.bench.ts @@ -0,0 +1,47 @@ +/** + * Sync Schema-based Assertions Benchmark Suite + * + * Tests assertions that use Zod schemas for validation (not function-based). + * These are pure schema-based assertions without callback functions. + * + * Generally faster than function-based equivalents due to optimized schema + * execution. + * + * This suite covers 44 assertions. + */ + +import type { BenchmarkDefinition } from './shared/benchmark-generator.js'; + +import { BupkisAssertionSchemaSync } from '../src/assertion/assertion-sync.js'; +import { SyncAssertions } from '../src/assertion/index.js'; +import { getTestDataForAssertion } from './shared/assertion-data.js'; +import { createSyncBenchmark } from './shared/benchmark-generator.js'; +import { SUITE_CONFIGS } from './shared/config.js'; + +const syncSchemaAssertions = SyncAssertions.filter( + (assertion): assertion is BupkisAssertionSchemaSync => + assertion instanceof BupkisAssertionSchemaSync, +); + +const benchmarks: Record = {}; + +for (const assertion of syncSchemaAssertions) { + const name = `${assertion}`; + const testData = getTestDataForAssertion(assertion); + + benchmarks[name] = createSyncBenchmark( + assertion, + testData, + ['sync', 'schema'], + {}, + ); +} + +export default { + suites: { + 'Sync Schema Assertions': { + benchmarks, + config: SUITE_CONFIGS['sync-schema'], + }, + }, +}; diff --git a/bench/value-to-schema-suite.ts b/bench/value-to-schema-suite.ts deleted file mode 100644 index 3ea62d7..0000000 --- a/bench/value-to-schema-suite.ts +++ /dev/null @@ -1,392 +0,0 @@ -/** - * ValueToSchema benchmark suite implementation. - * - * Provides comprehensive performance testing for the valueToSchema() function - * across different input categories and configurations. - */ - -import * as fc from 'fast-check'; -import * as os from 'node:os'; -import { Bench } from 'tinybench'; - -import type { - BenchmarkConfig, - BenchmarkResult, - ExecutionContext, - GeneratorOptions, - PerformanceAnalysis, - PerformanceMetrics, -} from '../src/types.js'; -import type { ValueToSchemaOptions } from '../src/value-to-schema.js'; - -import { valueToSchema } from '../src/value-to-schema.js'; -import { valueToSchemaGeneratorFactory } from '../test-data/value-to-schema-generators.js'; -import { colors, DEFAULT_BENCH_CONFIG } from './config.js'; -import { createEventHandlers } from './suites.js'; - -const P95_Z_SCORE = 1.645; -const P99_Z_SCORE = 2.326; - -/** - * Creates a benchmark suite for valueToSchema() function. Follows the standard - * benchmark creation pattern used by assertion benchmarks. - */ -export const createValueToSchemaBench = (): Bench => { - const factory = valueToSchemaGeneratorFactory(); - - // Define test categories and their generators - const testCategories = [ - { - category: 'primitives', - generator: () => factory.createForCategory('primitives'), - }, - { - category: 'objects', - generator: () => factory.createForCategory('objects'), - }, - { - category: 'arrays', - generator: () => factory.createForCategory('arrays'), - }, - { - category: 'builtinObjects', - generator: () => factory.createForCategory('builtinObjects'), - }, - ]; - - const bench = new Bench(DEFAULT_BENCH_CONFIG); - - // Set up event handlers - const handlers = createEventHandlers('value-to-schema'); - bench.addEventListener('start', handlers.startHandler); - bench.addEventListener('cycle', handlers.cycleHandler); - bench.addEventListener('complete', () => { - handlers.completeHandler(); - bench.removeEventListener('start', handlers.startHandler); - bench.removeEventListener('cycle', handlers.cycleHandler); - bench.removeEventListener('complete', handlers.completeHandler); - }); - - console.log( - `⏱️ Benchmarking ${colors.brightCyan}${testCategories.length * 2}${colors.reset} ${colors.yellow}valueToSchema() operations${colors.reset}`, - ); - - // Add benchmark tasks for each category/option combination - for (const { category, generator } of testCategories) { - for (const options of [{}, { literalPrimitives: true }]) { - const optionsStr = - Object.keys(options).length > 0 - ? `-${Object.entries(options) - .map(([k, v]) => `${k}:${v}`) - .join('-')}` - : '-default'; - const taskName = `${category}${optionsStr} [value-to-schema]`; - - bench.add(taskName, () => { - // Generate fresh test data for each execution - const testData = fc.sample(generator(), 5); - for (const value of testData) { - valueToSchema(value, options); - } - }); - } - } - - return bench; -}; - -/** - * Runs comprehensive valueToSchema() benchmarks based on configuration. - */ -export const runValueToSchemaBenchmark = async ( - config: BenchmarkConfig, -): Promise => { - // Validate config - reasonable limits for both testing and production - if (!config.sampleSize || config.sampleSize < 1) { - throw new Error( - `Invalid sampleSize: ${config.sampleSize}. Must be at least 1`, - ); - } - if (!config.timeout || config.timeout < 200) { - throw new Error( - `Invalid timeout: ${config.timeout}ms. Must be at least 200ms`, - ); - } - if (config.iterations < 1 || config.iterations > 10000) { - throw new Error('Iterations must be between 1 and 10000'); - } - - const startTime = Date.now(); - const executionContext = getExecutionContext(); - const factory = valueToSchemaGeneratorFactory(); - - // Generate test data for all categories - const testData = generateTestDataSets(config, factory); - - // Create benchmark suite with sensible defaults for testing - const bench = new Bench({ - iterations: Math.min(config.iterations, 50), // Limit total iterations - time: Math.min(1000, config.timeout / 10), // Max 1 second per benchmark - warmupIterations: Math.min(config.warmupIterations, 5), // Limit warmup - warmupTime: 50, // Short warmup time - }); - - // Add benchmark tasks for each category/option combination - const benchmarkTasks: Array<{ - category: string; - data: unknown[]; - options: Partial; - }> = []; - - for (const category of config.categories || ['primitives']) { - const categoryData = testData[category] || []; - - for (const options of config.options || [{}]) { - const taskName = `${category}-${JSON.stringify(options)}`; - // Use much smaller sample size for tests to avoid hanging - const sampleSize = Math.min(config.sampleSize, 10); - const taskData = categoryData.slice(0, sampleSize); - - benchmarkTasks.push({ category, data: taskData, options }); - - bench.add(taskName, () => { - // Run valueToSchema on each data sample - but limit to just a few iterations - for (let i = 0; i < Math.min(taskData.length, 5); i++) { - valueToSchema(taskData[i], options); - } - }); - } - } - - // Run benchmarks - await bench.run(); - - // Process results - const results: PerformanceMetrics[] = []; - - for (let i = 0; i < bench.tasks.length; i++) { - const task = bench.tasks[i]; - const taskConfig = benchmarkTasks[i]; - - if (task && task.result && taskConfig) { - results.push({ - executionTime: { - mean: task.result.latency.mean * 1000, - median: task.result.latency.mean * 1000, - p95: - (task.result.latency.mean + P95_Z_SCORE * task.result.latency.sd) * - 1000, - p99: - (task.result.latency.mean + P99_Z_SCORE * task.result.latency.sd) * - 1000, - }, - inputCategory: taskConfig.category, - memoryUsage: { - external: process.memoryUsage().external, - heapTotal: process.memoryUsage().heapTotal, - heapUsed: process.memoryUsage().heapUsed, - }, - operationsPerSecond: task.result.throughput.mean || 0, - options: taskConfig.options, - timestamp: new Date(), - }); - } - } - - const analysis = analyzeResults(results); - const executionTime = Date.now() - startTime; - - return { - analysis, - executionContext, - executionTime, - metadata: { - nodeVersion: process.version, - timestamp: new Date().toISOString(), - version: '1.0.0', // Version of the benchmark suite - }, - results, - suiteId: 'value-to-schema', - }; -}; - -/** - * Generates test data for specific categories and counts. - */ -export const generateTestData = ( - category: string, - count: number, - options: GeneratorOptions = {}, -): { - category: string; - count: number; - data: unknown[]; - metadata: { - actualCount: number; - generationTime: number; - seed?: number; - }; -} => { - const startTime = Date.now(); - const factory = valueToSchemaGeneratorFactory(); - - try { - const generator = factory.createForCategory(category, options); - - // Limit the count for performance - const limitedCount = Math.min(count, 50); - - // Generate test data using fast-check with type casting - const samples = fc.sample(generator, limitedCount); - - const generationTime = Date.now() - startTime; - - return { - category, - count: limitedCount, - data: samples, - metadata: { - actualCount: samples.length, - generationTime, - ...(options.seedValue && { seed: options.seedValue }), - }, - }; - } catch { - // Return empty result for unsupported categories - const generationTime = Date.now() - startTime; - return { - category, - count: 0, - data: [], - metadata: { - actualCount: 0, - generationTime, - ...(options.seedValue && { seed: options.seedValue }), - }, - }; - } -}; - -/** - * Analyzes benchmark results to identify bottlenecks and patterns. - */ -export const analyzeResults = ( - metrics: PerformanceMetrics[], -): PerformanceAnalysis => { - if (metrics.length === 0) { - return { - bottlenecks: [], - outliers: [], - summary: { - averageOpsPerSecond: 0, - fastestCategory: 'none', - slowestCategory: 'none', - totalExecutionTime: 0, - }, - trends: [], - }; - } - - // Calculate summary statistics - const avgOps = - metrics.reduce((sum, m) => sum + (m.operationsPerSecond || 0), 0) / - metrics.length; - const totalExecutionTime = metrics.reduce( - (sum, m) => sum + (m.executionTime?.mean || 0), - 0, - ); - const sortedByOps = [...metrics].sort( - (a, b) => (b.operationsPerSecond || 0) - (a.operationsPerSecond || 0), - ); - - const fastest = sortedByOps[0]; - const slowest = sortedByOps[sortedByOps.length - 1]; - - // Identify bottlenecks (operations significantly slower than average) - const bottlenecks = metrics - .filter((m) => (m.operationsPerSecond || 0) < avgOps * 0.5) // Less than 50% of average - .map((m) => ({ - category: m.inputCategory || 'unknown', - impact: ((m.operationsPerSecond || 0) < avgOps * 0.25 - ? 'high' - : 'medium') as 'high' | 'low' | 'medium', - opsPerSecond: m.operationsPerSecond || 0, - reason: `Low throughput: ${(m.operationsPerSecond || 0).toFixed(0)} ops/sec vs average ${avgOps.toFixed(0)} ops/sec`, - })); - - // Identify outliers (execution times significantly different from mean) - const outliers = metrics - .filter((m) => { - const meanTime = m.executionTime?.mean || 0; - // Use p95 as a proxy for variability measure - const variability = - (m.executionTime?.p95 || 0) - (m.executionTime?.mean || 0); - return Math.abs(meanTime - avgOps) > variability * 2; - }) - .map((m) => ({ - category: m.inputCategory || 'unknown', - deviation: Math.abs((m.executionTime?.mean || 0) - avgOps), - options: m.options || {}, - value: m.executionTime?.mean || 0, - })); - - return { - bottlenecks, - outliers, - summary: { - averageOpsPerSecond: avgOps, - fastestCategory: fastest?.inputCategory || 'none', - slowestCategory: slowest?.inputCategory || 'none', - totalExecutionTime, - }, - trends: [], // Add trend analysis later if needed - }; -}; - -/** - * Generates test data sets for all configured categories. - */ -const generateTestDataSets = ( - config: BenchmarkConfig, - factory: ReturnType, -): Record => { - const testData: Record = {}; - const generatorOptions: GeneratorOptions = { - includeEdgeCases: false, // Disable edge cases for faster generation - maxArrayLength: 5, // Small arrays - maxDepth: 2, // Shallow nesting - }; - - for (const category of config.categories || ['primitives']) { - try { - const generator = factory.createForCategory(category, generatorOptions); - // Generate much fewer samples for tests - const sampleCount = Math.min(config.sampleSize, 20); - const samples = fc.sample( - generator as fc.Arbitrary, - sampleCount, - ); - testData[category] = samples; - } catch (_error) { - // Fallback to empty array if category not supported - testData[category] = []; - } - } - - return testData; -}; - -/** - * Gets current execution context information. - */ -const getExecutionContext = (): ExecutionContext => { - const cpus = os.cpus(); - const cpuModel = cpus.length > 0 && cpus[0] ? cpus[0].model : 'Unknown'; - - return { - cpuModel, - memoryTotal: os.totalmem(), - nodeVersion: process.version, - platform: `${os.type()} ${os.release()} ${os.arch()}`, - }; -}; diff --git a/bench/value-to-schema.bench.ts b/bench/value-to-schema.bench.ts new file mode 100644 index 0000000..4971479 --- /dev/null +++ b/bench/value-to-schema.bench.ts @@ -0,0 +1,93 @@ +/** + * ValueToSchema Utility Benchmark Suite + * + * Tests the valueToSchema() function performance across different input + * categories and configuration options. + * + * Categories tested: + * + * - Primitives: Basic types (string, number, boolean, etc.) + * - Objects: Plain objects with various properties + * - Arrays: Arrays with different element types + * - BuiltinObjects: Built-in JS objects (Date, RegExp, etc.) + * + * Each category is tested with two option sets: + * + * - Default: Standard behavior + * - LiteralPrimitives: Use literal schemas for primitive values + * + * Total: 8 benchmarks (4 categories Γ— 2 option sets) + */ + +import fc from 'fast-check'; + +import type { ValueToSchemaOptions } from '../src/value-to-schema.js'; +import type { BenchmarkDefinition } from './shared/benchmark-generator.js'; +import type { BenchmarkConfig } from './shared/config.js'; + +import { valueToSchema } from '../src/value-to-schema.js'; +import { valueToSchemaGeneratorFactory } from '../test-data/value-to-schema-generators.js'; +import { SUITE_CONFIGS } from './shared/config.js'; + +const factory = valueToSchemaGeneratorFactory(); + +const categories = ['primitives', 'objects', 'arrays', 'builtinObjects']; + +interface OptionSet { + name: string; + options: ValueToSchemaOptions; +} + +const optionSets: OptionSet[] = [ + { name: 'default', options: {} }, + { name: 'literal-primitives', options: { literalPrimitives: true } }, +]; + +const testDataCache = new Map(); + +for (const category of categories) { + try { + const generator = factory.createForCategory(category); + const samples = fc.sample(generator, 50); + + for (const optionSet of optionSets) { + const key = `${category}-${optionSet.name}`; + testDataCache.set(key, samples); + } + } catch (error) { + console.warn( + `Failed to generate test data for category ${category}:`, + error, + ); + } +} + +const benchmarks: Record = {}; + +for (const category of categories) { + for (const optionSet of optionSets) { + const key = `${category}-${optionSet.name}`; + const testData = testDataCache.get(key); + + if (testData && testData.length > 0) { + benchmarks[key] = { + config: {}, + fn() { + for (const value of testData) { + valueToSchema(value, optionSet.options); + } + }, + tags: ['utility', 'value-to-schema', category], + }; + } + } +} + +export default { + suites: { + 'ValueToSchema Utility': { + benchmarks, + config: SUITE_CONFIGS['value-to-schema'] as Partial, + }, + }, +}; diff --git a/package-lock.json b/package-lock.json index 76bf219..4e6aef9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,13 +45,13 @@ "lint-staged": "16.2.5", "markdownlint-cli2": "0.18.1", "markdownlint-cli2-formatter-pretty": "0.0.8", + "modestbench": "0.0.3", "npm-run-all2": "8.0.4", "prettier": "3.6.2", "prettier-plugin-jsdoc": "1.3.3", "prettier-plugin-pkg": "0.21.2", "prettier-plugin-sort-json": "4.1.1", "serve": "14.2.5", - "tinybench": "5.0.1", "tshy": "3.0.3", "tsx": "4.20.6", "typedoc": "0.28.14", @@ -7996,6 +7996,178 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/modestbench": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/modestbench/-/modestbench-0.0.3.tgz", + "integrity": "sha512-kQ1eTZizzKOIIP13XIBn/hC//M00RzHYM9sQ5b7RCE9R0x4I2mnOWsb5ioxZ1SKS0a2tr3OMWANu2Fv1IyAefg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "cosmiconfig": "9.0.0", + "cosmiconfig-typescript-loader": "6.2.0", + "glob": "11.0.3", + "tinybench": "5.0.1", + "yargs": "18.0.0", + "zod": "4.1.12" + }, + "bin": { + "modestbench": "dist/cli/index.js" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/modestbench/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/modestbench/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "dev": true, + "license": "MIT" + }, + "node_modules/modestbench/node_modules/glob": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.3.tgz", + "integrity": "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.0.3", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/modestbench/node_modules/jackspeak": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", + "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/modestbench/node_modules/lru-cache": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", + "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/modestbench/node_modules/minimatch": { + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", + "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", + "dev": true, + "license": "ISC", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/modestbench/node_modules/path-scurry": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/modestbench/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/modestbench/node_modules/yargs": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", + "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^7.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/modestbench/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", diff --git a/package.json b/package.json index 88188d6..382a0f0 100644 --- a/package.json +++ b/package.json @@ -116,10 +116,13 @@ "tape" ], "scripts": { - "bench": "tsx bench/runner.ts", - "bench:dev": "tsx --watch bench/runner.ts", - "bench:value-to-schema": "tsx bench/runner.ts --suites=value-to-schema", - "bench:value-to-schema:dev": "tsx --watch bench/runner.ts --suites=value-to-schema", + "bench": "modestbench", + "bench:async": "modestbench \"bench/async-*.bench.js\"", + "bench:function-schema": "modestbench \"bench/sync-function-schema.bench.js\"", + "bench:pure": "modestbench \"bench/sync-function-pure.bench.js\"", + "bench:schema": "modestbench \"bench/sync-schema.bench.js\"", + "bench:sync": "modestbench \"bench/sync-*.bench.js\"", + "bench:value": "modestbench \"bench/value-to-schema.bench.js\"", "build": "tshy", "build:dev": "tshy --watch", "debug:assertion-ids": "tsx scripts/dump-assertion-ids.ts", @@ -141,8 +144,6 @@ "lint:types:dev": "run-s \"lint:types -- --watch\"", "prepare": "husky; run-s build", "profile:analyze": "tsx scripts/analyze-profiles.ts", - "profile:bench": "node --cpu-prof --cpu-prof-dir=.profiles --import tsx bench/runner.ts", - "profile:bench:clinic": "npx --yes clinic flame --dest .profiles -- node --import tsx bench/runner.ts", "profile:benchmarks": "tsx scripts/profile-benchmarks.ts", "profile:property": "node --cpu-prof --cpu-prof-dir=.profiles --import tsx --test \"test/property/**/*.test.ts\"", "profile:test": "run-s test:profile", @@ -200,13 +201,13 @@ "lint-staged": "16.2.5", "markdownlint-cli2": "0.18.1", "markdownlint-cli2-formatter-pretty": "0.0.8", + "modestbench": "0.0.3", "npm-run-all2": "8.0.4", "prettier": "3.6.2", "prettier-plugin-jsdoc": "1.3.3", "prettier-plugin-pkg": "0.21.2", "prettier-plugin-sort-json": "4.1.1", "serve": "14.2.5", - "tinybench": "5.0.1", "tshy": "3.0.3", "tsx": "4.20.6", "typedoc": "0.28.14", @@ -241,7 +242,6 @@ "node": { "entry": [ ".config/typedoc-plugin-bupkis.js", - "bench/*.ts", "scripts/*.js", "scripts/*.ts", "test/**/*.test.ts", @@ -252,7 +252,9 @@ "src/util.ts", "src/schema.ts", "src/guards.ts", - "src/diff.ts" + "src/diff.ts", + "test-data/index.ts", + "bench/**/*.bench.ts" ] } }, diff --git a/test/property/assertion-classification.test.ts b/test/property/assertion-classification.test.ts index 544e891..d403079 100644 --- a/test/property/assertion-classification.test.ts +++ b/test/property/assertion-classification.test.ts @@ -17,11 +17,15 @@ describe('assertion classification properties', () => { // All assertions should be either pure or schema, never both const allIds = new Set([ - ...classification.pure.map((a) => a.id), - ...classification.schema.map((a) => a.id), + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + ...classification.pure.map((a: any) => a.id), + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + ...classification.schema.map((a: any) => a.id), ]); - const pureIds = new Set(classification.pure.map((a) => a.id)); - const schemaIds = new Set(classification.schema.map((a) => a.id)); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const pureIds = new Set(classification.pure.map((a: any) => a.id)); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const schemaIds = new Set(classification.schema.map((a: any) => a.id)); // No overlaps between pure and schema const intersection = new Set( @@ -99,8 +103,10 @@ describe('assertion classification properties', () => { const classification2 = getSyncFunctionAssertions(); // Compare pure assertion IDs - const pure1Ids = new Set(classification1.pure.map((a) => a.id)); - const pure2Ids = new Set(classification2.pure.map((a) => a.id)); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const pure1Ids = new Set(classification1.pure.map((a: any) => a.id)); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const pure2Ids = new Set(classification2.pure.map((a: any) => a.id)); assert.deepEqual( [...pure1Ids].sort(), [...pure2Ids].sort(), @@ -108,8 +114,14 @@ describe('assertion classification properties', () => { ); // Compare schema assertion IDs - const schema1Ids = new Set(classification1.schema.map((a) => a.id)); - const schema2Ids = new Set(classification2.schema.map((a) => a.id)); + const schema1Ids = new Set( + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + classification1.schema.map((a: any) => a.id), + ); + const schema2Ids = new Set( + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + classification2.schema.map((a: any) => a.id), + ); assert.deepEqual( [...schema1Ids].sort(), [...schema2Ids].sort(),