diff --git a/packages/evalite-tests/package.json b/packages/evalite-tests/package.json index 83b3485..675d188 100644 --- a/packages/evalite-tests/package.json +++ b/packages/evalite-tests/package.json @@ -12,6 +12,7 @@ "evalite": "workspace:*", "strip-ansi": "^7.1.0", "autoevals": "^0.0.108", + "@ai-sdk/openai": "^1.0.8", "ai": "^4.0.11" } } diff --git a/packages/evalite/src/index.ts b/packages/evalite/src/index.ts index 3d68bc9..495aca1 100644 --- a/packages/evalite/src/index.ts +++ b/packages/evalite/src/index.ts @@ -81,8 +81,9 @@ export const evalite = ( return describe(evalName, async () => { const dataset = await opts.data(); - for (const [order, data] of Object.entries(dataset)) { - it(`${evalName} ${order}`, { concurrent: true }, async ({ task }) => { + it.concurrent.for(dataset.map((d, index) => ({ ...d, index })))( + evalName, + async (data, { task }) => { task.meta.evalite = { duration: undefined, initialResult: { @@ -90,7 +91,7 @@ export const evalite = ( filepath: task.file.filepath, input: data.input, expected: data.expected, - order: Number(order), + order: data.index, }, }; const start = performance.now(); @@ -109,7 +110,7 @@ export const evalite = ( result: { evalName: evalName, filepath: task.file.filepath, - order: Number(order), + order: data.index, duration, expected: data.expected, input: data.input, @@ -125,7 +126,7 @@ export const evalite = ( result: { evalName: evalName, filepath: task.file.filepath, - order: Number(order), + order: data.index, duration: Math.round(performance.now() - start), expected: data.expected, input: data.input, @@ -138,8 +139,8 @@ export const evalite = ( }; throw e; } - }); - } + } + ); }); }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8a18af..fac5ebc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -227,6 +227,9 @@ importers: packages/evalite-tests: dependencies: + '@ai-sdk/openai': + specifier: ^1.0.8 + version: 1.0.8(zod@3.23.8) '@evalite/core': specifier: workspace:* version: link:../evalite-core @@ -275,6 +278,12 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/openai@1.0.8': + resolution: {integrity: sha512-wcTHM9qgRWGYVO3WxPSTN/RwnZ9R5/17xyo61iUCCSCZaAuJyh6fKddO0/oamwDp3BG7g+4wbfAyuTo32H+fHw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/provider-utils@1.0.22': resolution: {integrity: sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==} engines: {node: '>=18'} @@ -293,6 +302,15 @@ packages: zod: optional: true + '@ai-sdk/provider-utils@2.0.4': + resolution: {integrity: sha512-GMhcQCZbwM6RoZCri0MWeEWXRt/T+uCxsmHEsTwNvEH3GDjNzchfX25C8ftry2MeEOOn6KfqCLSKomcgK6RoOg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider@0.0.26': resolution: {integrity: sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==} engines: {node: '>=18'} @@ -301,6 +319,10 @@ packages: resolution: {integrity: sha512-mV+3iNDkzUsZ0pR2jG0sVzU6xtQY5DtSCBy3JFycLp6PwjyLw/iodfL3MwdmMCRJWgs3dadcHejRnMvF9nGTBg==} engines: {node: '>=18'} + '@ai-sdk/provider@1.0.2': + resolution: {integrity: sha512-YYtP6xWQyaAf5LiWLJ+ycGTOeBLWrED7LUrvc+SQIWhGaneylqbaGsyQL7VouQUeQ4JZ1qKYZuhmi3W56HADPA==} + engines: {node: '>=18'} + '@ai-sdk/react@0.0.70': resolution: {integrity: sha512-GnwbtjW4/4z7MleLiW+TOZC2M29eCg1tOUpuEiYFMmFNZK8mkrqM0PFZMo6UsYeUYMWqEOOcPOU9OQVJMJh7IQ==} engines: {node: '>=18'} @@ -4292,6 +4314,11 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanoid@3.3.8: + resolution: {integrity: sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} @@ -5957,6 +5984,12 @@ snapshots: '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) zod: 3.23.8 + '@ai-sdk/openai@1.0.8(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.2 + '@ai-sdk/provider-utils': 2.0.4(zod@3.23.8) + zod: 3.23.8 + '@ai-sdk/provider-utils@1.0.22(zod@3.23.8)': dependencies: '@ai-sdk/provider': 0.0.26 @@ -5975,6 +6008,15 @@ snapshots: optionalDependencies: zod: 3.23.8 + '@ai-sdk/provider-utils@2.0.4(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.2 + eventsource-parser: 3.0.0 + nanoid: 3.3.8 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.23.8 + '@ai-sdk/provider@0.0.26': dependencies: json-schema: 0.4.0 @@ -5983,6 +6025,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@1.0.2': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/react@0.0.70(react@18.3.1)(zod@3.23.8)': dependencies: '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) @@ -10664,6 +10710,8 @@ snapshots: nanoid@3.3.7: {} + nanoid@3.3.8: {} + napi-build-utils@1.0.2: {} natural-compare@1.4.0: {}