diff --git a/packages/evalite/src/tests/ai-sdk-traces.test.ts b/packages/evalite/src/tests/ai-sdk-traces.test.ts new file mode 100644 index 0000000..116ad16 --- /dev/null +++ b/packages/evalite/src/tests/ai-sdk-traces.test.ts @@ -0,0 +1,20 @@ +import { getJsonDbEvals } from "@evalite/core"; +import { expect, it } from "vitest"; +import { runVitest } from "../command.js"; +import { captureStdout, loadFixture } from "./test-utils.js"; + +it("Should report traces from traceAISDKModel", async () => { + using fixture = loadFixture("ai-sdk-traces"); + + const captured = captureStdout(); + + await runVitest({ + cwd: fixture.dir, + path: undefined, + testOutputWritable: captured.writable, + }); + + const evals = await getJsonDbEvals({ dbLocation: fixture.jsonDbLocation }); + + expect(evals["AI SDK Traces"]![0]?.results[0]?.traces).toHaveLength(1); +}); diff --git a/packages/evalite/src/tests/fixtures/ai-sdk-traces/traces.eval.ts b/packages/evalite/src/tests/fixtures/ai-sdk-traces/traces.eval.ts new file mode 100644 index 0000000..8e8abcc --- /dev/null +++ b/packages/evalite/src/tests/fixtures/ai-sdk-traces/traces.eval.ts @@ -0,0 +1,35 @@ +import { generateText } from "ai"; +import { MockLanguageModelV1 } from "ai/test"; +import { traceAISDKModel } from "../../../ai-sdk.js"; +import { evalite, Levenshtein } from "../../../index.js"; + +const model = new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + text: `Hello, world!`, + }), +}); + +const tracedModel = traceAISDKModel(model); + +evalite("AI SDK Traces", { + data: () => { + return [ + { + input: "abc", + expected: "abcdef", + }, + ]; + }, + task: async (input) => { + const result = await generateText({ + model: tracedModel, + system: "Test system", + prompt: input, + }); + return result.text; + }, + scorers: [Levenshtein], +});