Added tests for ai-sdk-traces

packages/evalite/src/tests/ai-sdk-traces.test.ts

@@ -0,0 +1,20 @@
+import { getJsonDbEvals } from "@evalite/core";
+import { expect, it } from "vitest";
+import { runVitest } from "../command.js";
+import { captureStdout, loadFixture } from "./test-utils.js";
+
+it("Should report traces from traceAISDKModel", async () => {
+  using fixture = loadFixture("ai-sdk-traces");
+
+  const captured = captureStdout();
+
+  await runVitest({
+    cwd: fixture.dir,
+    path: undefined,
+    testOutputWritable: captured.writable,
+  });
+
+  const evals = await getJsonDbEvals({ dbLocation: fixture.jsonDbLocation });
+
+  expect(evals["AI SDK Traces"]![0]?.results[0]?.traces).toHaveLength(1);
+});

packages/evalite/src/tests/fixtures/ai-sdk-traces/traces.eval.ts

@@ -0,0 +1,35 @@
+import { generateText } from "ai";
+import { MockLanguageModelV1 } from "ai/test";
+import { traceAISDKModel } from "../../../ai-sdk.js";
+import { evalite, Levenshtein } from "../../../index.js";
+
+const model = new MockLanguageModelV1({
+  doGenerate: async () => ({
+    rawCall: { rawPrompt: null, rawSettings: {} },
+    finishReason: "stop",
+    usage: { promptTokens: 10, completionTokens: 20 },
+    text: `Hello, world!`,
+  }),
+});
+
+const tracedModel = traceAISDKModel(model);
+
+evalite("AI SDK Traces", {
+  data: () => {
+    return [
+      {
+        input: "abc",
+        expected: "abcdef",
+      },
+    ];
+  },
+  task: async (input) => {
+    const result = await generateText({
+      model: tracedModel,
+      system: "Test system",
+      prompt: input,
+    });
+    return result.text;
+  },
+  scorers: [Levenshtein],
+});