Added a cacheLanguageModel function

package.json

@@ -12,7 +12,8 @@
     "dev": "turbo watch build lint dev",
     "ci": "turbo build test lint",
     "build": "turbo build",
-    "release": "pnpm run ci && changeset publish"
+    "release": "pnpm run ci && changeset publish",
+    "test-example": "cd packages/example && evalite"
   },
   "keywords": [],
   "author": "Matt Pocock",

packages/evalite/.gitignore

@@ -1 +1,2 @@
-/src/tests/playground
+/src/tests/playground
+evalite-report.jsonl

packages/example/package.json

@@ -12,6 +12,7 @@
     "./reporter": "./dist/reporter.js"
   },
   "dependencies": {
+    "unstorage": "^1.13.1",
     "evalite": "workspace:*",
     "ai": "^3.4.33",
     "@ai-sdk/openai": "0.0.72",

packages/example/src/basics.eval.ts

@@ -1,12 +1,21 @@
 import { openai } from "@ai-sdk/openai";
 import { generateText } from "ai";
 import { evalite, Levenshtein } from "evalite";
+import { createStorage } from "unstorage";
+import fsDriver from "unstorage/drivers/fs";
+import { cacheLanguageModel } from "./cacheLanguageModel.js";
+
+const storage = createStorage({
+  driver: (fsDriver as any)({
+    base: "./llm-cache.local",
+  }),
+});
 
 evalite("Test basics", {
   data: async () => [
     {
       input: `What's the capital of France?`,
-      expected: `Lille.`,
+      expected: `Paris.`,
     },
     {
       input: `What's the capital of Germany?`,
@@ -20,10 +29,14 @@ evalite("Test basics", {
       input: `What's the capital of the United States?`,
       expected: `Washington, D.C.`,
     },
+    {
+      input: `What's the capital of Canada?`,
+      expected: `Ottawa.`,
+    },
   ],
   task: async (input) => {
     const result = await generateText({
-      model: openai("gpt-3.5-turbo"),
+      model: cacheLanguageModel(openai("gpt-3.5-turbo"), storage),
       system: `
         Answer the question concisely, in as few words as possible.
       `,

packages/example/src/cacheLanguageModel.ts

@@ -0,0 +1,45 @@
+import {
+  experimental_wrapLanguageModel,
+  type LanguageModelV1,
+  type LanguageModelV1CallOptions,
+} from "ai";
+import { createHash } from "node:crypto";
+import { type Storage } from "unstorage";
+
+const createKey = (params: LanguageModelV1CallOptions) => {
+  return createHash("sha256").update(JSON.stringify(params)).digest("hex");
+};
+
+const createResultFromCachedObject = (
+  obj: any
+): Awaited<ReturnType<LanguageModelV1["doGenerate"]>> => {
+  if (obj?.response?.timestamp) {
+    obj.response.timestamp = new Date(obj.response.timestamp);
+  }
+  return obj as any;
+};
+
+export const cacheLanguageModel = (
+  model: LanguageModelV1,
+  storage: Storage
+) => {
+  return experimental_wrapLanguageModel({
+    model,
+    middleware: {
+      wrapGenerate: async (opts) => {
+        const key = createKey(opts.params);
+
+        const resultFromCache = await storage.get(key);
+
+        if (resultFromCache && typeof resultFromCache === "object") {
+          return createResultFromCachedObject(resultFromCache);
+        }
+        const generated = await opts.doGenerate();
+
+        await storage.set(key, JSON.stringify(generated));
+
+        return generated;
+      },
+    },
+  });
+};

packages/example/vite.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    setupFiles: ["dotenv/config"],
+  },
+});