Removed levenshtein and made Evalite compatible with autoevals

packages/evalite-core/src/index.ts

@@ -20,7 +20,13 @@ export declare namespace Evalite {
   };
 
   export type Score = {
-    score: number;
+    /**
+     * A number between 0 and 1.
+     *
+     * Added null for compatibility with {@link https://github.com/braintrustdata/autoevals | autoevals}.
+     * null scores will be reported as 0.
+     */
+    score: number | null;
     name: string;
   };
 

packages/evalite/package.json

@@ -22,7 +22,6 @@
   "dependencies": {
     "table": "^6.8.2",
     "commander": "^12.1.0",
-    "js-levenshtein": "^1.1.6",
     "tinyrainbow": "^1.2.0",
     "fastify": "^5.1.0",
     "@fastify/websocket": "11.0.1",
@@ -33,6 +32,7 @@
     "strip-ansi": "^7.1.0",
     "@types/js-levenshtein": "^1.1.3",
     "unstorage": "^1.13.1",
-    "ai": "^4.0.10"
+    "ai": "^4.0.10",
+    "autoevals": "^0.0.108"
   }
 }

packages/evalite/src/index.ts

@@ -1,5 +1,4 @@
 import type { Evalite } from "@evalite/core";
-import levenshtein from "js-levenshtein";
 import { inject, it } from "vitest";
 import { reportTraceLocalStorage } from "./traces.js";
 
@@ -76,35 +75,3 @@ export const evalite = <TInput, TExpected>(
     };
   });
 };
-
-export const Levenshtein = (args: Evalite.ScoreInput<string>) => {
-  if (args.expected === undefined) {
-    throw new Error("LevenshteinScorer requires an expected value");
-  }
-
-  const [output, expected] = [`${args.output}`, `${args.expected}`];
-  const maxLen = Math.max(output.length, expected.length);
-
-  let score = 1;
-  if (maxLen > 0) {
-    score = 1 - levenshtein(output, expected) / maxLen;
-  }
-
-  return {
-    name: "Levenshtein",
-    score,
-  };
-};
-
-export const numericDifference = (args: Evalite.ScoreInput<number>) => {
-  if (args.expected === undefined) {
-    throw new Error("NumericDifferenceScorer requires an expected value");
-  }
-
-  return {
-    name: "NumericDifference",
-    score: 1 - Math.abs(args.output - args.expected) / args.expected,
-  };
-};
-
-export { reportTrace } from "./traces.js";

packages/evalite/src/tests/fixtures/ai-sdk-traces/traces.eval.ts

@@ -1,7 +1,8 @@
 import { generateText } from "ai";
 import { MockLanguageModelV1 } from "ai/test";
 import { traceAISDKModel } from "../../../ai-sdk.js";
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 
 const model = new MockLanguageModelV1({
   doGenerate: async () => ({

packages/evalite/src/tests/fixtures/basics/basics.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 import { setTimeout } from "node:timers/promises";
 
 evalite("Basics", {

packages/evalite/src/tests/fixtures/failing-test/failing-test.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 
 evalite("Failing", {
   data: () => {

packages/evalite/src/tests/fixtures/long-text/long-text.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 
 evalite("Long Text", {
   data: () => {

packages/evalite/src/tests/fixtures/much-data/much-data.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 import { setTimeout } from "node:timers/promises";
 
 evalite("Much Data", {

packages/evalite/src/tests/fixtures/multi/multi-1.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 import { setTimeout } from "node:timers/promises";
 
 evalite("Multiple 1", {

packages/evalite/src/tests/fixtures/multi/multi-2.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 import { setTimeout } from "node:timers/promises";
 
 evalite("Multiple 2", {

packages/evalite/src/tests/fixtures/multi/multi-3.eval.ts

@@ -1,4 +1,5 @@
-import { evalite, Levenshtein } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { Levenshtein } from "autoevals";
 import { setTimeout } from "node:timers/promises";
 
 evalite("Multiple 3", {

packages/evalite/src/tests/fixtures/traces/traces.eval.ts

@@ -1,4 +1,6 @@
-import { evalite, Levenshtein, reportTrace } from "../../../index.js";
+import { evalite } from "../../../index.js";
+import { reportTrace } from "../../../traces.js";
+import { Levenshtein } from "autoevals";
 
 evalite("Traces", {
   data: () => {

packages/evalite/tsconfig.json

@@ -3,5 +3,6 @@
   "compilerOptions": {
     "outDir": "dist",
     "rootDir": "src"
-  }
+  },
+  "exclude": ["src/tests/playground"]
 }

packages/example/package.json

@@ -17,6 +17,7 @@
     "ai": "^3.4.33",
     "@ai-sdk/openai": "0.0.72",
     "dotenv": "^16.4.5",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "autoevals": "^0.0.108"
   }
 }

packages/example/src/basics.eval.ts

@@ -1,8 +1,9 @@
 import { openai } from "@ai-sdk/openai";
 import { generateText } from "ai";
-import { evalite, Levenshtein } from "evalite";
+import { evalite } from "evalite";
 import { createStorage } from "unstorage";
 import fsDriver from "unstorage/drivers/fs";
+import { Levenshtein } from "autoevals";
 import { cacheModel } from "./cache-model.js";
 
 const storage = createStorage({