Various fixes

.vscode/settings.json

@@ -14,5 +14,5 @@
     "^@radix-ui",
     "^cmdk$"
   ],
-  "typescript.preferences.includePackageJsonAutoImports": "on",
-}
+  "typescript.preferences.includePackageJsonAutoImports": "on"
+}

apps/evalite-ui/app/components/score.tsx

@@ -46,3 +46,19 @@ export const Score = (props: {
     </span>
   );
 };
+
+export const getScoreState = (
+  score: number,
+  prevScore: number | null | undefined
+) => {
+  const state: ScoreState =
+    typeof prevScore === "undefined" || prevScore === null
+      ? "first"
+      : score > prevScore
+        ? "up"
+        : score < prevScore
+          ? "down"
+          : "same";
+
+  return state;
+};

apps/evalite-ui/app/root.tsx

@@ -35,7 +35,7 @@ import {
   TestServerStateContext,
   useSubscribeToTestServer,
 } from "./use-subscribe-to-socket";
-import { Score, type ScoreState } from "./components/score";
+import { getScoreState, Score, type ScoreState } from "./components/score";
 
 export const links: LinksFunction = () => [
   { rel: "preconnect", href: "https://fonts.googleapis.com" },
@@ -79,13 +79,7 @@ export const clientLoader = async () => {
 
       const score = mostRecentEval.score;
 
-      const state: ScoreState = !secondMostRecentEval
-        ? "first"
-        : score > secondMostRecentEval.score
-          ? "up"
-          : score < secondMostRecentEval.score
-            ? "down"
-            : "same";
+      const state = getScoreState(score, secondMostRecentEval?.score);
       return {
         name: key,
         state,

apps/evalite-ui/app/routes/eval.$name.tsx

@@ -3,7 +3,7 @@ import type { MetaFunction } from "@remix-run/node";
 import { useLoaderData, type ClientLoaderFunctionArgs } from "@remix-run/react";
 import { useContext } from "react";
 import { InnerPageLayout } from "~/components/page-header";
-import { Score } from "~/components/score";
+import { getScoreState, Score } from "~/components/score";
 import {
   Table,
   TableBody,
@@ -33,9 +33,10 @@ export const clientLoader = async (args: ClientLoaderFunctionArgs) => {
 };
 
 export default function Page() {
-  const { name, evaluation } = useLoaderData<typeof clientLoader>();
+  const { name, evaluation, prevEvaluation } =
+    useLoaderData<typeof clientLoader>();
 
-  const firstResult = evaluation.results[0]!;
+  const firstResult = evaluation.results[0];
 
   const serverState = useContext(TestServerStateContext);
 
@@ -47,7 +48,7 @@ export default function Page() {
             <TableHead>Input</TableHead>
             <TableHead>Output</TableHead>
             <TableHead>Expected</TableHead>
-            {firstResult.scores.map((scorer) => (
+            {firstResult?.scores.map((scorer) => (
               <TableHead key={scorer.name}>{scorer.name}</TableHead>
             ))}
           </TableRow>
@@ -60,6 +61,9 @@ export default function Page() {
                 <TableCell>{result.result as any}</TableCell>
                 <TableCell>{result.expected as any}</TableCell>
                 {result.scores.map((scorer) => {
+                  const scoreInPreviousEvaluation = prevEvaluation?.results
+                    .find((r) => r.input === result.input)
+                    ?.scores.find((s) => s.name === scorer.name);
                   return (
                     <TableCell key={scorer.name}>
                       <Score
@@ -68,7 +72,10 @@ export default function Page() {
                           serverState.state.type === "running" &&
                           serverState.state.filepaths.has(evaluation.filepath)
                         }
-                        state="up" // TODO
+                        state={getScoreState(
+                          scorer.score ?? 0,
+                          scoreInPreviousEvaluation?.score
+                        )}
                       />
                     </TableCell>
                   );

packages/evalite/src/index.ts

@@ -41,10 +41,6 @@ export const evalite = <TInput, TExpected>(
   opts: Evalite.RunnerOpts<TInput, TExpected>
 ) => {
   return it(testName, async ({ task }) => {
-    if (opts.scorers.length === 0) {
-      throw new Error("You must provide at least one scorer.");
-    }
-
     const traces: Evalite.StoredTrace[] = [];
 
     reportTraceLocalStorage.enterWith((trace) => traces.push(trace));

packages/evalite/src/reporter.ts

@@ -46,7 +46,7 @@ export default class EvaliteReporter extends BasicReporter {
   //   super.onTaskUpdate(packs);
   // }
 
-  override onWatcherStart(files: RunnerTestFile[], errors?: unknown[]): void {
+  override onWatcherStart(files?: RunnerTestFile[], errors?: unknown[]): void {
     super.onWatcherStart(files, errors);
   }
 

packages/evalite/src/run-vitest.ts

@@ -35,7 +35,7 @@ export const runVitest = async (opts: {
           },
         }),
       ],
-      slowTestThreshold: 30_000,
+      testTimeout: 30_000,
     },
     {},
     {

packages/example/src/content-generation.eval.ts

@@ -0,0 +1,48 @@
+import { generateText } from "ai";
+import { createScorer, evalite } from "evalite";
+import { cacheModel } from "./cache-model";
+import { openai } from "@ai-sdk/openai";
+import { createStorage } from "unstorage";
+import fsDriver from "unstorage/drivers/fs";
+import { Humor } from "autoevals";
+
+const storage = createStorage({
+  driver: (fsDriver as any)({
+    base: "./llm-cache.local",
+  }),
+});
+
+evalite("Content generation", {
+  data: async () => {
+    return [
+      {
+        input: "Write a TypeScript tweet",
+      },
+      {
+        input: "Write a tweet about TypeScript template literals types.",
+      },
+    ];
+  },
+  task: async (input) => {
+    const result = await generateText({
+      model: cacheModel(openai("gpt-4o-mini"), storage),
+      prompt: input,
+      system: `
+        You are a helpful social media assistant.
+        You will be asked to write a tweet on a given topic.
+        Return only the tweet.
+        Do not use emojis.
+        Do not use hashtags.
+        Use code examples where required.
+      `,
+    });
+
+    return result.text;
+  },
+  scorers: [
+    Humor,
+    createScorer("No Hashtags", ({ output }) => {
+      return output.includes("#") ? 0 : 1;
+    }),
+  ],
+});

packages/example/src/example.eval.ts

@@ -46,6 +46,10 @@ evalite("Test Capitals", {
       input: `Name all the capitals of each part of the UK.`,
       expected: `London, Edinburgh, Cardiff, Belfast`,
     },
+    {
+      input: `What's the capital of Antarctica?`,
+      expected: `Antarctica has no capital.`,
+    },
   ],
   task: async (input) => {
     const result = await generateText({