WIP

apps/evalite-ui/app/routes/_index.tsx

@@ -3,6 +3,7 @@ import type { MetaFunction } from "@remix-run/node";
 import { Link, useLoaderData } from "@remix-run/react";
 import { useSubscribeToTestServer } from "~/use-subscribe-to-socket";
 import { scoreToPercent } from "~/utils";
+import { Zap } from "lucide-react";
 
 export const meta: MetaFunction = () => {
   return [
@@ -11,23 +12,77 @@ export const meta: MetaFunction = () => {
   ];
 };
 
-export const clientLoader = () => {
-  return getEvals();
+export const clientLoader = async () => {
+  const evals = await getEvals();
+
+  console.log(evals);
+
+  return {
+    menu: Object.entries(evals).map(([key, value]) => {
+      const mostRecentEval = value[0]!;
+
+      const secondMostRecentEval = value[1];
+
+      const score = mostRecentEval.score;
+
+      const state = !secondMostRecentEval
+        ? "first"
+        : score > secondMostRecentEval.score
+          ? "up"
+          : score < secondMostRecentEval.score
+            ? "down"
+            : "same";
+      return {
+        name: key,
+        state,
+        score,
+      };
+    }),
+  };
 };
 
 export default function Index() {
-  const files = useLoaderData<typeof clientLoader>();
+  const evals = useLoaderData<typeof clientLoader>();
 
   const server = useSubscribeToTestServer();
 
   return (
     <div className="">
-      <nav className="flex items-center p-6 space-x-6">
-        <h1 className="text-xl">Evalite</h1>
-        <span className="bg-gray-800 uppercase px-4 py-2 rounded text-xs">
+      <nav className="flex items-center p-4 px-6 space-x-12">
+        <h1 className="text-xl flex items-center space-x-3 tracking-tight font-semibold">
+          <Zap />
+          <span>Evalite</span>
+        </h1>
+        <span className="bg-gray-800 uppercase px-4 py-1 rounded text-xs text-gray-200">
           {server.state === "idle" ? "idle" : "running"}
         </span>
       </nav>
+      <main className="px-6">
+        <div>
+          <ul className="space-y-4">
+            {evals.menu.map((menuItem) => (
+              <li key={menuItem.name}>
+                <Link to={`/${menuItem.name}`}>
+                  <div className="flex items-center justify-between">
+                    <span>{menuItem.name}</span>
+                    <span
+                      className={`${
+                        menuItem.state === "up"
+                          ? "text-green-500"
+                          : menuItem.state === "down"
+                            ? "text-red-500"
+                            : ""
+                      }`}
+                    >
+                      {scoreToPercent(menuItem.score)}
+                    </span>
+                  </div>
+                </Link>
+              </li>
+            ))}
+          </ul>
+        </div>
+      </main>
     </div>
   );
 }

apps/evalite-ui/app/use-subscribe-to-socket.ts

@@ -17,7 +17,7 @@ export const useSubscribeToTestServer = () => {
 
   useEffect(() => {
     const socket = new WebSocket(
-      `ws://localhost:${DEFAULT_SERVER_PORT}/socket`
+      `ws://localhost:${DEFAULT_SERVER_PORT}/api/socket`
     );
 
     socket.onmessage = (event) => {

apps/evalite-ui/package.json

@@ -10,13 +10,14 @@
     "lint": "tsc"
   },
   "dependencies": {
+    "@evalite/core": "workspace:*",
     "@remix-run/node": "^2.14.0",
     "@remix-run/react": "^2.14.0",
     "@remix-run/serve": "^2.14.0",
     "isbot": "^4.1.0",
+    "lucide-react": "^0.464.0",
     "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "@evalite/core": "workspace:*"
+    "react-dom": "^18.2.0"
   },
   "devDependencies": {
     "@remix-run/dev": "^2.14.0",

apps/evalite-ui/tsconfig.json

@@ -8,15 +8,8 @@
     "**/.client/**/*.tsx"
   ],
   "compilerOptions": {
-    "lib": [
-      "DOM",
-      "DOM.Iterable",
-      "ES2022"
-    ],
-    "types": [
-      "@remix-run/node",
-      "vite/client"
-    ],
+    "lib": ["DOM", "DOM.Iterable", "ES2022"],
+    "types": ["@remix-run/node", "vite/client"],
     "isolatedModules": true,
     "esModuleInterop": true,
     "jsx": "react-jsx",
@@ -30,13 +23,12 @@
     "forceConsistentCasingInFileNames": true,
     "allowImportingTsExtensions": true,
     "verbatimModuleSyntax": true,
+    "noUncheckedIndexedAccess": true,
     "baseUrl": ".",
     "paths": {
-      "~/*": [
-        "./app/*"
-      ]
+      "~/*": ["./app/*"]
     },
     // Vite takes care of building everything, not tsc.
     "noEmit": true
   }
-}
+}

packages/evalite-core/src/json-db.ts

@@ -109,6 +109,7 @@ export const getRows = async <T>(opts: {
   return dbContents
     .trim()
     .split("\n")
+    .filter(Boolean)
     .map((line) => {
       const parsed: JsonDBEval = JSON.parse(line);
 

packages/example/src/basics.eval.ts

@@ -52,3 +52,44 @@ evalite("Test basics", {
   },
   scorers: [Factuality],
 });
+
+evalite("Test basics 2", {
+  data: async () => [
+    {
+      input: `What's the capital of France?`,
+      expected: `Paris`,
+    },
+    {
+      input: `What's the capital of Germany?`,
+      expected: `Berlin`,
+    },
+    {
+      input: `What's the capital of Italy?`,
+      expected: `Rome`,
+    },
+    {
+      input: `What's the capital of the United States?`,
+      expected: `Washington DC`,
+    },
+    {
+      input: `What's the capital of Canada?`,
+      expected: `Ottawa`,
+    },
+    {
+      input: `What's the capital of Japan?`,
+      expected: `Tokyo`,
+    },
+  ],
+  task: async (input) => {
+    const result = await generateText({
+      model: cacheModel(openai("gpt-3.5-turbo"), storage),
+      system: `
+        Answer the question concisely, in as few words as possible.
+      `,
+      prompt: input,
+    });
+
+    return result.text;
+  },
+  scorers: [Factuality],
+});