Fixed a bug where the loading indicators were not accurate on first l…

…oad.

.changeset/good-games-fly.md

@@ -0,0 +1,6 @@
+---
+"@evalite/core": patch
+"evalite": patch
+---
+
+Fixed a bug where the loading indicators were not accurate on first load.

apps/evalite-ui/app/root.tsx

@@ -20,7 +20,7 @@ import {
   SidebarProvider,
 } from "~/components/ui/sidebar";
 
-import { getMenuItems } from "@evalite/core/sdk";
+import { getMenuItems, getServerState } from "@evalite/core/sdk";
 import { getScoreState, Score, type ScoreState } from "./components/score";
 import { cn } from "./lib/utils";
 import "./tailwind.css";
@@ -65,10 +65,13 @@ export function Layout({ children }: { children: React.ReactNode }) {
 }
 
 export const clientLoader = async () => {
-  const { archivedEvals, currentEvals, prevScore, score, evalStatus } =
-    await getMenuItems();
+  const [
+    { archivedEvals, currentEvals, prevScore, score, evalStatus },
+    serverState,
+  ] = await Promise.all([getMenuItems(), getServerState()]);
 
   return {
+    serverState,
     evalStatus,
     prevScore,
     score,
@@ -92,7 +95,7 @@ export const clientLoader = async () => {
 export default function App() {
   const data = useLoaderData<typeof clientLoader>();
 
-  const testServer = useSubscribeToTestServer();
+  const testServer = useSubscribeToTestServer(data.serverState);
 
   return (
     <TestServerStateContext.Provider value={testServer}>

apps/evalite-ui/app/use-subscribe-to-socket.ts

@@ -1,5 +1,6 @@
 import type { Evalite } from "@evalite/core";
 import { DEFAULT_SERVER_PORT } from "@evalite/core/constants";
+import type { GetServerStateResult } from "@evalite/core/sdk";
 import { useNavigate } from "@remix-run/react";
 import { createContext, useEffect, useMemo, useState } from "react";
 
@@ -16,8 +17,23 @@ export type TestServerState =
       type: "idle";
     };
 
-export const useSubscribeToTestServer = () => {
-  const [state, setState] = useState<TestServerState>({ type: "idle" });
+const serverStateToInitialState = (
+  serverState: GetServerStateResult
+): TestServerState => {
+  if (serverState.type === "idle") {
+    return { type: "idle" };
+  }
+
+  return {
+    type: "running",
+    filepaths: new Set(serverState.filepaths),
+  };
+};
+
+export const useSubscribeToTestServer = (serverState: GetServerStateResult) => {
+  const [state, setState] = useState<TestServerState>(
+    serverStateToInitialState(serverState)
+  );
 
   const navigate = useNavigate();
 

packages/evalite-core/src/sdk.ts

@@ -2,6 +2,20 @@ import type { Db } from "./db.js";
 
 const BASE_URL = "http://localhost:3006";
 
+export type GetServerStateResult =
+  | {
+      type: "running";
+      filepaths: string[];
+    }
+  | {
+      type: "idle";
+    };
+
+export const getServerState = async (): Promise<GetServerStateResult> => {
+  const res = await fetch(`${BASE_URL}/api/server-state`);
+  return res.json() as any;
+};
+
 export type GetMenuItemsResultEval = {
   filepath: string;
   score: number;

packages/evalite-core/src/server.ts

@@ -2,12 +2,13 @@ import { fastifyStatic } from "@fastify/static";
 import { fastifyWebsocket } from "@fastify/websocket";
 import fastify from "fastify";
 import path from "path";
+import { fileURLToPath } from "url";
 import {
   getAverageScoresFromResults,
+  getEvalByName,
   getEvals,
   getEvalsAverageScores,
   getHistoricalEvalsWithScoresByName,
-  getEvalByName,
   getMostRecentRun,
   getPreviousEval,
   getResults,
@@ -20,13 +21,44 @@ import {
   type GetMenuItemsResult,
   type GetMenuItemsResultEval,
   type GetResultResult,
+  type GetServerStateResult,
 } from "./sdk.js";
 import type { Evalite } from "./types.js";
-import { fileURLToPath } from "url";
 import { average } from "./utils.js";
 
 export type Server = ReturnType<typeof createServer>;
 
+export const handleWebsockets = (server: fastify.FastifyInstance) => {
+  const websocketListeners = new Map<
+    string,
+    (event: Evalite.WebsocketEvent) => void
+  >();
+
+  let lastEventReceived: Evalite.WebsocketEvent | undefined = undefined;
+
+  server.register(async (fastify) => {
+    fastify.get("/api/socket", { websocket: true }, (socket, req) => {
+      websocketListeners.set(req.id, (event) => {
+        socket.send(JSON.stringify(event));
+      });
+
+      socket.on("close", () => {
+        websocketListeners.delete(req.id);
+      });
+    });
+  });
+
+  return {
+    send: (event: Evalite.WebsocketEvent) => {
+      lastEventReceived = event;
+      for (const listener of websocketListeners.values()) {
+        listener(event);
+      }
+    },
+    getLastEventReceived: () => lastEventReceived,
+  };
+};
+
 export const createServer = (opts: { db: SQLiteDatabase }) => {
   const UI_ROOT = path.join(
     path.dirname(fileURLToPath(import.meta.url)),
@@ -49,17 +81,21 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
     done(null, payload);
   });
 
-  const listeners = new Map<string, (event: Evalite.WebsocketEvent) => void>();
+  const websockets = handleWebsockets(server);
 
-  server.register(async (fastify) => {
-    fastify.get("/api/socket", { websocket: true }, (socket, req) => {
-      listeners.set(req.id, (event) => {
-        socket.send(JSON.stringify(event));
+  server.get<{
+    Reply: GetServerStateResult;
+  }>("/api/server-state", async (req, reply) => {
+    const rawWebsocketEvent = websockets.getLastEventReceived();
+    if (!rawWebsocketEvent || rawWebsocketEvent.type !== "RUN_IN_PROGRESS") {
+      return reply.code(200).send({
+        type: "idle",
       });
+    }
 
-      socket.on("close", () => {
-        listeners.delete(req.id);
-      });
+    return reply.code(200).send({
+      type: "running",
+      filepaths: rawWebsocketEvent.filepaths,
     });
   });
 
@@ -351,11 +387,7 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
   });
 
   return {
-    send: (event: Evalite.WebsocketEvent) => {
-      for (const listener of listeners.values()) {
-        listener(event);
-      }
-    },
+    send: websockets.send,
     start: (port: number) => {
       server.listen(
         {

packages/evalite/src/run-vitest.ts

@@ -60,6 +60,13 @@ export const runVitest = async (opts: {
     }
   );
 
+  /**
+   * This is important to run before start, so that
+   * we immediately report the correct files to the
+   * server.
+   */
+  await vitest.collect(filters);
+
   await vitest.start(filters);
 
   const dispose = registerConsoleShortcuts(

packages/example/src/long.eval.ts

@@ -0,0 +1,17 @@
+import { Factuality, Levenshtein } from "autoevals";
+import { evalite } from "evalite";
+import { setTimeout } from "timers/promises";
+
+evalite("Long", {
+  data: async () => [
+    {
+      input: `What's the capital of France?`,
+      expected: `Paris`,
+    },
+  ],
+  task: async (input) => {
+    await setTimeout(6000);
+    return "Paris";
+  },
+  scorers: [Factuality, Levenshtein],
+});