From e3f64cfb96b7dce7ff765e5ab780db104c497ab3 Mon Sep 17 00:00:00 2001 From: Matt Pocock Date: Tue, 10 Dec 2024 15:47:53 +0000 Subject: [PATCH] Fixed a bug where the loading indicators were not accurate on first load. --- .changeset/good-games-fly.md | 6 ++ apps/evalite-ui/app/root.tsx | 11 ++-- .../evalite-ui/app/use-subscribe-to-socket.ts | 20 +++++- packages/evalite-core/src/sdk.ts | 14 +++++ packages/evalite-core/src/server.ts | 62 ++++++++++++++----- packages/evalite/src/run-vitest.ts | 7 +++ packages/example/src/long.eval.ts | 17 +++++ 7 files changed, 116 insertions(+), 21 deletions(-) create mode 100644 .changeset/good-games-fly.md create mode 100644 packages/example/src/long.eval.ts diff --git a/.changeset/good-games-fly.md b/.changeset/good-games-fly.md new file mode 100644 index 0000000..5845499 --- /dev/null +++ b/.changeset/good-games-fly.md @@ -0,0 +1,6 @@ +--- +"@evalite/core": patch +"evalite": patch +--- + +Fixed a bug where the loading indicators were not accurate on first load. diff --git a/apps/evalite-ui/app/root.tsx b/apps/evalite-ui/app/root.tsx index 8a793db..9e23d66 100644 --- a/apps/evalite-ui/app/root.tsx +++ b/apps/evalite-ui/app/root.tsx @@ -20,7 +20,7 @@ import { SidebarProvider, } from "~/components/ui/sidebar"; -import { getMenuItems } from "@evalite/core/sdk"; +import { getMenuItems, getServerState } from "@evalite/core/sdk"; import { getScoreState, Score, type ScoreState } from "./components/score"; import { cn } from "./lib/utils"; import "./tailwind.css"; @@ -65,10 +65,13 @@ export function Layout({ children }: { children: React.ReactNode }) { } export const clientLoader = async () => { - const { archivedEvals, currentEvals, prevScore, score, evalStatus } = - await getMenuItems(); + const [ + { archivedEvals, currentEvals, prevScore, score, evalStatus }, + serverState, + ] = await Promise.all([getMenuItems(), getServerState()]); return { + serverState, evalStatus, prevScore, score, @@ -92,7 +95,7 @@ export const clientLoader = async () => { export default function App() { const data = useLoaderData(); - const testServer = useSubscribeToTestServer(); + const testServer = useSubscribeToTestServer(data.serverState); return ( diff --git a/apps/evalite-ui/app/use-subscribe-to-socket.ts b/apps/evalite-ui/app/use-subscribe-to-socket.ts index c8a15dd..acb36e1 100644 --- a/apps/evalite-ui/app/use-subscribe-to-socket.ts +++ b/apps/evalite-ui/app/use-subscribe-to-socket.ts @@ -1,5 +1,6 @@ import type { Evalite } from "@evalite/core"; import { DEFAULT_SERVER_PORT } from "@evalite/core/constants"; +import type { GetServerStateResult } from "@evalite/core/sdk"; import { useNavigate } from "@remix-run/react"; import { createContext, useEffect, useMemo, useState } from "react"; @@ -16,8 +17,23 @@ export type TestServerState = type: "idle"; }; -export const useSubscribeToTestServer = () => { - const [state, setState] = useState({ type: "idle" }); +const serverStateToInitialState = ( + serverState: GetServerStateResult +): TestServerState => { + if (serverState.type === "idle") { + return { type: "idle" }; + } + + return { + type: "running", + filepaths: new Set(serverState.filepaths), + }; +}; + +export const useSubscribeToTestServer = (serverState: GetServerStateResult) => { + const [state, setState] = useState( + serverStateToInitialState(serverState) + ); const navigate = useNavigate(); diff --git a/packages/evalite-core/src/sdk.ts b/packages/evalite-core/src/sdk.ts index 01cc170..29838dd 100644 --- a/packages/evalite-core/src/sdk.ts +++ b/packages/evalite-core/src/sdk.ts @@ -2,6 +2,20 @@ import type { Db } from "./db.js"; const BASE_URL = "http://localhost:3006"; +export type GetServerStateResult = + | { + type: "running"; + filepaths: string[]; + } + | { + type: "idle"; + }; + +export const getServerState = async (): Promise => { + const res = await fetch(`${BASE_URL}/api/server-state`); + return res.json() as any; +}; + export type GetMenuItemsResultEval = { filepath: string; score: number; diff --git a/packages/evalite-core/src/server.ts b/packages/evalite-core/src/server.ts index c22ecfb..5e5f48c 100644 --- a/packages/evalite-core/src/server.ts +++ b/packages/evalite-core/src/server.ts @@ -2,12 +2,13 @@ import { fastifyStatic } from "@fastify/static"; import { fastifyWebsocket } from "@fastify/websocket"; import fastify from "fastify"; import path from "path"; +import { fileURLToPath } from "url"; import { getAverageScoresFromResults, + getEvalByName, getEvals, getEvalsAverageScores, getHistoricalEvalsWithScoresByName, - getEvalByName, getMostRecentRun, getPreviousEval, getResults, @@ -20,13 +21,44 @@ import { type GetMenuItemsResult, type GetMenuItemsResultEval, type GetResultResult, + type GetServerStateResult, } from "./sdk.js"; import type { Evalite } from "./types.js"; -import { fileURLToPath } from "url"; import { average } from "./utils.js"; export type Server = ReturnType; +export const handleWebsockets = (server: fastify.FastifyInstance) => { + const websocketListeners = new Map< + string, + (event: Evalite.WebsocketEvent) => void + >(); + + let lastEventReceived: Evalite.WebsocketEvent | undefined = undefined; + + server.register(async (fastify) => { + fastify.get("/api/socket", { websocket: true }, (socket, req) => { + websocketListeners.set(req.id, (event) => { + socket.send(JSON.stringify(event)); + }); + + socket.on("close", () => { + websocketListeners.delete(req.id); + }); + }); + }); + + return { + send: (event: Evalite.WebsocketEvent) => { + lastEventReceived = event; + for (const listener of websocketListeners.values()) { + listener(event); + } + }, + getLastEventReceived: () => lastEventReceived, + }; +}; + export const createServer = (opts: { db: SQLiteDatabase }) => { const UI_ROOT = path.join( path.dirname(fileURLToPath(import.meta.url)), @@ -49,17 +81,21 @@ export const createServer = (opts: { db: SQLiteDatabase }) => { done(null, payload); }); - const listeners = new Map void>(); + const websockets = handleWebsockets(server); - server.register(async (fastify) => { - fastify.get("/api/socket", { websocket: true }, (socket, req) => { - listeners.set(req.id, (event) => { - socket.send(JSON.stringify(event)); + server.get<{ + Reply: GetServerStateResult; + }>("/api/server-state", async (req, reply) => { + const rawWebsocketEvent = websockets.getLastEventReceived(); + if (!rawWebsocketEvent || rawWebsocketEvent.type !== "RUN_IN_PROGRESS") { + return reply.code(200).send({ + type: "idle", }); + } - socket.on("close", () => { - listeners.delete(req.id); - }); + return reply.code(200).send({ + type: "running", + filepaths: rawWebsocketEvent.filepaths, }); }); @@ -351,11 +387,7 @@ export const createServer = (opts: { db: SQLiteDatabase }) => { }); return { - send: (event: Evalite.WebsocketEvent) => { - for (const listener of listeners.values()) { - listener(event); - } - }, + send: websockets.send, start: (port: number) => { server.listen( { diff --git a/packages/evalite/src/run-vitest.ts b/packages/evalite/src/run-vitest.ts index a3af9b4..c72d2a1 100644 --- a/packages/evalite/src/run-vitest.ts +++ b/packages/evalite/src/run-vitest.ts @@ -60,6 +60,13 @@ export const runVitest = async (opts: { } ); + /** + * This is important to run before start, so that + * we immediately report the correct files to the + * server. + */ + await vitest.collect(filters); + await vitest.start(filters); const dispose = registerConsoleShortcuts( diff --git a/packages/example/src/long.eval.ts b/packages/example/src/long.eval.ts new file mode 100644 index 0000000..90f8a02 --- /dev/null +++ b/packages/example/src/long.eval.ts @@ -0,0 +1,17 @@ +import { Factuality, Levenshtein } from "autoevals"; +import { evalite } from "evalite"; +import { setTimeout } from "timers/promises"; + +evalite("Long", { + data: async () => [ + { + input: `What's the capital of France?`, + expected: `Paris`, + }, + ], + task: async (input) => { + await setTimeout(6000); + return "Paris"; + }, + scorers: [Factuality, Levenshtein], +});