diff --git a/apps/evalite-ui/app/root.tsx b/apps/evalite-ui/app/root.tsx
index 9e23d66..af302eb 100644
--- a/apps/evalite-ui/app/root.tsx
+++ b/apps/evalite-ui/app/root.tsx
@@ -183,7 +183,7 @@ const SidebarItem = (props: {
   const testServer = useContext(TestServerStateContext);
 
   if (testServer.state.type === "running") {
-    isRunning = testServer.state.filepaths.has(props.filepath);
+    isRunning = testServer.isRunningFilepath(props.filepath);
   }
   return (
     <SidebarMenuItem key={props.name}>
diff --git a/apps/evalite-ui/app/routes/eval.$name.trace.$index.tsx b/apps/evalite-ui/app/routes/eval.$name.trace.$index.tsx
index 685fc84..acfb491 100644
--- a/apps/evalite-ui/app/routes/eval.$name.trace.$index.tsx
+++ b/apps/evalite-ui/app/routes/eval.$name.trace.$index.tsx
@@ -71,9 +71,7 @@ export default function Page() {
 
   const serverState = useContext(TestServerStateContext);
 
-  const isRunning =
-    serverState.state.type === "running" &&
-    serverState.state.filepaths.has(evaluation.filepath);
+  const isRunning = serverState.isRunningFilepath(evaluation.filepath);
 
   const [searchParams] = useSearchParams();
 
diff --git a/apps/evalite-ui/app/routes/eval.$name.tsx b/apps/evalite-ui/app/routes/eval.$name.tsx
index 5b29d6b..7c90072 100644
--- a/apps/evalite-ui/app/routes/eval.$name.tsx
+++ b/apps/evalite-ui/app/routes/eval.$name.tsx
@@ -93,8 +93,7 @@ export default function Page() {
             <Score
               evalStatus={evaluation.status}
               isRunning={
-                serverState.state.type === "running" &&
-                serverState.state.filepaths.has(evaluation.filepath) &&
+                serverState.isRunningFilepath(evaluation.filepath) &&
                 evaluation.created_at === latestDate
               }
               score={score}
@@ -230,12 +229,9 @@ export default function Page() {
                             <Wrapper>
                               <Score
                                 score={scorer.score}
-                                isRunning={
-                                  serverState.state.type === "running" &&
-                                  serverState.state.filepaths.has(
-                                    evaluation.filepath
-                                  )
-                                }
+                                isRunning={serverState.isRunningFilepath(
+                                  evaluation.filepath
+                                )}
                                 evalStatus={evaluation.status}
                                 state={getScoreState(
                                   scorer.score,
diff --git a/apps/evalite-ui/app/use-subscribe-to-socket.ts b/apps/evalite-ui/app/use-subscribe-to-socket.ts
index acb36e1..b376a03 100644
--- a/apps/evalite-ui/app/use-subscribe-to-socket.ts
+++ b/apps/evalite-ui/app/use-subscribe-to-socket.ts
@@ -1,6 +1,5 @@
 import type { Evalite } from "@evalite/core";
 import { DEFAULT_SERVER_PORT } from "@evalite/core/constants";
-import type { GetServerStateResult } from "@evalite/core/sdk";
 import { useNavigate } from "@remix-run/react";
 import { createContext, useEffect, useMemo, useState } from "react";
 
@@ -8,36 +7,13 @@ export const TestServerStateContext = createContext<
   ReturnType<typeof useSubscribeToTestServer>
 >({} as never);
 
-export type TestServerState =
-  | {
-      type: "running";
-      filepaths: Set<string>;
-    }
-  | {
-      type: "idle";
-    };
-
-const serverStateToInitialState = (
-  serverState: GetServerStateResult
-): TestServerState => {
-  if (serverState.type === "idle") {
-    return { type: "idle" };
-  }
-
-  return {
-    type: "running",
-    filepaths: new Set(serverState.filepaths),
-  };
-};
-
-export const useSubscribeToTestServer = (serverState: GetServerStateResult) => {
-  const [state, setState] = useState<TestServerState>(
-    serverStateToInitialState(serverState)
-  );
+export const useSubscribeToTestServer = (serverState: Evalite.ServerState) => {
+  const [state, setState] = useState<Evalite.ServerState>(serverState);
 
   const navigate = useNavigate();
 
   useEffect(() => {
+    // Reload fetchers when the server state changes
     navigate(window.location, {
       preventScrollReset: true,
       replace: true,
@@ -50,18 +26,8 @@ export const useSubscribeToTestServer = (serverState: GetServerStateResult) => {
     );
 
     socket.onmessage = (event) => {
-      const data: Evalite.WebsocketEvent = JSON.parse(event.data);
-      switch (data.type) {
-        case "RUN_IN_PROGRESS":
-          setState({
-            type: "running",
-            filepaths: new Set(data.filepaths),
-          });
-          break;
-        case "RUN_COMPLETE":
-          setState({ type: "idle" });
-          break;
-      }
+      const newState: Evalite.ServerState = JSON.parse(event.data);
+      setState(newState);
     };
 
     return () => {
@@ -69,5 +35,15 @@ export const useSubscribeToTestServer = (serverState: GetServerStateResult) => {
     };
   }, []);
 
-  return useMemo(() => ({ state }), [state]);
+  return useMemo(() => {
+    const filePathSet: Set<string> =
+      state.type === "running" ? new Set(state.filepaths) : new Set();
+
+    const isRunningFilepath = (filepath: string) =>
+      filePathSet.has(filepath) && state.type === "running";
+    return {
+      state,
+      isRunningFilepath,
+    };
+  }, [state]);
 };
diff --git a/packages/evalite-core/src/sdk.ts b/packages/evalite-core/src/sdk.ts
index 29838dd..4fb558e 100644
--- a/packages/evalite-core/src/sdk.ts
+++ b/packages/evalite-core/src/sdk.ts
@@ -1,17 +1,9 @@
 import type { Db } from "./db.js";
+import type { Evalite } from "./types.js";
 
 const BASE_URL = "http://localhost:3006";
 
-export type GetServerStateResult =
-  | {
-      type: "running";
-      filepaths: string[];
-    }
-  | {
-      type: "idle";
-    };
-
-export const getServerState = async (): Promise<GetServerStateResult> => {
+export const getServerState = async (): Promise<Evalite.ServerState> => {
   const res = await fetch(`${BASE_URL}/api/server-state`);
   return res.json() as any;
 };
diff --git a/packages/evalite-core/src/server.ts b/packages/evalite-core/src/server.ts
index 5e5f48c..4a29554 100644
--- a/packages/evalite-core/src/server.ts
+++ b/packages/evalite-core/src/server.ts
@@ -21,7 +21,6 @@ import {
   type GetMenuItemsResult,
   type GetMenuItemsResultEval,
   type GetResultResult,
-  type GetServerStateResult,
 } from "./sdk.js";
 import type { Evalite } from "./types.js";
 import { average } from "./utils.js";
@@ -31,10 +30,12 @@ export type Server = ReturnType<typeof createServer>;
 export const handleWebsockets = (server: fastify.FastifyInstance) => {
   const websocketListeners = new Map<
     string,
-    (event: Evalite.WebsocketEvent) => void
+    (event: Evalite.ServerState) => void
   >();
 
-  let lastEventReceived: Evalite.WebsocketEvent | undefined = undefined;
+  let currentState: Evalite.ServerState = {
+    type: "idle",
+  };
 
   server.register(async (fastify) => {
     fastify.get("/api/socket", { websocket: true }, (socket, req) => {
@@ -49,13 +50,13 @@ export const handleWebsockets = (server: fastify.FastifyInstance) => {
   });
 
   return {
-    send: (event: Evalite.WebsocketEvent) => {
-      lastEventReceived = event;
+    updateState: (newState: Evalite.ServerState) => {
+      currentState = newState;
       for (const listener of websocketListeners.values()) {
-        listener(event);
+        listener(newState);
       }
     },
-    getLastEventReceived: () => lastEventReceived,
+    getState: () => currentState,
   };
 };
 
@@ -84,19 +85,9 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
   const websockets = handleWebsockets(server);
 
   server.get<{
-    Reply: GetServerStateResult;
+    Reply: Evalite.ServerState;
   }>("/api/server-state", async (req, reply) => {
-    const rawWebsocketEvent = websockets.getLastEventReceived();
-    if (!rawWebsocketEvent || rawWebsocketEvent.type !== "RUN_IN_PROGRESS") {
-      return reply.code(200).send({
-        type: "idle",
-      });
-    }
-
-    return reply.code(200).send({
-      type: "running",
-      filepaths: rawWebsocketEvent.filepaths,
-    });
+    return reply.code(200).send(websockets.getState());
   });
 
   server.get<{
@@ -387,7 +378,7 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
   });
 
   return {
-    send: websockets.send,
+    updateState: websockets.updateState,
     start: (port: number) => {
       server.listen(
         {
diff --git a/packages/evalite-core/src/types.ts b/packages/evalite-core/src/types.ts
index 8ea0652..8f92ef7 100644
--- a/packages/evalite-core/src/types.ts
+++ b/packages/evalite-core/src/types.ts
@@ -1,13 +1,13 @@
 export declare namespace Evalite {
   export type RunType = "full" | "partial";
-  export type WebsocketEvent =
+  export type ServerState =
     | {
-        type: "RUN_IN_PROGRESS";
+        type: "running";
         runType: RunType;
         filepaths: string[];
       }
     | {
-        type: "RUN_COMPLETE";
+        type: "idle";
       };
 
   export type MaybePromise<T> = T | Promise<T>;
diff --git a/packages/evalite/src/reporter.ts b/packages/evalite/src/reporter.ts
index 468942c..8a9a50c 100644
--- a/packages/evalite/src/reporter.ts
+++ b/packages/evalite/src/reporter.ts
@@ -11,7 +11,7 @@ import { saveRun, type SQLiteDatabase } from "@evalite/core/db";
 export interface EvaliteReporterOptions {
   isWatching: boolean;
   port: number;
-  logEvent: (event: Evalite.WebsocketEvent) => void;
+  logNewState: (event: Evalite.ServerState) => void;
   db: SQLiteDatabase;
 }
 
@@ -31,9 +31,20 @@ const renderers = {
   },
 };
 
+type ReporterEvent =
+  | {
+      type: "RUN_BEGUN";
+      filepaths: string[];
+      runType: Evalite.RunType;
+    }
+  | {
+      type: "RUN_ENDED";
+    };
+
 export default class EvaliteReporter extends BasicReporter {
   private opts: EvaliteReporterOptions;
   private lastRunTypeLogged: Evalite.RunType = "full";
+  private state: Evalite.ServerState = { type: "idle" };
 
   // private server: Server;
   constructor(opts: EvaliteReporterOptions) {
@@ -50,8 +61,8 @@ export default class EvaliteReporter extends BasicReporter {
     );
     this.ctx.logger.log("");
 
-    this.opts.logEvent({
-      type: "RUN_IN_PROGRESS",
+    this.sendEvent({
+      type: "RUN_BEGUN",
       filepaths: this.ctx.state.getFiles().map((f) => f.filepath),
       runType: "full",
     });
@@ -62,9 +73,35 @@ export default class EvaliteReporter extends BasicReporter {
     super.onWatcherStart(files, errors);
   }
 
+  updateState(state: Evalite.ServerState) {
+    this.state = state;
+    this.opts.logNewState(state);
+  }
+
+  /**
+   * Handles the state management for the reporter
+   */
+  sendEvent(event: ReporterEvent): void {
+    switch (event.type) {
+      case "RUN_BEGUN":
+        this.updateState({
+          filepaths: event.filepaths,
+          runType: event.runType,
+          type: "running",
+        });
+        break;
+      case "RUN_ENDED":
+        this.updateState({ type: "idle" });
+        break;
+      default:
+        event satisfies never;
+        throw new Error("Unknown event type");
+    }
+  }
+
   override onWatcherRerun(files: string[], trigger?: string): void {
-    this.opts.logEvent({
-      type: "RUN_IN_PROGRESS",
+    this.sendEvent({
+      type: "RUN_BEGUN",
       filepaths: files,
       runType: "partial",
     });
@@ -76,8 +113,8 @@ export default class EvaliteReporter extends BasicReporter {
     files = this.ctx.state.getFiles(),
     errors = this.ctx.state.getUnhandledErrors()
   ) => {
-    this.opts.logEvent({
-      type: "RUN_COMPLETE",
+    this.sendEvent({
+      type: "RUN_ENDED",
     });
 
     saveRun(this.opts.db, { files, runType: this.lastRunTypeLogged });
diff --git a/packages/evalite/src/run-vitest.ts b/packages/evalite/src/run-vitest.ts
index c72d2a1..eea59e0 100644
--- a/packages/evalite/src/run-vitest.ts
+++ b/packages/evalite/src/run-vitest.ts
@@ -43,8 +43,8 @@ export const runVitest = async (opts: {
       },
       reporters: [
         new EvaliteReporter({
-          logEvent: (event) => {
-            server?.send(event);
+          logNewState: (newState) => {
+            server?.updateState(newState);
           },
           port: DEFAULT_SERVER_PORT,
           isWatching: opts.mode === "watch-for-file-changes",