From 6c069039b5a0c414f2db6af155fc80a3d555340a Mon Sep 17 00:00:00 2001 From: Matt Pocock Date: Tue, 10 Dec 2024 12:37:15 +0000 Subject: [PATCH 1/2] Removed sourceCodeHash, no longer used --- packages/evalite-core/src/db/tests/db.test.ts | 1 - packages/evalite-core/src/types.ts | 1 - packages/evalite/src/command.ts | 6 ------ packages/evalite/src/index.ts | 3 --- packages/evalite/src/run-vitest.ts | 16 ---------------- 5 files changed, 27 deletions(-) diff --git a/packages/evalite-core/src/db/tests/db.test.ts b/packages/evalite-core/src/db/tests/db.test.ts index 5446736..dbcc9e2 100644 --- a/packages/evalite-core/src/db/tests/db.test.ts +++ b/packages/evalite-core/src/db/tests/db.test.ts @@ -23,7 +23,6 @@ describe("getEvalsAverageScores", () => { meta: { evalite: { duration: 100, - sourceCodeHash: "abc", results: [ { input: "input", diff --git a/packages/evalite-core/src/types.ts b/packages/evalite-core/src/types.ts index 39c90ac..8ea0652 100644 --- a/packages/evalite-core/src/types.ts +++ b/packages/evalite-core/src/types.ts @@ -48,7 +48,6 @@ export declare namespace Evalite { export type TaskMeta = { results: Result[]; duration: number | undefined; - sourceCodeHash: string; }; export type Task = ( diff --git a/packages/evalite/src/command.ts b/packages/evalite/src/command.ts index 8ec7d7c..8c91d51 100644 --- a/packages/evalite/src/command.ts +++ b/packages/evalite/src/command.ts @@ -2,12 +2,6 @@ import { Command } from "commander"; import { runVitest } from "./run-vitest.js"; import { createDatabase } from "@evalite/core/db"; -declare module "vitest" { - export interface ProvidedContext { - evaliteInputHash: string; - } -} - export const program = new Command(); program.description("Run evals once and exit").action(() => { diff --git a/packages/evalite/src/index.ts b/packages/evalite/src/index.ts index 6374778..075ec00 100644 --- a/packages/evalite/src/index.ts +++ b/packages/evalite/src/index.ts @@ -79,8 +79,6 @@ export const evalite = ( opts: Evalite.RunnerOpts ) => { return it(testName, async ({ task }) => { - const sourceCodeHash = inject("evaliteInputHash"); - const data = await opts.data(); const start = performance.now(); const results = await Promise.all( @@ -107,7 +105,6 @@ export const evalite = ( task.meta.evalite = { results, duration: Math.round(performance.now() - start), - sourceCodeHash, }; }); }; diff --git a/packages/evalite/src/run-vitest.ts b/packages/evalite/src/run-vitest.ts index 193d1ad..a3af9b4 100644 --- a/packages/evalite/src/run-vitest.ts +++ b/packages/evalite/src/run-vitest.ts @@ -60,22 +60,6 @@ export const runVitest = async (opts: { } ); - await vitest.collect(filters); - - const allFileResults = Array.from(vitest.vitenode.fetchCache); - - const sourceFileResults = allFileResults.filter(([path, item]) => { - return !(path.endsWith(".eval.ts") || path.endsWith(".eval.js")); - }); - - const codeFromSourceFiles = sourceFileResults.reduce((acc, [path, item]) => { - return acc + (item.result.code ?? ""); - }, ""); - - const hash = createHash("sha256").update(codeFromSourceFiles).digest("hex"); - - vitest.provide("evaliteInputHash", hash); - await vitest.start(filters); const dispose = registerConsoleShortcuts( From babe875e8b419a70a6e694edf7a960dae4b69160 Mon Sep 17 00:00:00 2001 From: Matt Pocock Date: Tue, 10 Dec 2024 12:55:34 +0000 Subject: [PATCH 2/2] Improved eval page layout, adding a title and a summary at the top --- .../evalite-ui/app/components/page-layout.tsx | 10 +---- apps/evalite-ui/app/components/score.tsx | 4 +- .../app/components/ui/live-date.tsx | 4 +- apps/evalite-ui/app/routes/eval.$name.tsx | 44 ++++++++++++++++--- packages/evalite/src/command.ts | 1 - 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/apps/evalite-ui/app/components/page-layout.tsx b/apps/evalite-ui/app/components/page-layout.tsx index 66cf044..da049f2 100644 --- a/apps/evalite-ui/app/components/page-layout.tsx +++ b/apps/evalite-ui/app/components/page-layout.tsx @@ -10,11 +10,9 @@ import { SidebarTrigger } from "./ui/sidebar"; export const InnerPageLayout = ({ children, - title, filepath, vscodeUrl, }: { - title: string; vscodeUrl: string; filepath: string; children: React.ReactNode; @@ -27,12 +25,6 @@ export const InnerPageLayout = ({ - - - {title} - - - {filepath} @@ -42,7 +34,7 @@ export const InnerPageLayout = ({ -
{children}
+
{children}
); }; diff --git a/apps/evalite-ui/app/components/score.tsx b/apps/evalite-ui/app/components/score.tsx index 53ae5d6..c2bf910 100644 --- a/apps/evalite-ui/app/components/score.tsx +++ b/apps/evalite-ui/app/components/score.tsx @@ -18,7 +18,9 @@ export const Score = (props: { }) => { return ( - {Math.round(props.score * 100)}% + + {Math.round((Number.isNaN(props.score) ? 0 : props.score) * 100)}% + {(() => { switch (true) { case props.isRunning: diff --git a/apps/evalite-ui/app/components/ui/live-date.tsx b/apps/evalite-ui/app/components/ui/live-date.tsx index c6d8459..cb46584 100644 --- a/apps/evalite-ui/app/components/ui/live-date.tsx +++ b/apps/evalite-ui/app/components/ui/live-date.tsx @@ -1,12 +1,14 @@ import { formatDistance } from "date-fns"; import { useEffect, useState } from "react"; +const ONE_MINUTE = 60_000; + export const LiveDate = (props: { date: string }) => { const [, setNow] = useState(new Date()); useEffect(() => { const interval = setInterval(() => { setNow(new Date()); - }, 1000); + }, ONE_MINUTE); return () => clearInterval(interval); }, []); return ( diff --git a/apps/evalite-ui/app/routes/eval.$name.tsx b/apps/evalite-ui/app/routes/eval.$name.tsx index 238516b..84bc6e5 100644 --- a/apps/evalite-ui/app/routes/eval.$name.tsx +++ b/apps/evalite-ui/app/routes/eval.$name.tsx @@ -1,4 +1,5 @@ import { getEvalByName } from "@evalite/core/sdk"; +import { average, sum } from "@evalite/core/utils"; import type { MetaFunction } from "@remix-run/node"; import { NavLink, @@ -12,6 +13,7 @@ import { DisplayInput } from "~/components/display-input"; import { InnerPageLayout } from "~/components/page-layout"; import { getScoreState, Score } from "~/components/score"; import { MyLineChart } from "~/components/ui/line-chart"; +import { LiveDate } from "~/components/ui/live-date"; import { Separator } from "~/components/ui/separator"; import { Table, @@ -22,6 +24,7 @@ import { } from "~/components/ui/table"; import { cn } from "~/lib/utils"; import { TestServerStateContext } from "~/use-subscribe-to-socket"; +import { formatTime } from "~/utils"; export const meta: MetaFunction = (args) => { return [ @@ -53,13 +56,43 @@ export default function Page() { const isTraceRoute = location.pathname.includes("trace"); + const score = average(evaluation.results, (r) => + average(r.scores, (s) => s.score) + ); + + const prevScore = prevEvaluation + ? average(prevEvaluation.results, (r) => average(r.scores, (s) => s.score)) + : undefined; + return ( <> +
+

+ {name} +

+
+ + + {formatTime(evaluation.duration)} + + + + +
+
+ {evaluation.status === "fail" && (
@@ -77,17 +110,14 @@ export default function Page() { {evaluation.status === "success" && (
{history.length > 1 && ( - <> +

History

{history.length > 1 && } - -

- Results -

- +
)} +

Results

diff --git a/packages/evalite/src/command.ts b/packages/evalite/src/command.ts index 8c91d51..dbfc143 100644 --- a/packages/evalite/src/command.ts +++ b/packages/evalite/src/command.ts @@ -1,6 +1,5 @@ import { Command } from "commander"; import { runVitest } from "./run-vitest.js"; -import { createDatabase } from "@evalite/core/db"; export const program = new Command();