Skip to content

Commit

Permalink
Began adding historical data back in
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattpocock
mattpocock committed Dec 10, 2024
1 parent 149b6c1 commit 1abcb45
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 90 deletions.
22 changes: 18 additions & 4 deletions apps/evalite-ui/app/components/ui/line-chart.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"use client";

import { formatDistance } from "date-fns";
import { Area, AreaChart, XAxis } from "recharts";

import {
Expand All @@ -8,6 +9,7 @@ import {
ChartTooltip,
ChartTooltipContent,
} from "~/components/ui/chart";
import { LiveDate } from "./live-date";

const chartConfig = {
score: {
Expand All @@ -16,13 +18,21 @@ const chartConfig = {
},
} satisfies ChartConfig;

export function MyLineChart(props: { data: { score: number }[] }) {
export function MyLineChart(props: {
data: { date: string; score: number }[];
}) {
return (
<ChartContainer
config={chartConfig}
className="h-24 overflow-visible max-w-[120ch]"
className="h-24 overflow-visible max-w-[120ch] -mb-6 w-full"
>
<AreaChart accessibilityLayer data={props.data}>
<AreaChart
accessibilityLayer
data={props.data.map((s) => ({
...s,
score: Math.round(s.score * 100),
}))}
>
<XAxis
dataKey="month"
tickLine={false}
Expand All @@ -32,7 +42,11 @@ export function MyLineChart(props: { data: { score: number }[] }) {
/>
<ChartTooltip
cursor={false}
content={<ChartTooltipContent hideLabel />}
content={
<ChartTooltipContent
labelFormatter={(l, p) => <LiveDate date={p[0]?.payload?.date} />}
/>
}
/>
<Area
isAnimationActive={false}
Expand Down
15 changes: 15 additions & 0 deletions apps/evalite-ui/app/components/ui/live-date.tsx
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { formatDistance } from "date-fns";
import { useEffect, useState } from "react";

export const LiveDate = (props: { date: string }) => {
const [, setNow] = useState(new Date());
useEffect(() => {
const interval = setInterval(() => {
setNow(new Date());
}, 1000);
return () => clearInterval(interval);
}, []);
return (
<span>{formatDistance(props.date, new Date(), { addSuffix: true })}</span>
);
};
189 changes: 104 additions & 85 deletions apps/evalite-ui/app/routes/eval.$name.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import React, { useContext } from "react";
import { DisplayInput } from "~/components/display-input";
import { InnerPageLayout } from "~/components/page-layout";
import { getScoreState, Score } from "~/components/score";
import { MyLineChart } from "~/components/ui/line-chart";
import { Separator } from "~/components/ui/separator";
import {
Table,
TableBody,
Expand Down Expand Up @@ -58,7 +60,6 @@ export default function Page() {
vscodeUrl={`vscode://file${evaluation.filepath}`}
filepath={evaluation.filepath.split(/(\/|\\)/).slice(-1)[0]!}
>
{/* {history.length > 1 && <MyLineChart data={history} />} */}
{evaluation.status === "fail" && (
<div className="flex gap-4 px-4">
<div className="flex-shrink-0">
Expand All @@ -74,97 +75,115 @@ export default function Page() {
</div>
)}
{evaluation.status === "success" && (
<Table>
<TableHeader>
<TableRow>
<TableHead>Input</TableHead>
<TableHead>Output</TableHead>
{showExpectedColumn && <TableHead>Expected</TableHead>}
{firstResult?.scores.map((scorer, index) => (
<TableHead
key={scorer.name}
className={cn(index === 0 && "border-l")}
>
{scorer.name}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{evaluation.results.map((result, index) => {
const Wrapper = (props: { children: React.ReactNode }) => (
<NavLink
prefetch="intent"
to={`trace/${index}`}
preventScrollReset
className={({ isActive }) => {
return cn("block h-full p-4", isActive && "active");
}}
>
{props.children}
</NavLink>
);
return (
<TableRow
key={JSON.stringify(result.input)}
className="has-[.active]:bg-gray-100"
>
<td className="align-top">
<DisplayInput
input={result.input}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
<td className="align-top">
<DisplayInput
input={result.output}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
{showExpectedColumn && (
<div className="">
{history.length > 1 && (
<>
<h2 className="mb-4 font-medium text-lg text-gray-600">
History
</h2>
{history.length > 1 && <MyLineChart data={history} />}
<Separator orientation="horizontal" className="my-8" />
<h2 className="mb-4 font-medium text-lg text-gray-600">
Results
</h2>
</>
)}
<Table>
<TableHeader>
<TableRow>
<TableHead>Input</TableHead>
<TableHead>Output</TableHead>
{showExpectedColumn && <TableHead>Expected</TableHead>}
{firstResult?.scores.map((scorer, index) => (
<TableHead
key={scorer.name}
className={cn(index === 0 && "border-l")}
>
{scorer.name}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{evaluation.results.map((result, index) => {
const Wrapper = (props: { children: React.ReactNode }) => (
<NavLink
prefetch="intent"
to={`trace/${index}`}
preventScrollReset
className={({ isActive }) => {
return cn("block h-full p-4", isActive && "active");
}}
>
{props.children}
</NavLink>
);
return (
<TableRow
key={JSON.stringify(result.input)}
className="has-[.active]:bg-gray-100"
>
<td className="align-top">
<DisplayInput
input={result.expected}
input={result.input}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
)}
{result.scores.map((scorer, index) => {
const scoreInPreviousEvaluation = prevEvaluation?.results
.find((r) => r.input === result.input)
?.scores.find((s) => s.name === scorer.name);
return (
<td
key={scorer.name}
className={cn(index === 0 && "border-l", "align-top")}
>
<Wrapper>
<Score
score={scorer.score}
isRunning={
serverState.state.type === "running" &&
serverState.state.filepaths.has(
evaluation.filepath
)
}
evalStatus={evaluation.status}
state={getScoreState(
scorer.score,
scoreInPreviousEvaluation?.score
)}
/>
</Wrapper>
<td className="align-top">
<DisplayInput
input={result.output}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
{showExpectedColumn && (
<td className="align-top">
<DisplayInput
input={result.expected}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
);
})}
</TableRow>
);
})}
</TableBody>
</Table>
)}
{result.scores.map((scorer, index) => {
const scoreInPreviousEvaluation =
prevEvaluation?.results
.find((r) => r.input === result.input)
?.scores.find((s) => s.name === scorer.name);
return (
<td
key={scorer.name}
className={cn(
index === 0 && "border-l",
"align-top"
)}
>
<Wrapper>
<Score
score={scorer.score}
isRunning={
serverState.state.type === "running" &&
serverState.state.filepaths.has(
evaluation.filepath
)
}
evalStatus={evaluation.status}
state={getScoreState(
scorer.score,
scoreInPreviousEvaluation?.score
)}
/>
</Wrapper>
</td>
);
})}
</TableRow>
);
})}
</TableBody>
</Table>
</div>
)}
</InnerPageLayout>
<div
Expand Down
19 changes: 19 additions & 0 deletions packages/evalite-core/src/db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,3 +456,22 @@ export const getMostRecentEvalByName = (
)
.get({ name });
};

export const getHistoricalEvalsWithScoresByName = (
db: BetterSqlite3.Database,
name: string
): (Db.Eval & { average_score: number })[] => {
return db
.prepare<{ name: string }, Db.Eval & { average_score: number }>(
`
SELECT evals.*, AVG(scores.score) as average_score
FROM evals
LEFT JOIN results ON evals.id = results.eval_id
LEFT JOIN scores ON results.id = scores.result_id
WHERE evals.name = @name
GROUP BY evals.id
ORDER BY evals.created_at ASC
`
)
.all({ name });
};
11 changes: 10 additions & 1 deletion packages/evalite-core/src/server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
getAverageScoresFromResults,
getEvals,
getEvalsAverageScores,
getHistoricalEvalsWithScoresByName,
getMostRecentEvalByName,
getMostRecentRun,
getPreviousEvalRun,
Expand Down Expand Up @@ -176,6 +177,7 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
server.route<{
Querystring: {
name: string;
timestamp?: string;
};
Reply: GetEvalByNameResult;
}>({
Expand All @@ -186,7 +188,9 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
type: "object",
properties: {
name: { type: "string" },
timestamp: { type: "string" },
},
required: ["name"],
},
},
handler: async (req, res) => {
Expand Down Expand Up @@ -214,8 +218,13 @@ export const createServer = (opts: { db: SQLiteDatabase }) => {
results.map((r) => r.id)
);

const history = getHistoricalEvalsWithScoresByName(opts.db, name);

return res.code(200).send({
history: [], // TODO when we enable chart
history: history.map((h) => ({
score: h.average_score,
date: h.created_at,
})),
evaluation: {
...evaluation,
results: results
Expand Down

0 comments on commit 1abcb45

Please sign in to comment.