Skip to content

Commit

Permalink
Merge pull request #45 from mattpocock/matt/failing-tests-status
Browse files Browse the repository at this point in the history 
Showed failing tests properly in the UI
  • Loading branch information
@mattpocock
mattpocock authored Dec 10, 2024
2 parents 19eb63d + ba8eaa4 commit 0981244
Show file tree
Hide file tree
Showing 9 changed files with 237 additions and 127 deletions.
80 changes: 47 additions & 33 deletions apps/evalite-ui/app/components/score.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
ChevronRightCircleIcon,
ChevronUpCircleIcon,
LoaderCircleIcon,
XCircleIcon,
} from "lucide-react";
import { cn } from "~/lib/utils";

Expand All @@ -12,44 +13,57 @@ export const Score = (props: {
score: number;
state: ScoreState;
isRunning: boolean;
evalStatus: "success" | "fail";
iconClassName?: string;
}) => {
return (
<span className="flex items-center space-x-2">
<span>{Math.round(props.score * 100)}%</span>
{props.isRunning ? (
<span className="text-gray-500">
<LoaderCircleIcon
className={cn(
"size-3 text-blue-500 animate-spin",
props.iconClassName
)}
/>
</span>
) : (
<>
{props.state === "up" && (
<ChevronUpCircleIcon
className={cn("size-3 text-green-600", props.iconClassName)}
/>
)}
{props.state === "down" && (
<ChevronDownCircleIcon
className={cn("size-3 text-red-600", props.iconClassName)}
/>
)}
{props.state === "same" && (
<ChevronRightCircleIcon
className={cn("size-3 text-blue-500", props.iconClassName)}
/>
)}
{props.state === "first" && (
<ChevronRightCircleIcon
className={cn("size-3 text-blue-500", props.iconClassName)}
/>
)}
</>
)}
{(() => {
switch (true) {
case props.isRunning:
return (
<LoaderCircleIcon
className={cn(
"size-3 text-blue-500 animate-spin",
props.iconClassName
)}
/>
);
case props.evalStatus === "fail":
return (
<XCircleIcon
className={cn("size-3 text-red-500", props.iconClassName)}
/>
);
case props.state === "up":
return (
<ChevronUpCircleIcon
className={cn("size-3 text-green-600", props.iconClassName)}
/>
);
case props.state === "down":
return (
<ChevronDownCircleIcon
className={cn("size-3 text-red-600", props.iconClassName)}
/>
);
case props.state === "same":
return (
<ChevronRightCircleIcon
className={cn("size-3 text-blue-500", props.iconClassName)}
/>
);
case props.state === "first":
return (
<ChevronRightCircleIcon
className={cn("size-3 text-blue-500", props.iconClassName)}
/>
);
default:
return null;
}
})()}
</span>
);
};
Expand Down
14 changes: 12 additions & 2 deletions apps/evalite-ui/app/root.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,11 @@ export function Layout({ children }: { children: React.ReactNode }) {
}

export const clientLoader = async () => {
const { archivedEvals, currentEvals, prevScore, score } =
const { archivedEvals, currentEvals, prevScore, score, evalStatus } =
await getMenuItems();

return {
evalStatus,
prevScore,
score,
archivedEvals: archivedEvals.map((e) => {
Expand Down Expand Up @@ -118,6 +119,7 @@ export default function App() {
score={data.score}
state={getScoreState(data.score, data.prevScore)}
iconClassName="size-4"
evalStatus={data.evalStatus}
/>
</div>
</div>
Expand All @@ -133,6 +135,7 @@ export default function App() {
name={e.name}
score={e.score}
state={e.state}
evalStatus={e.evalStatus}
/>
);
})}
Expand All @@ -150,6 +153,7 @@ export default function App() {
name={e.name}
score={e.score}
state={e.state}
evalStatus={e.evalStatus}
/>
);
})}
Expand All @@ -169,6 +173,7 @@ const SidebarItem = (props: {
name: string;
state: ScoreState;
score: number;
evalStatus: "success" | "fail";
}) => {
let isRunning = false;

Expand All @@ -191,7 +196,12 @@ const SidebarItem = (props: {
>
<span>{props.name}</span>

<Score score={props.score} state={props.state} isRunning={isRunning} />
<Score
score={props.score}
state={props.state}
isRunning={isRunning}
evalStatus={props.evalStatus}
/>
</NavLink>
</SidebarMenuItem>
);
Expand Down
4 changes: 3 additions & 1 deletion apps/evalite-ui/app/routes/eval.$name.trace.$index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ export const clientLoader = async (args: ClientLoaderFunctionArgs) => {

export default function Page() {
const {
data: { result, prevResult, filepath },
data: { result, prevResult, filepath, evalStatus },
name,
resultIndex,
} = useLoaderData<typeof clientLoader>();
Expand Down Expand Up @@ -108,6 +108,7 @@ export default function Page() {
isRunning={isRunning}
score={result.score}
state={getScoreState(result.score, prevResult?.score)}
evalStatus={evalStatus}
/>
</BreadcrumbItem>
<Separator orientation="vertical" className="mx-1 h-4" />
Expand Down Expand Up @@ -223,6 +224,7 @@ export default function Page() {
(prevScore) => prevScore.name === score.name
)?.score
)}
evalStatus={evalStatus}
/>
</MainBodySection>
{score.metadata && (
Expand Down
184 changes: 101 additions & 83 deletions apps/evalite-ui/app/routes/eval.$name.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
useLoaderData,
type ClientLoaderFunctionArgs,
} from "@remix-run/react";
import { XCircleIcon } from "lucide-react";
import React, { useContext } from "react";
import { DisplayInput } from "~/components/display-input";
import { InnerPageLayout } from "~/components/page-layout";
Expand Down Expand Up @@ -58,96 +59,113 @@ export default function Page() {
filepath={evaluation.filepath.split(/(\/|\\)/).slice(-1)[0]!}
>
{/* {history.length > 1 && <MyLineChart data={history} />} */}
<Table>
<TableHeader>
<TableRow>
<TableHead>Input</TableHead>
<TableHead>Output</TableHead>
{showExpectedColumn && <TableHead>Expected</TableHead>}
{firstResult?.scores.map((scorer, index) => (
<TableHead
key={scorer.name}
className={cn(index === 0 && "border-l")}
>
{scorer.name}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{evaluation.results.map((result, index) => {
const Wrapper = (props: { children: React.ReactNode }) => (
<NavLink
prefetch="intent"
to={`trace/${index}`}
preventScrollReset
className={({ isActive }) => {
return cn("block h-full p-4", isActive && "active");
}}
>
{props.children}
</NavLink>
);
return (
<TableRow
key={JSON.stringify(result.input)}
className="has-[.active]:bg-gray-100"
>
<td className="align-top">
<DisplayInput
input={result.input}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
<td className="align-top">
<DisplayInput
input={result.output}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
{showExpectedColumn && (
{evaluation.status === "fail" && (
<div className="flex gap-4 px-4">
<div className="flex-shrink-0">
<XCircleIcon className="text-red-500 size-7" />
</div>
<div className="text-sm text-gray-600 gap-1 flex flex-col">
<h3 className="font-semibold text-gray-700 mb-1 text-lg">
Evaluation Failed
</h3>
<p>This is likely because of an exception from your code.</p>
<p>Check the terminal for more information.</p>
</div>
</div>
)}
{evaluation.status === "success" && (
<Table>
<TableHeader>
<TableRow>
<TableHead>Input</TableHead>
<TableHead>Output</TableHead>
{showExpectedColumn && <TableHead>Expected</TableHead>}
{firstResult?.scores.map((scorer, index) => (
<TableHead
key={scorer.name}
className={cn(index === 0 && "border-l")}
>
{scorer.name}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{evaluation.results.map((result, index) => {
const Wrapper = (props: { children: React.ReactNode }) => (
<NavLink
prefetch="intent"
to={`trace/${index}`}
preventScrollReset
className={({ isActive }) => {
return cn("block h-full p-4", isActive && "active");
}}
>
{props.children}
</NavLink>
);
return (
<TableRow
key={JSON.stringify(result.input)}
className="has-[.active]:bg-gray-100"
>
<td className="align-top">
<DisplayInput
input={result.input}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
<td className="align-top">
<DisplayInput
input={result.expected}
input={result.output}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
)}
{result.scores.map((scorer, index) => {
const scoreInPreviousEvaluation = prevEvaluation?.results
.find((r) => r.input === result.input)
?.scores.find((s) => s.name === scorer.name);
return (
<td
key={scorer.name}
className={cn(index === 0 && "border-l", "align-top")}
>
<Wrapper>
<Score
score={scorer.score}
isRunning={
serverState.state.type === "running" &&
serverState.state.filepaths.has(
evaluation.filepath
)
}
state={getScoreState(
scorer.score,
scoreInPreviousEvaluation?.score
)}
/>
</Wrapper>
{showExpectedColumn && (
<td className="align-top">
<DisplayInput
input={result.expected}
shouldTruncateText
Wrapper={Wrapper}
/>
</td>
);
})}
</TableRow>
);
})}
</TableBody>
</Table>
)}
{result.scores.map((scorer, index) => {
const scoreInPreviousEvaluation = prevEvaluation?.results
.find((r) => r.input === result.input)
?.scores.find((s) => s.name === scorer.name);
return (
<td
key={scorer.name}
className={cn(index === 0 && "border-l", "align-top")}
>
<Wrapper>
<Score
score={scorer.score}
isRunning={
serverState.state.type === "running" &&
serverState.state.filepaths.has(
evaluation.filepath
)
}
evalStatus={evaluation.status}
state={getScoreState(
scorer.score,
scoreInPreviousEvaluation?.score
)}
/>
</Wrapper>
</td>
);
})}
</TableRow>
);
})}
</TableBody>
</Table>
)}
</InnerPageLayout>
<div
className={cn(
Expand Down
Loading

0 comments on commit 0981244

Please sign in to comment.