diff --git a/app/(protected)/project/[project_id]/annotations/annotations.tsx b/app/(protected)/project/[project_id]/annotations/annotations.tsx index 01ec912e..9716a3bf 100644 --- a/app/(protected)/project/[project_id]/annotations/annotations.tsx +++ b/app/(protected)/project/[project_id]/annotations/annotations.tsx @@ -1,6 +1,7 @@ "use client"; import { AnnotationsTable } from "@/components/annotations/annotations-table"; +import { ChartTabs } from "@/components/annotations/chart-tabs"; import { CreateTest } from "@/components/annotations/create-test"; import { EditTest } from "@/components/annotations/edit-test"; import { TableSkeleton } from "@/components/project/traces/table-skeleton"; @@ -11,13 +12,13 @@ import { Checkbox } from "@/components/ui/checkbox"; import { PAGE_SIZE } from "@/lib/constants"; import { LLMSpan, processLLMSpan } from "@/lib/llm_span_util"; import { correctTimestampFormat } from "@/lib/trace_utils"; -import { cn, formatDateTime } from "@/lib/utils"; +import { formatDateTime } from "@/lib/utils"; import { Skeleton } from "@mui/material"; import { Evaluation, Test } from "@prisma/client"; import { ColumnDef } from "@tanstack/react-table"; import { RabbitIcon } from "lucide-react"; import { useParams } from "next/navigation"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { useBottomScrollListener } from "react-bottom-scroll-listener"; import { useQuery } from "react-query"; import { toast } from "sonner"; @@ -32,11 +33,25 @@ export default function Annotations({ email }: { email: string }) { const projectId = useParams()?.project_id as string; const [currentData, setCurrentData] = useState([]); const [processedData, setProcessedData] = useState([]); + const [enableFetch, setEnableFetch] = useState(true); const [page, setPage] = useState(1); const [totalPages, setTotalPages] = useState(1); const [selectedData, setSelectedData] = useState([]); const [showBottomLoader, setShowBottomLoader] = useState(false); + useEffect(() => { + const handleFocusChange = () => { + setPage(1); + setEnableFetch(true); + }; + + window.addEventListener("focus", handleFocusChange); + + return () => { + window.removeEventListener("focus", handleFocusChange); + }; + }, []); + const scrollableDivRef = useBottomScrollListener(() => { if (fetchLlmPromptSpans.isRefetching) { return; @@ -109,11 +124,11 @@ export default function Annotations({ email }: { email: string }) { setPage(parseInt(metadata?.page) + 1); } - const updatedData = []; - if (currentData.length > 0) { - updatedData.push(...currentData, ...newData); + let updatedData = []; + if (page === 1) { + updatedData = [...newData]; } else { - updatedData.push(...newData); + updatedData = [...currentData, ...newData]; } // Remove duplicates const uniqueData = updatedData.filter( @@ -143,13 +158,17 @@ export default function Annotations({ email }: { email: string }) { setProcessedData(pData); setCurrentData(uniqueData); setShowBottomLoader(false); + setEnableFetch(false); }, onError: (error) => { setShowBottomLoader(false); + setEnableFetch(false); toast.error("Failed to fetch traces", { description: error instanceof Error ? error.message : String(error), }); }, + refetchOnWindowFocus: false, + enabled: enableFetch, }); const [columns, setColumns] = useState[]>([ @@ -246,6 +265,7 @@ export default function Annotations({ email }: { email: string }) { }, { size: 500, + minSize: 20, accessorKey: "input", header: "Input", cell: ({ row }) => { @@ -256,14 +276,7 @@ export default function Annotations({ email }: { email: string }) { return (
{input.map((item, i) => ( - + ))}
); @@ -271,6 +284,7 @@ export default function Annotations({ email }: { email: string }) { }, { size: 500, + minSize: 20, accessorKey: "output", header: "Output", cell: ({ row }) => { @@ -281,14 +295,7 @@ export default function Annotations({ email }: { email: string }) { return (
{output.map((item, i) => ( - + ))}
); @@ -310,7 +317,7 @@ export default function Annotations({ email }: { email: string }) { // Cell content component const CellContent = ({ test, row }: { test: Test; row: any }) => { - const isEval = test.id !== "user_id"; + const isEval = test.id !== "user_id" && test.id !== "user_score"; const spanId = row.original.span_id; const testId = test.id; const { isError, isLoading, data } = useQuery({ @@ -341,18 +348,19 @@ export default function Annotations({ email }: { email: string }) { {evaluation ? evaluation.ltUserScore : "Not evaluated"}

); + } else { + if (test.id === "user_id") { + const userId = data[0]?.userId || "Not Reported"; + return ( + + {userId} + + ); + } else if (test.id === "user_score") { + const userScore = data[0]?.userScore || "Not Reported"; + return

{userScore}

; + } } - - const userScore = data[0]?.userScore || ""; - const userId = data[0]?.userId || "Not Reported"; - if (test.id === "user_id") { - return ( - - {userId} - - ); - } - return

{userScore}

; }; const { @@ -448,6 +456,7 @@ export default function Annotations({ email }: { email: string }) { ) : tests?.length > 0 ? (
+ { + setPage(1); + setEnableFetch(true); + }} paginationLoading={showBottomLoader} scrollableDivRef={scrollableDivRef} /> diff --git a/app/api/metrics/score/route.ts b/app/api/metrics/score/route.ts index 46ba7bc8..0d9f0210 100644 --- a/app/api/metrics/score/route.ts +++ b/app/api/metrics/score/route.ts @@ -87,15 +87,15 @@ export async function POST(req: NextRequest) { dateScoreMap[date] = {}; } - if (!dateScoreMap[date][`${testId}-${evaluation.Test?.name}`]) { - dateScoreMap[date][`${testId}-${evaluation.Test?.name}`] = [0, 0]; + if (!dateScoreMap[date][`${evaluation.Test?.name}(${testId})`]) { + dateScoreMap[date][`${evaluation.Test?.name}(${testId})`] = [0, 0]; } const total = - dateScoreMap[date][`${testId}-${evaluation.Test?.name}`][0] + + dateScoreMap[date][`${evaluation.Test?.name}(${testId})`][0] + evaluation.ltUserScore || 0; - dateScoreMap[date][`${testId}-${evaluation.Test?.name}`][0] = total; - dateScoreMap[date][`${testId}-${evaluation.Test?.name}`][1] += 1; + dateScoreMap[date][`${evaluation.Test?.name}(${testId})`][0] = total; + dateScoreMap[date][`${evaluation.Test?.name}(${testId})`][1] += 1; }); } @@ -104,7 +104,11 @@ export async function POST(req: NextRequest) { const entry: any = { date }; Object.entries(scoresByTestId as any).forEach( ([testId, scores]: any) => { - entry[testId] = scores[0]; + if (scores[1] === 0) { + entry[testId] = 0; + } else { + entry[testId] = Math.round((scores[0] / scores[1]) * 100); + } } ); return entry; @@ -124,7 +128,8 @@ export async function POST(req: NextRequest) { if (scores[1] === 0) { scoresChartData[testId] = 0; } - scoresChartData[testId] = scores[0] / scores[1]; + scoresChartData[testId] = + Math.round((scores[0] / scores[1]) * 100) / 100; }); }); diff --git a/app/api/metrics/usage/cost/route.ts b/app/api/metrics/usage/cost/route.ts index d5a8fb56..b8a1b89b 100644 --- a/app/api/metrics/usage/cost/route.ts +++ b/app/api/metrics/usage/cost/route.ts @@ -25,6 +25,26 @@ export async function GET(req: NextRequest) { userId, model ); + + // aggregate cost by date + const costPerDate: any = []; + for (const c of cost) { + const date = c.date; + const existing = costPerDate.find((d: any) => d.date === date); + if (existing) { + existing.total += c.total; + existing.input += c.input; + existing.output += c.output; + } else { + costPerDate.push({ + date, + total: c.total, + input: c.input, + output: c.output, + }); + } + } + const total = { total: cost.reduce( (acc: any, curr: { total: any }) => acc + curr.total, @@ -42,7 +62,7 @@ export async function GET(req: NextRequest) { return NextResponse.json( { - cost, + cost: costPerDate, ...total, }, { diff --git a/components/annotations/annotations-table.tsx b/components/annotations/annotations-table.tsx index 24b21bfd..7c969d9e 100644 --- a/components/annotations/annotations-table.tsx +++ b/components/annotations/annotations-table.tsx @@ -27,7 +27,7 @@ import { HOW_TO_DO_ANNOTATIONS } from "@/lib/constants"; import { LLMSpan } from "@/lib/llm_span_util"; import { cn } from "@/lib/utils"; import { Evaluation, Test } from "@prisma/client"; -import { ResetIcon } from "@radix-ui/react-icons"; +import { ArrowTopRightIcon, ResetIcon } from "@radix-ui/react-icons"; import { ColumnDef, flexRender, @@ -36,7 +36,13 @@ import { VisibilityState, } from "@tanstack/react-table"; import { ProgressCircle } from "@tremor/react"; -import { ChevronDown, ChevronLeft, ChevronRight, XIcon } from "lucide-react"; +import { + ChevronDown, + ChevronLeft, + ChevronRight, + RefreshCwIcon, + XIcon, +} from "lucide-react"; import Link from "next/link"; import { useEffect, useMemo, useState } from "react"; import { useQuery, useQueryClient } from "react-query"; @@ -53,6 +59,7 @@ interface AnnotationsTableProps { tests: Test[]; loading?: boolean; fetching?: boolean; + refetch: () => void; paginationLoading?: boolean; scrollableDivRef?: React.RefObject; } @@ -64,6 +71,7 @@ export function AnnotationsTable({ tests, loading, fetching, + refetch, paginationLoading, scrollableDivRef, }: AnnotationsTableProps) { @@ -158,26 +166,29 @@ export function AnnotationsTable({ {!loading && data && data.length > 0 && (
-

- {fetching - ? "Fetching conversations..." - : `Fetched the last ${data.length} conversations`} -

-
- Seeing related spans as separate rows?{" "} - + +

- Learn how to do annotations - + {fetching + ? "Fetching conversations..." + : `Fetched the last ${data.length} conversations`} +

+ + Learn how to do annotations + +
diff --git a/components/annotations/chart-tabs.tsx b/components/annotations/chart-tabs.tsx index 08c7d607..188db007 100644 --- a/components/annotations/chart-tabs.tsx +++ b/components/annotations/chart-tabs.tsx @@ -1,19 +1,25 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { cn } from "@/lib/utils"; import { Test } from "@prisma/client"; import { ProgressCircle } from "@tremor/react"; import { useState } from "react"; import { useQuery } from "react-query"; import { EvalChart } from "../charts/eval-chart"; import { timeRanges } from "../shared/day-filter"; +import { Skeleton } from "../ui/skeleton"; export function ChartTabs({ projectId, tests, + defaultTab = "score", }: { projectId: string; tests: Test[]; + defaultTab?: "metrics" | "score"; }) { - const [lastNHours, setLastNHours] = useState(timeRanges[0].value); + const [lastNHours, setLastNHours] = useState( + timeRanges[timeRanges.length - 1].value + ); const { data: chartData, isLoading } = useQuery({ queryKey: ["fetch-score", projectId, lastNHours], @@ -60,53 +66,105 @@ export function ChartTabs({ }, }); - if (isLoading || !chartData) { - return
Loading
; - } else { - return ( - - - Overall Metrics - Total Score - - - - - - {Object.keys(chartData?.scores).length > 0 ? ( -
- {Object.keys(chartData?.scores).map( - (testId: string, index: number) => { - return ( -
+ + Total Score (%) + Trend (%) + + + + + + {isLoading && !chartData ? ( +
+ + + +
+ ) : Object.keys(chartData?.scores).length > 0 ? ( +
+ {Object.keys(chartData?.scores).map( + (testId: string, index: number) => { + const score = (chartData?.scores[testId] || 0) * 100; + const testName = tests.find((test) => + testId.includes(test.id) + )?.name; + const color = + score < 50 ? "red" : score < 80 ? "orange" : "green"; + const bgColor = `bg-${color}-100`; + return ( +
+ - - - {(chartData?.scores[testId] || 0) * 100}% - - - - {tests.find((test) => testId.includes(test.id))?.name} + {(chartData?.scores[testId] || 0) * 100}% -
- ); - } - )} -
- ) : ( -
-

No data available

+ + + {testName} + +
+ ); + } + )} +
+ ) : ( +
+

+ Get Started and measure the baseline performance of your + application. +

+
+ {["accuracy", "quality", "relevance"].map((testName, index) => { + const score = Math.floor(Math.random() * 100); + const color = + score < 50 ? "red" : score < 80 ? "orange" : "green"; + const bgColor = `bg-${color}-100`; + return ( +
+ + + {score}% + + + + {testName} + +
+ ); + })}
- )} - - - ); - } +
+ )} + + + ); } diff --git a/components/charts/eval-chart.tsx b/components/charts/eval-chart.tsx index 68c36e16..fe5b700d 100644 --- a/components/charts/eval-chart.tsx +++ b/components/charts/eval-chart.tsx @@ -1,103 +1,62 @@ -"use client"; - import { formatDurationForDisplay } from "@/lib/utils"; import { Test } from "@prisma/client"; -// import { AreaChart } from "@tremor/react"; import { AreaChart } from "@tremor/react"; -import { useState } from "react"; -import { useQuery } from "react-query"; -import DayFilter, { timeRanges } from "../shared/day-filter"; +import DayFilter from "../shared/day-filter"; import { Info } from "../shared/info"; -import LargeChartLoading from "./large-chart-skeleton"; +import { Skeleton } from "../ui/skeleton"; export function EvalChart({ - projectId, tests, + lastNHours, + setLastNHours, + isLoading, + chartData, chartDescription = "Trend of test scores over the selected period of time", info = "Score is the sum total all the evaluated score for a test over the selected period of time.", }: { - projectId: string; tests: Test[]; + lastNHours: number; + setLastNHours: (value: number) => void; + chartData: any; + isLoading: boolean; chartDescription?: string; info?: string; }) { - const [lastNHours, setLastNHours] = useState(timeRanges[0].value); - - const { data: chartData, isLoading } = useQuery({ - queryKey: ["fetch-score", projectId, lastNHours], - queryFn: async () => { - const filters = { - filters: [ - { - operation: "OR", - filters: [ - { - key: "llm.prompts", - operation: "NOT_EQUALS", - value: "", - type: "attribute", - }, - { - key: "name", - operation: "EQUALS", - value: "gen_ai.content.prompt", - type: "event", - }, - ], - }, - { - key: "status_code", - operation: "EQUALS", - value: "OK", - type: "property", - }, - ], - operation: "AND", - }; - const response = await fetch("/api/metrics/score", { - method: "POST", - body: JSON.stringify({ - projectId, - testIds: tests.map((test) => test.id), - lastNHours, - filters, - }), - }); - const result = await response.json(); - return result; - }, - }); - - if (isLoading || !chartData) { - return ; - } else { - return ( - <> -
- -
-
- {chartDescription} - -
+ return ( + <> +
+ +
+
+ {chartDescription} +
- `${test.id}-${test.name}`)} - colors={["purple", "blue", "red", "green", "orange", "black"]} - showAnimation={true} - showTooltip={true} - yAxisWidth={33} - noDataText="Get started by sending traces to your project." - /> -

- Evaulated Accuracy(%) over time{" "} - {formatDurationForDisplay(lastNHours)} -

- - ); - } + {!isLoading ? ( + chartData ? ( + `${test.name}(${test.id})`)} + colors={["purple", "blue", "red", "green", "orange", "black"]} + showAnimation={true} + showTooltip={true} + yAxisWidth={33} + noDataText="Get started by sending traces to your project." + /> + ) : ( +
+

No data available

+
+ ) + ) : ( + + )} +

+ Evaulated Accuracy(%) over time {formatDurationForDisplay(lastNHours)} +

+
+ + ); } diff --git a/components/charts/small-chart-skeleton.tsx b/components/charts/small-chart-skeleton.tsx index 2dd3eb07..eec4abea 100644 --- a/components/charts/small-chart-skeleton.tsx +++ b/components/charts/small-chart-skeleton.tsx @@ -4,7 +4,7 @@ import { Skeleton } from "@/components/ui/skeleton"; export default function SmallChartSkeleton() { return ( -
+
diff --git a/components/charts/token-chart.tsx b/components/charts/token-chart.tsx index 115319a1..84c40a7a 100644 --- a/components/charts/token-chart.tsx +++ b/components/charts/token-chart.tsx @@ -52,7 +52,7 @@ export function TokenChart({ } else { return ( <> -
+

@@ -137,7 +137,7 @@ export function CostChart({ } else { return ( <> -

+

diff --git a/components/charts/trace-chart.tsx b/components/charts/trace-chart.tsx index 6953bc33..1691d4ac 100644 --- a/components/charts/trace-chart.tsx +++ b/components/charts/trace-chart.tsx @@ -100,7 +100,7 @@ export function TraceSpanChart({ return ( <> -

+

diff --git a/components/project/metrics.tsx b/components/project/metrics.tsx index 71a953aa..99628925 100644 --- a/components/project/metrics.tsx +++ b/components/project/metrics.tsx @@ -1,7 +1,14 @@ "use client"; +import { Test } from "@prisma/client"; +import { ArrowTopRightIcon } from "@radix-ui/react-icons"; +import Link from "next/link"; import { useParams } from "next/navigation"; import { useState } from "react"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; +import { ChartTabs } from "../annotations/chart-tabs"; +import LargeChartSkeleton from "../charts/large-chart-skeleton"; import { TraceLatencyChart } from "../charts/latency-chart"; import { CostChart, TokenChart } from "../charts/token-chart"; import { TraceSpanChart } from "../charts/trace-chart"; @@ -16,6 +23,36 @@ export default function Metrics({ email }: { email: string }) { const [userId, setUserId] = useState(""); const [model, setModel] = useState(""); + const { + data: tests, + isLoading: testsLoading, + error: testsError, + } = useQuery({ + queryKey: ["fetch-tests-query", project_id], + queryFn: async () => { + const response = await fetch(`/api/test?projectId=${project_id}`); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch tests"); + } + const result = await response.json(); + + // sort tests by created date + result.tests.sort( + (a: Test, b: Test) => + new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime() + ); + + return result.tests as Test[]; + }, + refetchOnWindowFocus: false, + onError: (error) => { + toast.error("Failed to fetch tests", { + description: error instanceof Error ? error.message : String(error), + }); + }, + }); + return (

@@ -26,7 +63,7 @@ export default function Metrics({ email }: { email: string }) {
-
+
+
+
+
+

Manual Evaluation Scores

+ + Jump to Annotations + + +
+ + {testsLoading || testsError ? ( + + ) : ( + + )} +
+
diff --git a/components/project/traces/traces-table.tsx b/components/project/traces/traces-table.tsx index 08983c23..6bb56918 100644 --- a/components/project/traces/traces-table.tsx +++ b/components/project/traces/traces-table.tsx @@ -27,7 +27,7 @@ import { useReactTable, VisibilityState, } from "@tanstack/react-table"; -import { ChevronDown } from "lucide-react"; +import { ChevronDown, RefreshCwIcon } from "lucide-react"; import Link from "next/link"; import { useEffect, useMemo, useState } from "react"; import TraceRowSkeleton from "../../shared/row-skeleton"; @@ -41,6 +41,7 @@ interface TracesTableProps { data: TData[]; loading?: boolean; fetching?: boolean; + refetch: () => void; paginationLoading?: boolean; scrollableDivRef?: React.RefObject; } @@ -51,6 +52,7 @@ export function TracesTable({ data, loading, fetching, + refetch, paginationLoading, scrollableDivRef, }: TracesTableProps) { @@ -145,16 +147,21 @@ export function TracesTable({ {!loading && data && data.length > 0 && (
-

- {fetching - ? "Fetching traces..." - : `Fetched the last ${data.length} traces`} -

+
+ +

+ {fetching + ? "Fetching traces..." + : `Fetched the last ${data.length} traces`} +

+
Seeing related spans as separate rows?{" "} { @@ -215,10 +217,7 @@ export default function Traces({ email }: { email: string }) { )) : null @@ -228,6 +227,8 @@ export default function Traces({ email }: { email: string }) { }, }, { + size: 500, + minSize: 20, accessorKey: "outputs", header: "Outputs", cell: ({ row }) => { @@ -243,10 +244,7 @@ export default function Traces({ email }: { email: string }) { )) : null @@ -564,6 +562,8 @@ export default function Traces({ email }: { email: string }) { checked={group} onCheckedChange={(check) => { setGroup(check); + setPage(1); + setEnableFetch(true); // Save the preference in local storage if (typeof window !== "undefined") { @@ -589,6 +589,10 @@ export default function Traces({ email }: { email: string }) { loading={ (fetchTraces.isLoading || showFreshLoading) && !showBottomLoader } + refetch={() => { + setPage(1); + setEnableFetch(true); + }} fetching={fetchTraces.isFetching} paginationLoading={showBottomLoader} scrollableDivRef={scrollableDivRef} diff --git a/components/shared/hover-cell.tsx b/components/shared/hover-cell.tsx index 1f51bc18..b475f435 100644 --- a/components/shared/hover-cell.tsx +++ b/components/shared/hover-cell.tsx @@ -4,17 +4,21 @@ import { HoverCardTrigger, } from "@/components/ui/hover-card"; import { cn, safeStringify } from "@/lib/utils"; -import { ClipboardIcon } from "lucide-react"; +import { ClipboardIcon, MoveDiagonal, X } from "lucide-react"; +import { useState } from "react"; import { toast } from "sonner"; export function HoverCell({ values, className, + expand = false, }: { values: any[]; className?: string; + expand?: boolean; }) { try { + const [expandedView, setExpandedView] = useState(expand); if (!values || !Array.isArray(values)) { return null; } @@ -57,8 +61,30 @@ export function HoverCell({ className="h-4 w-4 hover:bg-primary-foreground hover:text-primary cursor-pointer text-muted-foreground absolute top-1 right-1" onClick={copyToClipboard} /> + {!expandedView && ( + { + e.stopPropagation(); + setExpandedView(!expandedView); + }} + /> + )} + {expandedView && ( + { + e.stopPropagation(); + setExpandedView(!expandedView); + }} + /> + )}