diff --git a/apps/evalite-ui/app/routes/_index.tsx b/apps/evalite-ui/app/routes/_index.tsx index ebb549b..fdef1ad 100644 --- a/apps/evalite-ui/app/routes/_index.tsx +++ b/apps/evalite-ui/app/routes/_index.tsx @@ -3,6 +3,7 @@ import type { MetaFunction } from "@remix-run/node"; import { Link, useLoaderData } from "@remix-run/react"; import { useSubscribeToTestServer } from "~/use-subscribe-to-socket"; import { scoreToPercent } from "~/utils"; +import { Zap } from "lucide-react"; export const meta: MetaFunction = () => { return [ @@ -11,23 +12,77 @@ export const meta: MetaFunction = () => { ]; }; -export const clientLoader = () => { - return getEvals(); +export const clientLoader = async () => { + const evals = await getEvals(); + + console.log(evals); + + return { + menu: Object.entries(evals).map(([key, value]) => { + const mostRecentEval = value[0]!; + + const secondMostRecentEval = value[1]; + + const score = mostRecentEval.score; + + const state = !secondMostRecentEval + ? "first" + : score > secondMostRecentEval.score + ? "up" + : score < secondMostRecentEval.score + ? "down" + : "same"; + return { + name: key, + state, + score, + }; + }), + }; }; export default function Index() { - const files = useLoaderData(); + const evals = useLoaderData(); const server = useSubscribeToTestServer(); return (
-
); } diff --git a/apps/evalite-ui/app/use-subscribe-to-socket.ts b/apps/evalite-ui/app/use-subscribe-to-socket.ts index 8208646..197cb1a 100644 --- a/apps/evalite-ui/app/use-subscribe-to-socket.ts +++ b/apps/evalite-ui/app/use-subscribe-to-socket.ts @@ -17,7 +17,7 @@ export const useSubscribeToTestServer = () => { useEffect(() => { const socket = new WebSocket( - `ws://localhost:${DEFAULT_SERVER_PORT}/socket` + `ws://localhost:${DEFAULT_SERVER_PORT}/api/socket` ); socket.onmessage = (event) => { diff --git a/apps/evalite-ui/package.json b/apps/evalite-ui/package.json index e95583f..ed1c17c 100644 --- a/apps/evalite-ui/package.json +++ b/apps/evalite-ui/package.json @@ -10,13 +10,14 @@ "lint": "tsc" }, "dependencies": { + "@evalite/core": "workspace:*", "@remix-run/node": "^2.14.0", "@remix-run/react": "^2.14.0", "@remix-run/serve": "^2.14.0", "isbot": "^4.1.0", + "lucide-react": "^0.464.0", "react": "^18.2.0", - "react-dom": "^18.2.0", - "@evalite/core": "workspace:*" + "react-dom": "^18.2.0" }, "devDependencies": { "@remix-run/dev": "^2.14.0", diff --git a/apps/evalite-ui/tsconfig.json b/apps/evalite-ui/tsconfig.json index da46a93..279141a 100644 --- a/apps/evalite-ui/tsconfig.json +++ b/apps/evalite-ui/tsconfig.json @@ -8,15 +8,8 @@ "**/.client/**/*.tsx" ], "compilerOptions": { - "lib": [ - "DOM", - "DOM.Iterable", - "ES2022" - ], - "types": [ - "@remix-run/node", - "vite/client" - ], + "lib": ["DOM", "DOM.Iterable", "ES2022"], + "types": ["@remix-run/node", "vite/client"], "isolatedModules": true, "esModuleInterop": true, "jsx": "react-jsx", @@ -30,13 +23,12 @@ "forceConsistentCasingInFileNames": true, "allowImportingTsExtensions": true, "verbatimModuleSyntax": true, + "noUncheckedIndexedAccess": true, "baseUrl": ".", "paths": { - "~/*": [ - "./app/*" - ] + "~/*": ["./app/*"] }, // Vite takes care of building everything, not tsc. "noEmit": true } -} \ No newline at end of file +} diff --git a/packages/evalite-core/src/json-db.ts b/packages/evalite-core/src/json-db.ts index 444222e..ed12e4c 100644 --- a/packages/evalite-core/src/json-db.ts +++ b/packages/evalite-core/src/json-db.ts @@ -109,6 +109,7 @@ export const getRows = async (opts: { return dbContents .trim() .split("\n") + .filter(Boolean) .map((line) => { const parsed: JsonDBEval = JSON.parse(line); diff --git a/packages/example/src/basics.eval.ts b/packages/example/src/basics.eval.ts index ca54da6..52edf6c 100644 --- a/packages/example/src/basics.eval.ts +++ b/packages/example/src/basics.eval.ts @@ -52,3 +52,44 @@ evalite("Test basics", { }, scorers: [Factuality], }); + +evalite("Test basics 2", { + data: async () => [ + { + input: `What's the capital of France?`, + expected: `Paris`, + }, + { + input: `What's the capital of Germany?`, + expected: `Berlin`, + }, + { + input: `What's the capital of Italy?`, + expected: `Rome`, + }, + { + input: `What's the capital of the United States?`, + expected: `Washington DC`, + }, + { + input: `What's the capital of Canada?`, + expected: `Ottawa`, + }, + { + input: `What's the capital of Japan?`, + expected: `Tokyo`, + }, + ], + task: async (input) => { + const result = await generateText({ + model: cacheModel(openai("gpt-3.5-turbo"), storage), + system: ` + Answer the question concisely, in as few words as possible. + `, + prompt: input, + }); + + return result.text; + }, + scorers: [Factuality], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e35583..0e2e348 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: isbot: specifier: ^4.1.0 version: 4.4.0 + lucide-react: + specifier: ^0.464.0 + version: 0.464.0(react@18.3.1) react: specifier: ^18.2.0 version: 18.3.1 @@ -120,18 +123,9 @@ importers: '@evalite/core': specifier: workspace:* version: link:../evalite-core - '@fastify/static': - specifier: ^8.0.3 - version: 8.0.3 - '@fastify/websocket': - specifier: 11.0.1 - version: 11.0.1 commander: specifier: ^12.1.0 version: 12.1.0 - fastify: - specifier: ^5.1.0 - version: 5.1.0 table: specifier: ^6.8.2 version: 6.8.2 @@ -3237,6 +3231,11 @@ packages: resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} engines: {node: '>=12'} + lucide-react@0.464.0: + resolution: {integrity: sha512-eCx1qClbnw5qRqB2Z1AFFp71wdJXEwhPp5ii8LviyvHb7o/7eMXFiTyDHh7JpjM9BO9pC6ZUp/c7mCwwxbPIcg==} + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc + magic-string@0.30.11: resolution: {integrity: sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==} @@ -8425,6 +8424,10 @@ snapshots: lru-cache@7.18.3: {} + lucide-react@0.464.0(react@18.3.1): + dependencies: + react: 18.3.1 + magic-string@0.30.11: dependencies: '@jridgewell/sourcemap-codec': 1.5.0