diff --git a/evals/bananalyzer-ts/.gitignore b/evals/bananalyzer-ts/.gitignore new file mode 100644 index 00000000..ecdef5b8 --- /dev/null +++ b/evals/bananalyzer-ts/.gitignore @@ -0,0 +1 @@ +static \ No newline at end of file diff --git a/evals/bananalyzer-ts/evaluate.ts b/evals/bananalyzer-ts/evaluate.ts index 6939fe38..dc98bf7a 100644 --- a/evals/bananalyzer-ts/evaluate.ts +++ b/evals/bananalyzer-ts/evaluate.ts @@ -38,10 +38,7 @@ function validateEndUrlMatch(expected: string, actual: string): boolean { // Updated evaluateExample function export async function evaluateExample(exampleId: string): Promise { const examples = JSON.parse( - fs.readFileSync( - path.join(__dirname, "../bananalyzer/static/examples.json"), - "utf-8", - ), + fs.readFileSync(path.join(__dirname, "./static/examples.json"), "utf-8"), ); const example = examples.find((example: Example) => example.id === exampleId); const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 }); @@ -55,7 +52,7 @@ export async function evaluateExample(exampleId: string): Promise { // Handle MHTML Source const mhtmlFilePath = path.resolve( __dirname, - `../bananalyzer/static/${example.id}/index.mhtml`, + `./static/${example.id}/index.mhtml`, ); const parsedMHTML = await parseMHTMLFile(mhtmlFilePath); diff --git a/evals/bananalyzer-ts/init.sh b/evals/bananalyzer-ts/init.sh new file mode 100755 index 00000000..84ea5ae1 --- /dev/null +++ b/evals/bananalyzer-ts/init.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Get the directory of the script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Clone the repository +git clone https://github.com/reworkd/bananalyzer.git + +# Copy the static folder to the script's directory +rm -rf "$SCRIPT_DIR/static" +cp -r bananalyzer/static "$SCRIPT_DIR" + +# Remove the cloned repository +rm -rf bananalyzer \ No newline at end of file diff --git a/evals/bananalyzer-ts/schemas.ts b/evals/bananalyzer-ts/schemas.ts index c27c89d6..5e439250 100644 --- a/evals/bananalyzer-ts/schemas.ts +++ b/evals/bananalyzer-ts/schemas.ts @@ -4,10 +4,7 @@ import path from "path"; // Load examples const examples = JSON.parse( - fs.readFileSync( - path.join(__dirname, "../bananalyzer/static/examples.json"), - "utf-8", - ), + fs.readFileSync(path.join(__dirname, "./static/examples.json"), "utf-8"), ); export const ExampleType = z.enum(["listing", "detail", "listing_detail"]); @@ -73,7 +70,7 @@ export type Eval = z.infer; // Separate function to get predefined schema by name export function getSchemaByName(schemaName: SchemaName): z.ZodRawShape { - const schemaPath = path.join(__dirname, "../bananalyzer/static/schemas.json"); + const schemaPath = path.join(__dirname, "./static/schemas.json"); const schemasJson = JSON.parse(fs.readFileSync(schemaPath, "utf-8")); if (!(schemaName in schemasJson)) { @@ -130,6 +127,6 @@ function zodTypeFromJsonSchema(jsonSchema: any): z.ZodTypeAny { // Function to read and parse the goals.json file export function getGoals(): Record { - const goalsPath = path.join(__dirname, "../bananalyzer/static/goals.json"); + const goalsPath = path.join(__dirname, "./static/goals.json"); return JSON.parse(fs.readFileSync(goalsPath, "utf-8")); } diff --git a/evals/bananalyzer-ts/server/public/f1b99917fa284587.css b/evals/bananalyzer-ts/server/public/f1b99917fa284587.css deleted file mode 100644 index bbc56683..00000000 --- a/evals/bananalyzer-ts/server/public/f1b99917fa284587.css +++ /dev/null @@ -1,273 +0,0 @@ -@charset "utf-8"; - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/d1d9458b69004127-s.woff2") format("woff2"); unicode-range: U+460-52F, U+1C80-1C88, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F; } - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/b967158bc7d7a9fb-s.woff2") format("woff2"); unicode-range: U+301, U+400-45F, U+490-491, U+4B0-4B1, U+2116; } - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/ae9ae6716d4f8bf8-s.woff2") format("woff2"); unicode-range: U+1F00-1FFF; } - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/c0f5ec5bbf5913b7-s.woff2") format("woff2"); unicode-range: U+370-3FF; } - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/b1db3e28af9ef94a-s.woff2") format("woff2"); unicode-range: U+102-103, U+110-111, U+128-129, U+168-169, U+1A0-1A1, U+1AF-1B0, U+300-301, U+303-304, U+308-309, U+323, U+329, U+1EA0-1EF9, U+20AB; } - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/9c4f34569c9b36ca-s.woff2") format("woff2"); unicode-range: U+100-2AF, U+304, U+308, U+329, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; } - -@font-face { font-family: __Inter_20951f; font-style: normal; font-weight: 100 900; font-display: swap; src: url("/_next/static/media/2aaf0723e720e8b9-s.p.woff2") format("woff2"); unicode-range: U+0-FF, U+131, U+152-153, U+2BB-2BC, U+2C6, U+2DA, U+2DC, U+304, U+308, U+329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; } - -@font-face { font-family: __Inter_Fallback_20951f; src: local("Arial"); ascent-override: 90.2%; descent-override: 22.48%; line-gap-override: 0%; size-adjust: 107.4%; } - -.__className_20951f { font-family: __Inter_20951f, __Inter_Fallback_20951f; font-style: normal; } - -*, ::after, ::before { box-sizing: border-box; border: 0px solid rgb(229, 231, 235); } - -::after, ::before { --tw-content: ""; } - -html { line-height: 1.5; text-size-adjust: 100%; tab-size: 4; font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; font-feature-settings: normal; font-variation-settings: normal; } - -body { margin: 0px; line-height: inherit; } - -hr { height: 0px; color: inherit; border-top-width: 1px; } - -abbr:where([title]) { text-decoration: underline dotted; } - -h1, h2, h3, h4, h5, h6 { font-size: inherit; font-weight: inherit; } - -a { color: inherit; text-decoration: inherit; } - -b, strong { font-weight: bolder; } - -code, kbd, pre, samp { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; font-size: 1em; } - -small { font-size: 80%; } - -sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; } - -sub { bottom: -0.25em; } - -sup { top: -0.5em; } - -table { text-indent: 0px; border-color: inherit; border-collapse: collapse; } - -button, input, optgroup, select, textarea { font-family: inherit; font-feature-settings: inherit; font-variation-settings: inherit; font-size: 100%; font-weight: inherit; line-height: inherit; color: inherit; margin: 0px; padding: 0px; } - -button, select { text-transform: none; } - -[type="button"], [type="reset"], [type="submit"], button { appearance: button; background-color: transparent; background-image: none; } - -progress { vertical-align: baseline; } - -::-webkit-inner-spin-button, ::-webkit-outer-spin-button { height: auto; } - -[type="search"] { appearance: textfield; outline-offset: -2px; } - -::-webkit-search-decoration { appearance: none; } - -::-webkit-file-upload-button { appearance: button; font: inherit; } - -summary { display: list-item; } - -blockquote, dd, dl, figure, h1, h2, h3, h4, h5, h6, hr, p, pre { margin: 0px; } - -fieldset { margin: 0px; } - -fieldset, legend { padding: 0px; } - -menu, ol, ul { list-style: none; margin: 0px; padding: 0px; } - -dialog { padding: 0px; } - -textarea { resize: vertical; } - -input::placeholder, textarea::placeholder { opacity: 1; color: rgb(156, 163, 175); } - -[role="button"], button { cursor: pointer; } - -:disabled { cursor: default; } - -audio, canvas, embed, iframe, img, object, svg, video { display: block; vertical-align: middle; } - -img, video { max-width: 100%; height: auto; } - -[hidden] { display: none; } - -*, ::after, ::before { --tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; --tw-rotate: 0; --tw-skew-x: 0; --tw-skew-y: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-pan-x: ; --tw-pan-y: ; --tw-pinch-zoom: ; --tw-scroll-snap-strictness: proximity; --tw-gradient-from-position: ; --tw-gradient-via-position: ; --tw-gradient-to-position: ; --tw-ordinal: ; --tw-slashed-zero: ; --tw-numeric-figure: ; --tw-numeric-spacing: ; --tw-numeric-fraction: ; --tw-ring-inset: ; --tw-ring-offset-width: 0px; --tw-ring-offset-color: #fff; --tw-ring-color: rgba(59,130,246,.5); --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-shadow: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-shadow-colored: 0 0 #0000; --tw-blur: ; --tw-brightness: ; --tw-contrast: ; --tw-grayscale: ; --tw-hue-rotate: ; --tw-invert: ; --tw-saturate: ; --tw-sepia: ; --tw-drop-shadow: ; --tw-backdrop-blur: ; --tw-backdrop-brightness: ; --tw-backdrop-contrast: ; --tw-backdrop-grayscale: ; --tw-backdrop-hue-rotate: ; --tw-backdrop-invert: ; --tw-backdrop-opacity: ; --tw-backdrop-saturate: ; --tw-backdrop-sepia: ; } - -::backdrop { --tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; --tw-rotate: 0; --tw-skew-x: 0; --tw-skew-y: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-pan-x: ; --tw-pan-y: ; --tw-pinch-zoom: ; --tw-scroll-snap-strictness: proximity; --tw-gradient-from-position: ; --tw-gradient-via-position: ; --tw-gradient-to-position: ; --tw-ordinal: ; --tw-slashed-zero: ; --tw-numeric-figure: ; --tw-numeric-spacing: ; --tw-numeric-fraction: ; --tw-ring-inset: ; --tw-ring-offset-width: 0px; --tw-ring-offset-color: #fff; --tw-ring-color: rgba(59,130,246,.5); --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-shadow: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-shadow-colored: 0 0 #0000; --tw-blur: ; --tw-brightness: ; --tw-contrast: ; --tw-grayscale: ; --tw-hue-rotate: ; --tw-invert: ; --tw-saturate: ; --tw-sepia: ; --tw-drop-shadow: ; --tw-backdrop-blur: ; --tw-backdrop-brightness: ; --tw-backdrop-contrast: ; --tw-backdrop-grayscale: ; --tw-backdrop-hue-rotate: ; --tw-backdrop-invert: ; --tw-backdrop-opacity: ; --tw-backdrop-saturate: ; --tw-backdrop-sepia: ; } - -.absolute { position: absolute; } - -.relative { position: relative; } - -.-right-8 { right: -2rem; } - -.-top-1 { top: -0.25rem; } - -.bottom-0 { bottom: 0px; } - -.left-1\/2 { left: 50%; } - -.mb-2 { margin-bottom: 0.5rem; } - -.ml-16 { margin-left: 4rem; } - -.ml-36 { margin-left: 9rem; } - -.ml-6 { margin-left: 1.5rem; } - -.flex { display: flex; } - -.grid { display: grid; } - -.hidden { display: none; } - -.h-10 { height: 2.5rem; } - -.h-\[4\.3rem\] { height: 4.3rem; } - -.h-\[80vh\] { height: 80vh; } - -.min-h-screen { min-height: 100vh; } - -.w-\[4\.3rem\] { width: 4.3rem; } - -.w-\[80vw\] { width: 80vw; } - -.w-fit { width: fit-content; } - -.w-full { width: 100%; } - -.max-w-screen-sm { max-width: 640px; } - -.-translate-x-1\/2 { --tw-translate-x: -50%; } - -.-translate-x-12, .-translate-x-1\/2 { transform: translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y)); } - -.-translate-x-12 { --tw-translate-x: -3rem; } - -.cursor-pointer { cursor: pointer; } - -.list-inside { list-style-position: inside; } - -.flex-col { flex-direction: column; } - -.items-center { align-items: center; } - -.justify-center { justify-content: center; } - -.gap-1 { gap: 0.25rem; } - -.gap-2 { gap: 0.5rem; } - -.gap-6 { gap: 1.5rem; } - -.rounded-full { border-radius: 9999px; } - -.rounded-md { border-radius: 0.375rem; } - -.border-4 { border-width: 4px; } - -.border-8 { border-width: 8px; } - -.border-\[3px\] { border-width: 3px; } - -.border-x-\[3px\] { border-left-width: 3px; border-right-width: 3px; } - -.border-l-4 { border-left-width: 4px; } - -.border-blue-600 { --tw-border-opacity: 1; border-color: rgb(37 99 235/var(--tw-border-opacity)); } - -.border-red-600\/80 { border-color: rgba(220, 38, 38, 0.8); } - -.border-yellow-400 { --tw-border-opacity: 1; border-color: rgb(250 204 21/var(--tw-border-opacity)); } - -.border-zinc-300 { --tw-border-opacity: 1; border-color: rgb(212 212 216/var(--tw-border-opacity)); } - -.bg-black\/80 { background-color: rgba(0, 0, 0, 0.8); } - -.bg-indigo-900 { --tw-bg-opacity: 1; background-color: rgb(49 46 129/var(--tw-bg-opacity)); } - -.bg-red-600\/30 { background-color: rgba(220, 38, 38, 0.3); } - -.bg-white { --tw-bg-opacity: 1; background-color: rgb(255 255 255/var(--tw-bg-opacity)); } - -.p-1 { padding: 0.25rem; } - -.p-16 { padding: 4rem; } - -.px-5 { padding-left: 1.25rem; padding-right: 1.25rem; } - -.py-6 { padding-top: 1.5rem; padding-bottom: 1.5rem; } - -.pb-4 { padding-bottom: 1rem; } - -.pb-6 { padding-bottom: 1.5rem; } - -.pl-2 { padding-left: 0.5rem; } - -.pl-4 { padding-left: 1rem; } - -.pt-2 { padding-top: 0.5rem; } - -.text-center { text-align: center; } - -.font-arimo { font-family: Arimo, sans-serif; } - -.font-serif { font-family: ui-serif, Georgia, Cambria, "Times New Roman", Times, serif; } - -.text-3xl { font-size: 1.875rem; line-height: 2.25rem; } - -.text-4xl { font-size: 2.25rem; line-height: 2.5rem; } - -.text-6xl { font-size: 3.75rem; line-height: 1; } - -.text-sm { font-size: 0.875rem; line-height: 1.25rem; } - -.text-xl { font-size: 1.25rem; line-height: 1.75rem; } - -.font-black { font-weight: 900; } - -.font-bold { font-weight: 700; } - -.italic { font-style: italic; } - -.text-black { --tw-text-opacity: 1; color: rgb(0 0 0/var(--tw-text-opacity)); } - -.text-blue-600 { --tw-text-opacity: 1; color: rgb(37 99 235/var(--tw-text-opacity)); } - -.text-gray-500 { --tw-text-opacity: 1; color: rgb(107 114 128/var(--tw-text-opacity)); } - -.text-slate-600 { --tw-text-opacity: 1; color: rgb(71 85 105/var(--tw-text-opacity)); } - -.text-yellow-500 { --tw-text-opacity: 1; color: rgb(234 179 8/var(--tw-text-opacity)); } - -.text-zinc-300 { --tw-text-opacity: 1; color: rgb(212 212 216/var(--tw-text-opacity)); } - -.line-through { text-decoration-line: line-through; } - -.transition { transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter, -webkit-backdrop-filter; transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1); transition-duration: 0.15s; } - -.transition-colors { transition-property: color, background-color, border-color, text-decoration-color, fill, stroke; transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1); transition-duration: 0.15s; } - -html { background: rgb(255, 255, 255); } - -:root { --foreground-rgb: 0,0,0; --background-start-rgb: 214,219,220; --background-end-rgb: 255,255,255; } - -@media (prefers-color-scheme: dark) { - :root { --foreground-rgb: 255,255,255; --background-start-rgb: 0,0,0; --background-end-rgb: 0,0,0; } -} - -.grid-background { background-size: 30px 30px; background-image: linear-gradient(90deg, rgba(128, 128, 128, 0.2) 2px, transparent 0px), linear-gradient(rgba(128, 128, 128, 0.2) 2px, transparent 0px); } - -.custom-list li::before { content: "○"; display: inline-block; margin-right: 0.5em; } - -.hover\:cursor-not-allowed:hover { cursor: not-allowed; } - -.hover\:bg-yellow-400:hover { --tw-bg-opacity: 1; background-color: rgb(250 204 21/var(--tw-bg-opacity)); } - -.hover\:text-black:hover { --tw-text-opacity: 1; color: rgb(0 0 0/var(--tw-text-opacity)); } - -.hover\:text-blue-900:hover { --tw-text-opacity: 1; color: rgb(30 58 138/var(--tw-text-opacity)); } - -@media (min-width: 768px) { - .md\:w-\[8em\] { width: 8em; } -} \ No newline at end of file diff --git a/evals/playground.ts b/evals/playground.ts index b0f6b777..bfcf870c 100644 --- a/evals/playground.ts +++ b/evals/playground.ts @@ -3,7 +3,11 @@ import { Stagehand } from "../lib"; import { z } from "zod"; const google_jobs = async () => { - const stagehand = new Stagehand({ env: "LOCAL", verbose: true, debugDom: true }); + const stagehand = new Stagehand({ + env: "LOCAL", + verbose: true, + debugDom: true, + }); await stagehand.init({ modelName: "gpt-4o-2024-08-06" }); await stagehand.page.goto("https://www.google.com/"); @@ -19,38 +23,51 @@ const google_jobs = async () => { await stagehand.act({ action: "click on the search button" }); - await stagehand.act({ action: "click on the learn more button for the first job" }); + await stagehand.act({ + action: "click on the learn more button for the first job", + }); const jobDetails = await stagehand.extract({ - instruction: "Extract the following details from the job posting: application deadline, minimum qualifications (degree and years of experience), and preferred qualifications (degree and years of experience)", + instruction: + "Extract the following details from the job posting: application deadline, minimum qualifications (degree and years of experience), and preferred qualifications (degree and years of experience)", schema: z.object({ - applicationDeadline: z.string().describe("The date until which the application window will be open"), + applicationDeadline: z + .string() + .describe("The date until which the application window will be open"), minimumQualifications: z.object({ degree: z.string().describe("The minimum required degree"), - yearsOfExperience: z.number().describe("The minimum required years of experience") + yearsOfExperience: z + .number() + .describe("The minimum required years of experience"), }), preferredQualifications: z.object({ degree: z.string().describe("The preferred degree"), - yearsOfExperience: z.number().describe("The preferred years of experience") - }) + yearsOfExperience: z + .number() + .describe("The preferred years of experience"), + }), }), - modelName: "gpt-4o-2024-08-06" + modelName: "gpt-4o-2024-08-06", }); console.log("Job Details:", jobDetails); - const isJobDetailsValid = jobDetails && - Object.values(jobDetails).every(value => - value !== null && - value !== undefined && - value !== '' && - (typeof value !== 'object' || Object.values(value).every(v => - v !== null && - v !== undefined && - v !== '' && - (typeof v === 'number' || typeof v === 'string') - )) - ); + const isJobDetailsValid = + jobDetails && + Object.values(jobDetails).every( + (value) => + value !== null && + value !== undefined && + value !== "" && + (typeof value !== "object" || + Object.values(value).every( + (v) => + v !== null && + v !== undefined && + v !== "" && + (typeof v === "number" || typeof v === "string"), + )), + ); await stagehand.context.close(); @@ -60,11 +77,9 @@ const google_jobs = async () => { }; async function main() { - const [googleJobsResult] = await Promise.all([ - google_jobs(), - ]); - + const [googleJobsResult] = await Promise.all([google_jobs()]); + console.log("Google Jobs result:", googleJobsResult); } -main().catch(console.error); \ No newline at end of file +main().catch(console.error);