diff --git a/evals/index.eval.ts b/evals/index.eval.ts index 0d1a693e..795f4e67 100644 --- a/evals/index.eval.ts +++ b/evals/index.eval.ts @@ -157,6 +157,7 @@ const costar = async () => { } }; + const google_jobs = async () => { const stagehand = new Stagehand({ env: "LOCAL", verbose: 2, debugDom: true, headless: process.env.HEADLESS !== 'false' }); await stagehand.init({ modelName: "gpt-4o-2024-08-06" }); @@ -173,8 +174,8 @@ const google_jobs = async () => { await stagehand.act({ action: "input new york city into location" }); await stagehand.act({ action: "click on the search button" }); - - await stagehand.act({ action: "click on the learn more button for the first job" }); + // NOTE: "click on the first Learn More button" is not working - the span for learn more is not clickable and the a href is after it + await stagehand.act({ action: "click on the first job link" }); const jobDetails = await stagehand.extract({ instruction: "Extract the following details from the job posting: application deadline, minimum qualifications (degree and years of experience), and preferred qualifications (degree and years of experience)", @@ -198,11 +199,9 @@ const google_jobs = async () => { Object.values(jobDetails).every(value => value !== null && value !== undefined && - value !== '' && (typeof value !== 'object' || Object.values(value).every(v => v !== null && v !== undefined && - v !== '' && (typeof v === 'number' || typeof v === 'string') )) ); diff --git a/evals/playground.ts b/evals/playground.ts index 19ae96c3..4ffa7bc2 100644 --- a/evals/playground.ts +++ b/evals/playground.ts @@ -5,7 +5,7 @@ import { z } from "zod"; const costar = async () => { const stagehand = new Stagehand({ env: "LOCAL", verbose: 2, debugDom: true, headless: process.env.HEADLESS !== 'false' }); await stagehand.init(); - // TODO: fix this eval + // TODO: fix this eval - it works only on some days depending on the article try { await Promise.race([ stagehand.page.goto("https://www.costar.com/"), @@ -43,12 +43,67 @@ const costar = async () => { } }; +const google_jobs = async () => { + const stagehand = new Stagehand({ env: "LOCAL", verbose: 2, debugDom: true, headless: process.env.HEADLESS !== 'false' }); + await stagehand.init({ modelName: "gpt-4o-2024-08-06" }); + + await stagehand.page.goto("https://www.google.com/"); + await stagehand.waitForSettledDom(); + + await stagehand.act({ action: "click on the about page" }); + + await stagehand.act({ action: "click on the careers page" }); + + await stagehand.act({ action: "input data scientist into role" }); + + await stagehand.act({ action: "input new york city into location" }); + + await stagehand.act({ action: "click on the search button" }); + // NOTE: "click on the first Learn More button" is not working - the span for learn more is not clickable and the a href is after it + await stagehand.act({ action: "click on the first job link" }); + + const jobDetails = await stagehand.extract({ + instruction: "Extract the following details from the job posting: application deadline, minimum qualifications (degree and years of experience), and preferred qualifications (degree and years of experience)", + schema: z.object({ + applicationDeadline: z.string().describe("The date until which the application window will be open"), + minimumQualifications: z.object({ + degree: z.string().describe("The minimum required degree"), + yearsOfExperience: z.number().describe("The minimum required years of experience") + }), + preferredQualifications: z.object({ + degree: z.string().describe("The preferred degree"), + yearsOfExperience: z.number().describe("The preferred years of experience") + }) + }), + modelName: "gpt-4o-2024-08-06" + }); + + console.log("Job Details:", jobDetails); + + const isJobDetailsValid = jobDetails && + Object.values(jobDetails).every(value => + value !== null && + value !== undefined && + (typeof value !== 'object' || Object.values(value).every(v => + v !== null && + v !== undefined && + (typeof v === 'number' || typeof v === 'string') + )) + ); + + await stagehand.context.close(); + + console.log("Job Details valid:", isJobDetailsValid); + + return isJobDetailsValid; +}; + async function main() { - const [costarResult] = await Promise.all([ - costar(), + const [googleJobsResult] = await Promise.all([ + google_jobs(), ]); - console.log("Costar result:", costarResult); + console.log("Google jobs result:", googleJobsResult); } main().catch(console.error); \ No newline at end of file diff --git a/examples/index.ts b/examples/index.ts index 2bad293c..d6ae0871 100644 --- a/examples/index.ts +++ b/examples/index.ts @@ -9,10 +9,20 @@ async function example() { debugDom: true, }); - await stagehand.init({ modelName: "claude-3-5-sonnet-20240620" }); // optionally specify model_name, defaults to "gpt-4o" (as of sept 18, 2024, we need to specify the model name with date, changing on 10/2/2024) + await stagehand.init({ modelName: "claude-3-5-sonnet-20240620" }); await stagehand.page.goto("https://www.nytimes.com/games/wordle/index.html"); - await stagehand.act({ action: "start the game" }); - await stagehand.act({ action: "close tutorial popup" }); + + const startGameResult = await stagehand.act({ action: "start the game" }); + if (!startGameResult.success) { + console.error("Failed to start the game:", startGameResult.error); + return; + } + + const closeTutorialResult = await stagehand.act({ action: "close tutorial popup" }); + if (!closeTutorialResult.success) { + console.error("Failed to close tutorial:", closeTutorialResult.error); + // Decide whether to continue or return based on the importance of this action + } let guesses: { guess: string | null; description: string | null }[] = []; for (let i = 0; i < 6; i++) { @@ -22,8 +32,13 @@ async function example() { throw new Error("no response when asking for a guess"); } - await stagehand.page.locator("body").pressSequentially(response); - await stagehand.page.keyboard.press("Enter"); + try { + await stagehand.page.locator("body").pressSequentially(response); + await stagehand.page.keyboard.press("Enter"); + } catch (error) { + console.error("Failed to input guess:", error.message); + continue; + } const guess = await stagehand.extract({ instruction: "extract the five letter guess at the bottom", diff --git a/lib/index.ts b/lib/index.ts index 9fd655e9..af15b312 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -120,30 +120,41 @@ export class Stagehand { await download.delete(); } - async init({ modelName = "gpt-4o" }: { modelName?: string } = {}) { - const { context } = await getBrowser(this.env, this.headless); - this.context = context; - this.page = context.pages()[0]; - this.defaultModelName = modelName; + async init({ modelName = "gpt-4o" }: { modelName?: string } = {}): Promise<{ success: boolean; error?: string }> { + try { + const { context } = await getBrowser(this.env, this.headless); + this.context = context; + this.page = context.pages()[0]; + this.defaultModelName = modelName; + + // Set the browser to headless mode if specified + if (this.headless) { + await this.page.setViewportSize({ width: 1280, height: 720 }); + } - // Set the browser to headless mode if specified - if (this.headless) { - await this.page.setViewportSize({ width: 1280, height: 720 }); - } + // This can be greatly improved, but the tldr is we put our built web scripts in dist, which should always + // be one level above our running directly across evals, example, and as a package + await this.page.addInitScript({ + path: path.join(__dirname, "..", "dist", "dom", "build", "process.js"), + }); - // This can be greatly improved, but the tldr is we put our built web scripts in dist, which should always - // be one level above our running directly across evals, example, and as a package - await this.page.addInitScript({ - path: path.join(__dirname, "..", "dist", "dom", "build", "process.js"), - }); + await this.page.addInitScript({ + path: path.join(__dirname, "..", "dist", "dom", "build", "utils.js"), + }); - await this.page.addInitScript({ - path: path.join(__dirname, "..", "dist", "dom", "build", "utils.js"), - }); + await this.page.addInitScript({ + path: path.join(__dirname, "..", "dist", "dom", "build", "debug.js"), + }); - await this.page.addInitScript({ - path: path.join(__dirname, "..", "dist", "dom", "build", "debug.js"), - }); + return { success: true }; + } catch (error) { + this.log({ + category: "init", + message: `Error during initialization: ${error.message}`, + level: 1, + }); + return { success: false, error: error.message }; + } } async waitForSettledDom() { @@ -200,58 +211,67 @@ export class Stagehand { content?: z.infer; chunksSeen?: Array; modelName?: string; - }): Promise> { - this.log({ - category: "extraction", - message: `starting extraction ${instruction}`, - level: 1 - }); + }): Promise<{ success: boolean; data?: z.infer; error?: string }> { + try { + this.log({ + category: "extraction", + message: `starting extraction ${instruction}`, + level: 1, + }); - await this.waitForSettledDom(); - await this.startDomDebug(); - const { outputString, chunk, chunks } = await this.page.evaluate(() => - window.processDom([]) - ); + await this.waitForSettledDom(); + await this.startDomDebug(); + const { outputString, chunk, chunks } = await this.page.evaluate(() => + window.processDom([]) + ); - const extractionResponse = await extract({ - instruction, - progress, - domElements: outputString, - llmProvider: this.llmProvider, - schema, - modelName: modelName || this.defaultModelName, - }); - const { progress: newProgress, completed, ...output } = extractionResponse; - await this.cleanupDomDebug(); + const extractionResponse = await extract({ + instruction, + progress, + domElements: outputString, + llmProvider: this.llmProvider, + schema, + modelName: modelName || this.defaultModelName, + }); + const { progress: newProgress, completed, ...output } = extractionResponse; + await this.cleanupDomDebug(); - chunksSeen.push(chunk); + chunksSeen.push(chunk); - if (completed || chunksSeen.length === chunks.length) { - this.log({ - category: "extraction", - message: `response: ${JSON.stringify(extractionResponse)}`, - level: 1 - }); + if (completed || chunksSeen.length === chunks.length) { + this.log({ + category: "extraction", + message: `response: ${JSON.stringify(extractionResponse)}`, + level: 1 + }); - return merge(content, output); - } else { + return { success: true, data: merge(content, output) }; + } else { + this.log({ + category: "extraction", + message: `continuing extraction, progress: ${progress + newProgress + ", "}`, + level: 1 + }); + return this.extract({ + instruction, + schema, + progress: progress + newProgress + ", ", + content: merge(content, output), + chunksSeen, + modelName, + }); + } + } catch (error) { this.log({ category: "extraction", - message: `continuing extraction, progress: ${progress + newProgress + ", "}`, - level: 1 - }); - return this.extract({ - instruction, - schema, - progress: progress + newProgress + ", ", - content: merge(content, output), - chunksSeen, - modelName, + message: `Error during extraction: ${error.message}`, + level: 1, }); + return { success: false, error: error.message }; } } - async observe(observation: string, modelName?: string): Promise { + async observe(observation: string, modelName?: string): Promise<{ success: boolean; result?: string; error?: string }> { this.log({ category: "observation", message: `starting observation: ${observation}`, @@ -278,7 +298,7 @@ export class Stagehand { message: `no element found for ${observation}`, level: 1 }); - return null; + return { success: false, error: `No element found for observation: ${observation}` }; } this.log({ @@ -299,20 +319,46 @@ export class Stagehand { // the locator string found by the LLM might resolve to multiple places in the DOM const firstLocator = this.page.locator(locatorString).first(); - await expect(firstLocator).toBeAttached(); + try { + await expect(firstLocator).toBeAttached(); + } catch (error) { + return { success: false, error: `Element found but not attached: ${error.message}` }; + } + const observationId = await this.recordObservation( observation, locatorString ); - return observationId; + return { success: true, result: observationId }; } - async ask(question: string, modelName?: string): Promise { - return ask({ - question, - llmProvider: this.llmProvider, - modelName: modelName || this.defaultModelName, + async ask(question: string, modelName?: string): Promise<{ success: boolean; result?: string; error?: string }> { + this.log({ + category: "ask", + message: `Asking question: ${question}`, + level: 1, }); + + try { + const response = await ask({ + question, + llmProvider: this.llmProvider, + modelName: modelName || this.defaultModelName, + }); + + if (!response) { + throw new Error("No response from LLM"); + } + + return { success: true, result: response }; + } catch (error) { + this.log({ + category: "ask", + message: `Error during ask: ${error.message}`, + level: 1, + }); + return { success: false, error: error.message }; + } } async recordObservation( @@ -344,7 +390,7 @@ export class Stagehand { steps?: string; chunksSeen?: Array; modelName?: string; - }): Promise { + }): Promise<{ success: boolean; error?: string }> { this.log({ category: "action", message: `taking action: ${action}`, @@ -390,7 +436,7 @@ export class Stagehand { level: 1 }); this.recordAction(action, null); - return; + return { success: false, error: "Action could not be performed after checking all chunks" }; } } @@ -416,22 +462,24 @@ export class Stagehand { }); const locator = await this.page.locator(`xpath=${path}`).first(); - if (method === 'scrollIntoView') { // this is not a native playwright function + if (method === 'scrollIntoView') { await locator.evaluate((element) => { element.scrollIntoView({ behavior: 'smooth', block: 'center' }); }); } else if (typeof locator[method as keyof typeof locator] === "function") { - - const isLink = await locator.evaluate((element) => { - return element.tagName.toLowerCase() === 'a' && element.hasAttribute('href'); - }); - - // Perform the action - //@ts-ignore playwright's TS does not think this is valid, but we proved it with the check above - await locator[method](...args); + try { + //@ts-ignore playwright's TS does not think this is valid, but we proved it with the check above + await locator[method](...args); + } catch (error) { + return { success: false, error: `Failed to perform ${method} on element: ${error.message}` }; + } // Check if a new page was created, but only if the method is 'click' if (method === 'click') { + const isLink = await locator.evaluate((element) => { + return element.tagName.toLowerCase() === 'a' && element.hasAttribute('href'); + }); + if (isLink) { // Create a promise that resolves when a new page is created console.log("clicking link"); @@ -452,7 +500,7 @@ export class Stagehand { } } } else { - throw new Error(`stagehand: chosen method ${method} is invalid`); + return { success: false, error: `Invalid method: ${method}` }; } if (!response.completed) { @@ -468,6 +516,8 @@ export class Stagehand { modelName, }); } + + return { success: true }; } setPage(page: Page) { this.page = page;