support multi-chunk extracts + prompt updates

evals/index.eval.ts

@@ -148,6 +148,81 @@ const peeler_complex = async () => {
   };
 };
 
+const homedepot = async () => {
+  const stagehand = new Stagehand({
+    env,
+    verbose: 1,
+    headless: process.env.HEADLESS !== "false",
+  });
+  await stagehand.init();
+
+  try {
+    await stagehand.page.goto("https://www.homedepot.com/");
+    await stagehand.waitForSettledDom();
+
+    await stagehand.act({ action: "search for gas grills" });
+    await stagehand.waitForSettledDom();
+
+    await stagehand.act({ action: "click on the best selling gas grill" });
+    await stagehand.waitForSettledDom();
+
+    await stagehand.act({ action: "click on the Product Details" });
+    await stagehand.waitForSettledDom();
+
+    await stagehand.act({ action: "find the Primary Burner BTU" });
+    await stagehand.waitForSettledDom();
+
+    const productSpecs = await stagehand.extract({
+      instruction: "Extract the Primary exact Burner BTU of the product",
+      schema: z.object({
+        productSpecs: z
+          .array(
+            z.object({
+              burnerBTU: z.string().describe("Primary Burner BTU exact value"),
+            }),
+          )
+          .describe("Gas grill Primary Burner BTU exact value"),
+      }),
+      modelName: "gpt-4o-2024-08-06",
+    });
+    console.log("The gas grill primary burner BTU is:", productSpecs);
+
+    if (
+      !productSpecs ||
+      !productSpecs.productSpecs ||
+      productSpecs.productSpecs.length !== 1
+    ) {
+      return {
+        _success: false,
+        productSpecs,
+      };
+    }
+
+    if (
+      (productSpecs.productSpecs[0].burnerBTU.match(/0/g) || []).length == 4 &&
+      (productSpecs.productSpecs[0].burnerBTU.match(/4/g) || []).length === 1
+    ) {
+      return {
+        _success: true,
+        productSpecs,
+      };
+    } else {
+      return {
+        _success: false,
+        productSpecs,
+      };
+    }
+  } catch (error) {
+    console.error(`Error in homedepot function: ${error.message}`);
+    return {
+      _success: false,
+      error: JSON.parse(JSON.stringify(error, null, 2)),
+    };
+  } finally {
+    await stagehand.context.close();
+  }
+};
+
 const extract_collaborators_from_github_repository = async () => {
   const stagehand = new Stagehand({
     env: "LOCAL",
@@ -459,8 +534,7 @@ const tasks = {
   extract_last_twenty_github_commits,
   costar,
   google_jobs,
-  homedepot,
-  nonsense_action
+  homedepot
 };
 
 const exactMatch = (args: { input: any; output: any; expected?: any }) => {
@@ -509,6 +583,7 @@ const testcases = [
   { input: { name: "extract_last_twenty_github_commits" } },
   // { input: { name: "costar", expected: true } },
   { input: { name: "google_jobs" } },
+  { input: { name: "homedepot" } },
   ...chosenBananalyzerEvals.map((evalItem: any) => ({
     input: {
       name: evalItem.name,

evals/playground.ts

@@ -50,7 +50,6 @@ const homedepot = async () => {
   }
 };
 
-
 async function main() {
   const homedepotResult = await homedepot();
 

lib/index.ts

@@ -52,14 +52,64 @@ async function getBrowser(env: "LOCAL" | "BROWSERBASE" = "LOCAL", headless: bool
           width: 1250,
           height: 800,
         },
-      }
+        locale: "en-US",
+        timezoneId: "America/New_York",
+        deviceScaleFactor: 1,
+        args: [
+          "--enable-webgl",
+          "--use-gl=swiftshader",
+          "--enable-accelerated-2d-canvas",
+        ],
+        excludeSwitches: "enable-automation",
+        userDataDir: "./user_data",
+      },
     );
 
     console.log("Local browser started successfully.");
+
+    await applyStealthScripts(context);
+
     return { context };
   }
 }
 
+async function applyStealthScripts(context: BrowserContext) {
+  await context.addInitScript(() => {
+    // Override the navigator.webdriver property
+    Object.defineProperty(navigator, "webdriver", {
+      get: () => undefined,
+    });
+
+    // Mock languages and plugins to mimic a real browser
+    Object.defineProperty(navigator, "languages", {
+      get: () => ["en-US", "en"],
+    });
+
+    Object.defineProperty(navigator, "plugins", {
+      get: () => [1, 2, 3, 4, 5],
+    });
+
+    // Remove Playwright-specific properties
+    delete (window as any).__playwright;
+    delete (window as any).__pw_manual;
+    delete (window as any).__PW_inspect;
+
+    // Redefine the headless property
+    Object.defineProperty(navigator, "headless", {
+      get: () => false,
+    });
+
+    // Override the permissions API
+    const originalQuery = window.navigator.permissions.query;
+    window.navigator.permissions.query = (parameters: any) =>
+      parameters.name === "notifications"
+        ? Promise.resolve({
+            state: Notification.permission,
+          } as PermissionStatus)
+        : originalQuery(parameters);
+  });
+}
+
 export class Stagehand {
   private llmProvider: LLMProvider;
   public observations: {
@@ -219,8 +269,9 @@ export class Stagehand {
 
     await this.waitForSettledDom();
     await this.startDomDebug();
-    const { outputString, chunk, chunks } = await this.page.evaluate(() =>
-      window.processDom([])
+    const { outputString, chunk, chunks } = await this.page.evaluate(
+      (chunksSeen?: number[]) => window.processDom(chunksSeen ?? []),
+      chunksSeen,
     );
     this.log({
       category: "extraction",
@@ -231,12 +282,16 @@ export class Stagehand {
     const extractionResponse = await extract({
       instruction,
       progress,
+      previouslyExtractedContent: content,
       domElements: outputString,
       llmProvider: this.llmProvider,
       schema,
       modelName: modelName || this.defaultModelName,
     });
-    const { progress: newProgress, completed, ...output } = extractionResponse;
+    const {
+      metadata: { progress: newProgress, completed },
+      ...output
+    } = extractionResponse;
     await this.cleanupDomDebug();
 
     this.log({

lib/inference.ts

@@ -62,30 +62,41 @@ export async function act({
 export async function extract({
   instruction,
   progress,
+  previouslyExtractedContent,
   domElements,
   schema,
   llmProvider,
   modelName,
 }: {
   instruction: string;
   progress: string;
+  previouslyExtractedContent: any;
   domElements: string;
   schema: z.ZodObject<any>;
   llmProvider: LLMProvider;
   modelName: string;
 }) {
   const llmClient = llmProvider.getClient(modelName);
-  
+
   const fullSchema = schema.extend({
-    progress: z.string().describe("progress of what has been extracted so far"),
-    completed: z.boolean().describe("true if the goal is now accomplished"),
+    metadata: z.object({
+      progress: z
+        .string()
+        .describe("progress of what has been extracted so far"),
+      completed: z.boolean().describe("true if the goal is now accomplished"),
+    }),
   });
 
   return llmClient.createExtraction({
     model: modelName,
     messages: [
       buildExtractSystemPrompt() as ChatMessage,
-      buildExtractUserPrompt(instruction, progress, domElements) as ChatMessage,
+      buildExtractUserPrompt(
+        instruction,
+        progress,
+        previouslyExtractedContent,
+        domElements,
+      ) as ChatMessage,
     ],
     response_model: {
       schema: fullSchema,
@@ -143,7 +154,10 @@ export async function ask({
   const llmClient = llmProvider.getClient(modelName);
   const response = await llmClient.createChatCompletion({
     model: modelName,
-    messages: [buildAskSystemPrompt() as ChatMessage, buildAskUserPrompt(question) as ChatMessage],
+    messages: [
+      buildAskSystemPrompt() as ChatMessage,
+      buildAskUserPrompt(question) as ChatMessage,
+    ],
     temperature: 0.1,
     top_p: 1,
     frequency_penalty: 0,

lib/prompt.ts

@@ -9,7 +9,7 @@ You are given:
 2. the steps that have been taken so far
 3. a list of active DOM elements in this chunk to consider to accomplish the goal. 
 
-You have 2 tools that you can call: doAction, and skipSection
+You have 2 tools that you can call: doAction, and skipSection. Do action only performs Playwright actions. Do not perform any other actions.
 `;
 
 export function buildActSystemPrompt(): OpenAI.ChatCompletionMessageParam {
@@ -104,10 +104,7 @@ export const actTools: Array<OpenAI.ChatCompletionTool> = [
 ];
 
 // extract
-const extractSystemPrompt = `
-'you are extracting content on behalf of a user. You will be given an instruction, progress so far, and a list of DOM elements to extract from',
-
-`;
+const extractSystemPrompt = `you are extracting content on behalf of a user. You will be given an instruction, progress so far, and a list of DOM elements to extract from. Where applicable, return the exact text from the DOM elements with all symbols, characters and endlines as is. Only extract new information that has not already been extracted. Make sure you include the extraction in your response. Return null or an empty string if no new information is found for a string variable`;
 
 export function buildExtractSystemPrompt(): OpenAI.ChatCompletionMessageParam {
   const content = extractSystemPrompt.replace(/\s+/g, " ");
@@ -120,12 +117,18 @@ export function buildExtractSystemPrompt(): OpenAI.ChatCompletionMessageParam {
 export function buildExtractUserPrompt(
   instruction: string,
   progress: string,
+  previouslyExtractedContent: object,
   domElements: string,
 ): OpenAI.ChatCompletionMessageParam {
   return {
     role: "user",
     content: `instruction: ${instruction}
     progress: ${progress}
+    Previously Extracted Content:\n${JSON.stringify(
+      previouslyExtractedContent,
+      null,
+      2,
+    )}
     DOM: ${domElements}`,
   };
 }