Use browserbase in github action eval

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: "20"
 
       - name: Install pnpm
         run: npm install -g pnpm
@@ -33,6 +33,8 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+          BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
           HEADLESS: true
+          EVAL_ENV: browserbase
         run: pnpm evals
         timeout-minutes: 20

evals/index.eval.ts

@@ -5,9 +5,14 @@ import { evaluateExample, chosenBananalyzerEvals } from "./bananalyzer-ts";
 import { createExpressServer } from "./bananalyzer-ts/server/expressServer";
 import process from "process";
 
+const env =
+  process.env.EVAL_ENV?.toLowerCase() === "browserbase"
+    ? "BROWSERBASE"
+    : "LOCAL";
+
 const vanta = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -17,7 +22,10 @@ const vanta = async () => {
 
   const observation = await stagehand.observe("find the request demo button");
 
-  if (!observation) return false;
+  if (!observation) {
+    await stagehand.context.close();
+    return false;
+  }
 
   const observationResult = await stagehand.page
     .locator(stagehand.observations[observation].result)
@@ -38,7 +46,7 @@ const vanta = async () => {
 
 const vanta_h = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -56,7 +64,7 @@ const vanta_h = async () => {
 
 const simple_google_search = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -69,14 +77,15 @@ const simple_google_search = async () => {
 
   const expectedUrl = "https://www.google.com/search?q=OpenAI";
   const currentUrl = await stagehand.page.url();
+
   await stagehand.context.close();
 
   return currentUrl.startsWith(expectedUrl);
 };
 
 const peeler_simple = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -97,7 +106,7 @@ const peeler_simple = async () => {
 
 const peeler_complex = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 1,
     headless: process.env.HEADLESS !== "false",
   });
@@ -123,8 +132,9 @@ const peeler_complex = async () => {
 
   return price !== null;
 };
+
 const extract_collaborators_from_github_repository = async () => {
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
+  const stagehand = new Stagehand({ env, verbose: 1 });
   await stagehand.init();
 
   const timeoutPromise = new Promise((_, reject) =>
@@ -168,7 +178,7 @@ const extract_collaborators_from_github_repository = async () => {
 };
 
 const extract_last_twenty_github_commits = async () => {
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
+  const stagehand = new Stagehand({ env, verbose: 1 });
   await stagehand.init();
 
   const timeoutPromise = new Promise((_, reject) =>
@@ -210,7 +220,7 @@ const extract_last_twenty_github_commits = async () => {
 };
 
 const twitter_signup = async () => {
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
+  const stagehand = new Stagehand({ env, verbose: 1 });
   await stagehand.init();
 
   const timeoutPromise = new Promise((_, reject) =>
@@ -246,7 +256,7 @@ const twitter_signup = async () => {
 
 const wikipedia = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 2,
     headless: process.env.HEADLESS !== "false",
   });
@@ -266,7 +276,7 @@ const wikipedia = async () => {
 
 const costar = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 2,
     debugDom: true,
     headless: process.env.HEADLESS !== "false",
@@ -314,7 +324,7 @@ const costar = async () => {
 
 const google_jobs = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 2,
     debugDom: true,
     headless: process.env.HEADLESS !== "false",