Resolved merge conflicts between feat/added-anthropic-vision-api and …

…main

apps/api/.env.local

@@ -0,0 +1,14 @@
+NUM_WORKERS_PER_QUEUE=8
+PORT=
+HOST=
+SUPABASE_ANON_TOKEN=
+SUPABASE_URL=
+SUPABASE_SERVICE_TOKEN=
+REDIS_URL=
+SCRAPING_BEE_API_KEY=
+OPENAI_API_KEY=
+ANTHROPIC_API_KEY=
+BULL_AUTH_KEY=
+LOGTAIL_KEY=
+PLAYWRIGHT_MICROSERVICE_URL=
+

apps/api/package.json

@@ -41,6 +41,7 @@
     "typescript": "^5.4.2"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.20.5",
     "@brillout/import": "^0.2.2",
     "@bull-board/api": "^5.14.2",
     "@bull-board/express": "^5.8.0",

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -193,7 +193,7 @@ const TEST_URL = "http://127.0.0.1:3002";
         expect(response.body).toHaveProperty("success");
         expect(response.body.success).toBe(true);
         expect(response.body).toHaveProperty("data");
-      }, 20000); 
+      }, 30000); // 30 seconds timeout
     });
 
     describe("GET /v0/crawl/status/:jobId", () => {

apps/api/src/scraper/WebScraper/index.ts

@@ -4,11 +4,10 @@ import { scrapSingleUrl } from "./single_url";
 import { SitemapEntry, fetchSitemapData, getLinksFromSitemap } from "./sitemap";
 import { WebCrawler } from "./crawler";
 import { getValue, setValue } from "../../services/redis";
-import { getImageDescription } from "./utils/gptVision";
+import { getImageDescription } from "./utils/imageDescription";
 import { fetchAndProcessPdf } from "./utils/pdfProcessor";
 import { replaceImgPathsWithAbsolutePaths, replacePathsWithAbsolutePaths } from "./utils/replacePaths";
 
-
 export class WebScraperDataProvider {
   private urls: string[] = [""];
   private mode: "single_urls" | "sitemap" | "crawl" = "single_urls";
@@ -21,6 +20,7 @@ export class WebScraperDataProvider {
   private generateImgAltText: boolean = false;
   private pageOptions?: PageOptions;
   private replaceAllPathsWithAbsolutePaths?: boolean = false;
+  private generateImgAltTextModel: "gpt-4-turbo" | "claude-3-opus" = "gpt-4-turbo";
 
   authorize(): void {
     throw new Error("Method not implemented.");
@@ -443,7 +443,7 @@ export class WebScraperDataProvider {
                 imageUrl,
                 backText,
                 frontText
-              );
+              , this.generateImgAltTextModel);
             }
 
             document.content = document.content.replace(

apps/api/src/scraper/WebScraper/utils/imageDescription.ts

@@ -0,0 +1,88 @@
+import Anthropic from '@anthropic-ai/sdk';
+import axios from 'axios';
+
+export async function getImageDescription(
+  imageUrl: string,
+  backText: string,
+  frontText: string,
+  model: string = "gpt-4-turbo"
+): Promise<string> {
+  try {
+    const prompt = "What's in the image? You need to answer with the content for the alt tag of the image. To help you with the context, the image is in the following text: " +
+      backText +
+      " and the following text: " +
+      frontText +
+      ". Be super concise."
+
+    switch (model) {
+      case 'claude-3-opus': {
+        if (!process.env.ANTHROPIC_API_KEY) {
+          throw new Error("No Anthropic API key provided");
+        }
+        const imageRequest = await axios.get(imageUrl, { responseType: 'arraybuffer' });
+        const imageMediaType = 'image/png';
+        const imageData = Buffer.from(imageRequest.data, 'binary').toString('base64');
+
+        const anthropic = new Anthropic();
+        const response = await anthropic.messages.create({
+          model: "claude-3-opus-20240229",
+          max_tokens: 1024,
+          messages: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "image",
+                  source: {
+                    type: "base64",
+                    media_type: imageMediaType,
+                    data: imageData,
+                  },
+                },
+                {
+                  type: "text",
+                  text: prompt
+                }
+              ],
+            }
+          ]
+        });
+
+        return response.content[0].text;
+      }
+      default: {
+        if (!process.env.OPENAI_API_KEY) {
+          throw new Error("No OpenAI API key provided");
+        }
+
+        const { OpenAI } = require("openai");
+        const openai = new OpenAI();
+
+        const response = await openai.chat.completions.create({
+          model: "gpt-4-turbo",
+          messages: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "text",
+                  text: prompt,
+                },
+                {
+                  type: "image_url",
+                  image_url: {
+                    url: imageUrl,
+                  },
+                },
+              ],
+            },
+          ],
+        });
+        return response.choices[0].message.content;
+      }
+    }
+  } catch (error) {
+    console.error("Error generating image alt text:", error?.message);
+    return "";
+  }
+}