diff --git a/cofounder/api/.env b/cofounder/api/.env index 0151e62..54158f8 100644 --- a/cofounder/api/.env +++ b/cofounder/api/.env @@ -3,6 +3,7 @@ PORT = 667 OPENAI_API_KEY = "REPLACE_WITH_OPENAI_KEY" ANTHROPIC_API_KEY = "REPLACE_WITH_ANTHROPIC_KEY" COFOUNDER_API_KEY = "REPLACE_WITH_COFOUNDER.OPENINTERFACE.AI_KEY" +OPENROUTER_TOKEN = "REPLACE_WITH_OPENROUTER_TOKEN" # llm, can be 'ANTHROPIC' (for claude sonnet 3.5) or 'OPENAI' (uses diff. models for diff. passes) # make sure there are matching api keys @@ -29,4 +30,4 @@ DESIGNER_DESIGN_SYSTEM = "presets/shadcn" #"presets/shadcn" SWARM_ENABLE = TRUE # OPTIONAL -COFOUNDER_NICKNAME = "Cofounder" \ No newline at end of file +COFOUNDER_NICKNAME = "Cofounder" diff --git a/cofounder/api/utils/anthropic.js b/cofounder/api/utils/anthropic.js index 4f651a3..aeb95bb 100644 --- a/cofounder/api/utils/anthropic.js +++ b/cofounder/api/utils/anthropic.js @@ -1,5 +1,6 @@ import Anthropic from "@anthropic-ai/sdk"; import dotenv from "dotenv"; +import openrouter from "./openrouter.js"; // Import the new openrouter module dotenv.config(); const anthropic = new Anthropic(); @@ -66,6 +67,10 @@ async function inference({ messages, stream = process.stdout, }) { + if (process.env.OPENROUTER_TOKEN) { + return openrouter.inference({ model, messages, stream }); + } + // messages are in openai format , need conversion const converted = await _convertFromOpenaiFormat({ messages }); // console.dir({ "debug:utils:anthropic": {messages : converted.messages} } , {depth:null}) diff --git a/cofounder/api/utils/index.js b/cofounder/api/utils/index.js index da54126..ddfd0d4 100644 --- a/cofounder/api/utils/index.js +++ b/cofounder/api/utils/index.js @@ -6,6 +6,7 @@ import firebase from "@/utils/firebase.js"; import storage from "@/utils/storage.js"; import load from "@/utils/load.js"; import anthropic from "@/utils/anthropic.js"; +import openrouter from "@/utils/openrouter.js"; export default { parsers, @@ -16,4 +17,5 @@ export default { firebase, storage, load, + openrouter, }; diff --git a/cofounder/api/utils/openai.js b/cofounder/api/utils/openai.js index 20470af..6e3733a 100644 --- a/cofounder/api/utils/openai.js +++ b/cofounder/api/utils/openai.js @@ -1,6 +1,7 @@ import fs from "fs"; import OpenAI from "openai"; import dotenv from "dotenv"; +import openrouter from "./openrouter.js"; // Import the new openrouter module dotenv.config(); let openai; @@ -17,6 +18,10 @@ async function inference({ messages, stream = process.stdout, }) { + if (process.env.OPENROUTER_TOKEN) { + return openrouter.inference({ model, messages, stream }); + } + const streaming = await openai.chat.completions.create({ model, messages, @@ -55,6 +60,7 @@ async function inference({ usage: { model, ...usage }, }; } + async function vectorize({ texts, model = process.env.EMBEDDING_MODEL || `text-embedding-3-small`, diff --git a/cofounder/api/utils/openrouter.js b/cofounder/api/utils/openrouter.js new file mode 100644 index 0000000..2a22d15 --- /dev/null +++ b/cofounder/api/utils/openrouter.js @@ -0,0 +1,58 @@ +import fetch from "node-fetch"; + +async function inference({ model, messages, stream = process.stdout }) { + const response = await fetch("https://api.openrouter.ai/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env.OPENROUTER_TOKEN}`, + }, + body: JSON.stringify({ + model, + messages, + stream: true, + }), + }); + + if (!response.ok) { + throw new Error(`OpenRouter API request failed: ${response.statusText}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8"); + let text = ""; + let usage = {}; + let cutoff_reached = false; + let chunks_buffer = ""; + let chunks_iterator = 0; + const chunks_every = 5; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + text += chunk; + chunks_buffer += chunk; + chunks_iterator++; + if (stream?.cutoff) { + if (!cutoff_reached && text.includes(stream.cutoff)) { + cutoff_reached = true; + } + } + if (!(chunks_iterator % chunks_every)) { + stream.write(!cutoff_reached ? chunks_buffer : " ..."); + chunks_buffer = ""; + } + } + + stream.write(`\n`); + + return { + text, + usage: { model, ...usage }, + }; +} + +export default { + inference, +};