adding api endpoints

JS/edgechains/api/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.env

JS/edgechains/api/index.d.ts

@@ -0,0 +1 @@
+export {};

JS/edgechains/api/index.js

@@ -0,0 +1,24 @@
+import { ArakooServer } from "../arakooserver/index.js";
+import { UploadPdfRouter } from "./routes/uploadPdf/uploadPdf.js";
+import { uploadToSupabaseRouter } from "./routes/uploadToSupabase/uploadToSupabase.js";
+import cluster from "node:cluster";
+import os from "node:os";
+const totalCPUs = os.cpus().length;
+const server = new ArakooServer();
+if (cluster.isPrimary) {
+    for (let i = 0; i < totalCPUs; i++) {
+        cluster.fork();
+    }
+    cluster.on('exit', (worker, code, signal) => {
+        console.log(`worker ${worker.process.pid} died from some region`);
+    });
+}
+else {
+    const app = server.createApp();
+    app.get("/", (c) => {
+        return c.text("Hello, from Arakoo");
+    });
+    app.route("/v1/uploadPdf", UploadPdfRouter);
+    app.route("/v1/vectorUpload", uploadToSupabaseRouter);
+    server.listen(5000);
+}

JS/edgechains/api/index.ts

@@ -0,0 +1,34 @@
+import { ArakooServer } from "../arakooserver/index.js"
+import { UploadPdfRouter } from "./routes/uploadPdf/uploadPdf.js"
+import { uploadToSupabaseRouter } from "./routes/uploadToSupabase/uploadToSupabase.js";
+import cluster from "node:cluster"
+import os from "node:os";
+
+const totalCPUs = os.cpus().length;
+const server = new ArakooServer();
+
+
+if (cluster.isPrimary) {
+
+  for (let i = 0; i < totalCPUs; i++) {
+    cluster.fork();
+  }
+
+  cluster.on('exit', (worker, code, signal) => {
+    console.log(`worker ${worker.process.pid} died from some region`);
+  });
+
+} else {
+  const app = server.createApp();
+
+  app.get("/", (c) => {
+    return c.text("Hello, from Arakoo");
+  });
+
+  app.route("/v1/uploadPdf", UploadPdfRouter);
+  app.route("/v1/vectorUpload", uploadToSupabaseRouter);
+
+  server.listen(5000)
+}
+
+

JS/edgechains/api/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "api",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@hono/node-server": "^1.9.1",
+    "chalk": "^5.3.0",
+    "connect-multiparty": "^2.2.0",
+    "cors": "^2.8.5",
+    "dotenv": "^16.4.5",
+    "express": "^4.19.2",
+    "hono": "^4.2.2",
+    "retry": "^0.13.1"
+  }
+}

JS/edgechains/api/routes/uploadPdf/uploadPdf.d.ts

@@ -0,0 +1 @@
+export declare const UploadPdfRouter: import("hono").Hono<import("hono").Env, import("hono/types").BlankSchema, "/">;

JS/edgechains/api/routes/uploadPdf/uploadPdf.js

@@ -0,0 +1,27 @@
+import { TextSplitter } from "../../../splitter/index.js";
+import { ArakooServer } from '../../../arakooserver/index.js';
+import { PdfLoader } from "../../../document-loader/index.js";
+import TokenBucket from "../../utils/tokenBucket.js";
+const server = new ArakooServer();
+const splitter = new TextSplitter();
+const loader = new PdfLoader();
+const bucket = new TokenBucket(10, 1, 1);
+export const UploadPdfRouter = server.createApp();
+UploadPdfRouter.post("/", async (c) => {
+    try {
+        if (!bucket.handleRequest("uploadPdf")) {
+            return c.json({ error: "Rate limit exceeded" });
+        }
+        const body = await c.req.parseBody();
+        const text = body['file'];
+        // @ts-ignore
+        const buffer = await text?.arrayBuffer();
+        const docs = await loader.loadPdf(buffer);
+        const chunks = await splitter.splitTextIntoChunks(docs, 1000);
+        return c.json({ chunks });
+    }
+    catch (error) {
+        console.log(error);
+        return c.json({ error: "An error occurred while processing the file." });
+    }
+});

JS/edgechains/api/routes/uploadPdf/uploadPdf.ts

@@ -0,0 +1,32 @@
+import { TextSplitter } from "../../../splitter/index.js";
+import { ArakooServer } from '../../../arakooserver/index.js';
+import { PdfLoader } from "../../../document-loader/index.js"
+import TokenBucket from "../../utils/tokenBucket.js";
+
+const server = new ArakooServer();
+const splitter = new TextSplitter();
+const loader = new PdfLoader();
+const bucket = new TokenBucket(10, 1, 1);
+
+export const UploadPdfRouter = server.createApp();
+
+
+UploadPdfRouter.post("/", async (c) => {
+    try {
+        if (!bucket.handleRequest("uploadPdf")) {
+            return c.json({ error: "Rate limit exceeded" });
+        }
+        const body = await c.req.parseBody()
+        const text = body['file']
+
+        // @ts-ignore
+        const buffer = await text?.arrayBuffer();
+        const docs = await loader.loadPdf(buffer)
+        const chunks = await splitter.splitTextIntoChunks(docs, 1000);
+        return c.json({ chunks })
+
+    } catch (error) {
+        console.log(error);
+        return c.json({ error: "An error occurred while processing the file." });
+    }
+});

JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.d.ts

@@ -0,0 +1 @@
+export declare const uploadToSupabaseRouter: import("hono").Hono<import("hono").Env, import("hono/types").BlankSchema, "/">;

JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.js

@@ -0,0 +1,46 @@
+import { ArakooServer } from "../../../arakooserver/index.js";
+import { Supabase } from "../../../supabase/index.js";
+import { config } from "dotenv";
+import TokenBucket from "../../utils/tokenBucket.js";
+config();
+const server = new ArakooServer();
+const bucket = new TokenBucket(4, 1, 1);
+const supabase = new Supabase(process.env.SUPABASE_URL, process.env.SUPABASE_API_KEY);
+const client = supabase.createClient();
+export const uploadToSupabaseRouter = server.createApp();
+uploadToSupabaseRouter.post("/", async (c) => {
+    try {
+        if (!bucket.handleRequest("uploadToSupabase")) {
+            return c.json({ statusCode: 429, message: "Rate limit exceeded" });
+        }
+        const body = await JSON.parse(await c.req.text());
+        const { embeddings, content, tableName } = body;
+        if (!embeddings) {
+            return c.json({ statusCode: 400, message: "embeddings is required" });
+        }
+        if (!content) {
+            return c.json({ statusCode: 400, message: "content is required" });
+        }
+        if (!tableName) {
+            return c.json({ statusCode: 400, message: "tableName is required" });
+        }
+        for (let i = 0; i < embeddings.length; i++) {
+            if (content[i].length <= 1) {
+                continue;
+            }
+            const element = embeddings[i].embedding;
+            const res = await supabase.insertVectorData({
+                client,
+                tableName,
+                content: content[i],
+                embedding: element
+            });
+            console.log(res);
+        }
+        return c.json({ statusCode: 200, message: `successfully uploaded to ${tableName}` });
+    }
+    catch (error) {
+        console.error("Error uploading to supabase:", error);
+        return c.json({ statusCode: 500, message: "Error uploading to supabase" });
+    }
+});

JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.ts

@@ -0,0 +1,62 @@
+import { ArakooServer } from "../../../arakooserver/index.js";
+import { Supabase } from "../../../supabase/index.js";
+import { config } from "dotenv"
+import TokenBucket from "../../utils/tokenBucket.js";
+config()
+
+const server = new ArakooServer();
+const bucket = new TokenBucket(4, 1, 1);
+const supabase = new Supabase(process.env.SUPABASE_URL!, process.env.SUPABASE_API_KEY!)
+
+const client = supabase.createClient()
+
+export const uploadToSupabaseRouter = server.createApp();
+
+interface BodyType {
+    embeddings: Array<{ embedding: Array<number> }>,
+    content: Array<string>,
+    tableName: string
+}
+
+uploadToSupabaseRouter.post("/", async (c) => {
+    try {
+        if (!bucket.handleRequest("uploadToSupabase")) {
+            return c.json({ statusCode: 429, message: "Rate limit exceeded" })
+        }
+        const body = await JSON.parse(await c.req.text());
+        const { embeddings, content, tableName } = body as BodyType
+
+        if (!embeddings) {
+            return c.json({ statusCode: 400, message: "embeddings is required" })
+        }
+        if (!content) {
+            return c.json({ statusCode: 400, message: "content is required" })
+        }
+        if (!tableName) {
+            return c.json({ statusCode: 400, message: "tableName is required" })
+        }
+
+        for (let i = 0; i < embeddings.length; i++) {
+            if (content[i].length <= 1) {
+                continue;
+            }
+
+            const element = embeddings[i].embedding;
+
+            const res = await supabase.insertVectorData({
+                client,
+                tableName,
+                content: content[i],
+                embedding: element
+            })
+            console.log(res)
+        }
+
+        return c.json({ statusCode: 200, message: `successfully uploaded to ${tableName}` })
+
+    } catch (error) {
+        console.error("Error uploading to supabase:", error);
+        return c.json({ statusCode: 500, message: "Error uploading to supabase" })
+    }
+
+})

JS/edgechains/api/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+      "types": ["dotenv/config", "jest", "node"],
+      "target": "ES2022",
+      "module": "NodeNext",
+      "esModuleInterop": true,
+      "forceConsistentCasingInFileNames": true,
+      "strict": true,
+      "skipLibCheck": true,
+      "jsx": "react-jsx",
+      "jsxImportSource": "hono/jsx",
+      "noImplicitAny": false,
+      "moduleResolution": "NodeNext",
+      "declaration": true
+  }
+}

JS/edgechains/api/utils/tokenBucket.d.ts

@@ -0,0 +1,21 @@
+export default class TokenBucket {
+    capacity: number;
+    refillTime: number;
+    refillAmount: number;
+    db: {
+        [key: string]: {
+            tokens: number;
+            ts: number;
+        };
+    };
+    constructor(capacity: number, refillAmount: number, refillTime: number);
+    refillBucket(key: any): {
+        tokens: number;
+        ts: number;
+    } | null;
+    createBucket(key: any): {
+        tokens: number;
+        ts: number;
+    };
+    handleRequest(key: any): boolean;
+}

JS/edgechains/api/utils/tokenBucket.js

@@ -0,0 +1,59 @@
+import chalk from 'chalk';
+export default class TokenBucket {
+    capacity;
+    refillTime;
+    refillAmount;
+    db;
+    constructor(capacity, refillAmount, refillTime) {
+        this.capacity = capacity;
+        this.refillTime = refillTime; // amount of time between refills (in sec)
+        this.refillAmount = refillAmount; // number of tokens to add per refill cycle
+        this.db = {};
+    }
+    refillBucket(key) {
+        if (this.db[key] === undefined)
+            return null;
+        const { tokens, ts } = this.db[key];
+        const currentTime = Date.now();
+        const elapsedTime = Math.floor((currentTime - ts) / (this.refillTime * 1000) // convert to seconds
+        );
+        const newTokens = elapsedTime * this.refillAmount;
+        this.db[key] = {
+            tokens: Math.min(this.capacity, tokens + newTokens),
+            ts: currentTime,
+        };
+        return this.db[key];
+    }
+    createBucket(key) {
+        if (this.db[key] === undefined) {
+            this.db[key] = {
+                tokens: this.capacity,
+                ts: Date.now(),
+            };
+        }
+        return this.db[key];
+    }
+    handleRequest(key) {
+        let bucket = this.createBucket(key);
+        const currentTime = Date.now();
+        // check if the time elapsed since the (convert to seconds)
+        const elapsedTime = Math.floor((currentTime - bucket.ts) / 1000);
+        if (elapsedTime >= this.refillTime) {
+            //@ts-ignore
+            bucket = this.refillBucket(key);
+        }
+        else {
+            if (bucket?.tokens <= 0) {
+                console.log(chalk.red(`Request[REJECTED] for ${key} (tokens - ${bucket.tokens}) -- ${new Date().toLocaleTimeString()}\n`));
+                return false;
+            }
+        }
+        if (!bucket) {
+            chalk.red(`Request[REJECTED] for ${key} -- ${new Date().toLocaleTimeString()} -- BUCKET NOT FOUND\n`);
+            return false;
+        }
+        console.log(chalk.green(`Request[ACCEPTED] for ${key} (tokens - ${bucket.tokens}) -- ${new Date().toLocaleTimeString()}\n`));
+        bucket.tokens -= 1;
+        return true;
+    }
+}