From 52206eea4ddea7efa3e2b69fc863d3ba9f304a08 Mon Sep 17 00:00:00 2001 From: shyam-raghuwanshi Date: Tue, 9 Apr 2024 15:12:19 +0530 Subject: [PATCH 1/2] adding api endpoints --- JS/edgechains/api/.gitignore | 2 + JS/edgechains/api/index.d.ts | 1 + JS/edgechains/api/index.html | 19 +++++ JS/edgechains/api/index.js | 24 ++++++ JS/edgechains/api/index.ts | 34 ++++++++ JS/edgechains/api/package.json | 23 +++++ .../api/routes/uploadPdf/uploadPdf.d.ts | 1 + .../api/routes/uploadPdf/uploadPdf.js | 27 ++++++ .../api/routes/uploadPdf/uploadPdf.ts | 32 +++++++ .../uploadToSupabase/uploadToSupabase.d.ts | 1 + .../uploadToSupabase/uploadToSupabase.js | 46 ++++++++++ .../uploadToSupabase/uploadToSupabase.ts | 62 ++++++++++++++ JS/edgechains/api/tsconfig.json | 16 ++++ JS/edgechains/api/utils/tokenBucket.d.ts | 21 +++++ JS/edgechains/api/utils/tokenBucket.js | 59 +++++++++++++ JS/edgechains/api/utils/tokenBucket.ts | 84 +++++++++++++++++++ 16 files changed, 452 insertions(+) create mode 100644 JS/edgechains/api/.gitignore create mode 100644 JS/edgechains/api/index.d.ts create mode 100644 JS/edgechains/api/index.html create mode 100644 JS/edgechains/api/index.js create mode 100644 JS/edgechains/api/index.ts create mode 100644 JS/edgechains/api/package.json create mode 100644 JS/edgechains/api/routes/uploadPdf/uploadPdf.d.ts create mode 100644 JS/edgechains/api/routes/uploadPdf/uploadPdf.js create mode 100644 JS/edgechains/api/routes/uploadPdf/uploadPdf.ts create mode 100644 JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.d.ts create mode 100644 JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.js create mode 100644 JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.ts create mode 100644 JS/edgechains/api/tsconfig.json create mode 100644 JS/edgechains/api/utils/tokenBucket.d.ts create mode 100644 JS/edgechains/api/utils/tokenBucket.js create mode 100644 JS/edgechains/api/utils/tokenBucket.ts diff --git a/JS/edgechains/api/.gitignore b/JS/edgechains/api/.gitignore new file mode 100644 index 000000000..9b3103495 --- /dev/null +++ b/JS/edgechains/api/.gitignore @@ -0,0 +1,2 @@ +node_modules +.env \ No newline at end of file diff --git a/JS/edgechains/api/index.d.ts b/JS/edgechains/api/index.d.ts new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/JS/edgechains/api/index.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/JS/edgechains/api/index.html b/JS/edgechains/api/index.html new file mode 100644 index 000000000..c3ac73e41 --- /dev/null +++ b/JS/edgechains/api/index.html @@ -0,0 +1,19 @@ + + + + + + + Document + + + +
+
+ + +
+
+ + + \ No newline at end of file diff --git a/JS/edgechains/api/index.js b/JS/edgechains/api/index.js new file mode 100644 index 000000000..6eeb8bb0e --- /dev/null +++ b/JS/edgechains/api/index.js @@ -0,0 +1,24 @@ +import { ArakooServer } from "../arakooserver/index.js"; +import { UploadPdfRouter } from "./routes/uploadPdf/uploadPdf.js"; +import { uploadToSupabaseRouter } from "./routes/uploadToSupabase/uploadToSupabase.js"; +import cluster from "node:cluster"; +import os from "node:os"; +const totalCPUs = os.cpus().length; +const server = new ArakooServer(); +if (cluster.isPrimary) { + for (let i = 0; i < totalCPUs; i++) { + cluster.fork(); + } + cluster.on('exit', (worker, code, signal) => { + console.log(`worker ${worker.process.pid} died from some region`); + }); +} +else { + const app = server.createApp(); + app.get("/", (c) => { + return c.text("Hello, from Arakoo"); + }); + app.route("/v1/uploadPdf", UploadPdfRouter); + app.route("/v1/vectorUpload", uploadToSupabaseRouter); + server.listen(5000); +} diff --git a/JS/edgechains/api/index.ts b/JS/edgechains/api/index.ts new file mode 100644 index 000000000..3a7652749 --- /dev/null +++ b/JS/edgechains/api/index.ts @@ -0,0 +1,34 @@ +import { ArakooServer } from "../arakooserver/index.js" +import { UploadPdfRouter } from "./routes/uploadPdf/uploadPdf.js" +import { uploadToSupabaseRouter } from "./routes/uploadToSupabase/uploadToSupabase.js"; +import cluster from "node:cluster" +import os from "node:os"; + +const totalCPUs = os.cpus().length; +const server = new ArakooServer(); + + +if (cluster.isPrimary) { + + for (let i = 0; i < totalCPUs; i++) { + cluster.fork(); + } + + cluster.on('exit', (worker, code, signal) => { + console.log(`worker ${worker.process.pid} died from some region`); + }); + +} else { + const app = server.createApp(); + + app.get("/", (c) => { + return c.text("Hello, from Arakoo"); + }); + + app.route("/v1/uploadPdf", UploadPdfRouter); + app.route("/v1/vectorUpload", uploadToSupabaseRouter); + + server.listen(5000) +} + + diff --git a/JS/edgechains/api/package.json b/JS/edgechains/api/package.json new file mode 100644 index 000000000..079368cc9 --- /dev/null +++ b/JS/edgechains/api/package.json @@ -0,0 +1,23 @@ +{ + "name": "api", + "version": "1.0.0", + "description": "", + "main": "index.js", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@hono/node-server": "^1.9.1", + "chalk": "^5.3.0", + "connect-multiparty": "^2.2.0", + "cors": "^2.8.5", + "dotenv": "^16.4.5", + "express": "^4.19.2", + "hono": "^4.2.2", + "retry": "^0.13.1" + } +} diff --git a/JS/edgechains/api/routes/uploadPdf/uploadPdf.d.ts b/JS/edgechains/api/routes/uploadPdf/uploadPdf.d.ts new file mode 100644 index 000000000..691af237a --- /dev/null +++ b/JS/edgechains/api/routes/uploadPdf/uploadPdf.d.ts @@ -0,0 +1 @@ +export declare const UploadPdfRouter: import("hono").Hono; diff --git a/JS/edgechains/api/routes/uploadPdf/uploadPdf.js b/JS/edgechains/api/routes/uploadPdf/uploadPdf.js new file mode 100644 index 000000000..dd76a4f59 --- /dev/null +++ b/JS/edgechains/api/routes/uploadPdf/uploadPdf.js @@ -0,0 +1,27 @@ +import { TextSplitter } from "../../../splitter/index.js"; +import { ArakooServer } from '../../../arakooserver/index.js'; +import { PdfLoader } from "../../../document-loader/index.js"; +import TokenBucket from "../../utils/tokenBucket.js"; +const server = new ArakooServer(); +const splitter = new TextSplitter(); +const loader = new PdfLoader(); +const bucket = new TokenBucket(10, 1, 1); +export const UploadPdfRouter = server.createApp(); +UploadPdfRouter.post("/", async (c) => { + try { + if (!bucket.handleRequest("uploadPdf")) { + return c.json({ error: "Rate limit exceeded" }); + } + const body = await c.req.parseBody(); + const text = body['file']; + // @ts-ignore + const buffer = await text?.arrayBuffer(); + const docs = await loader.loadPdf(buffer); + const chunks = await splitter.splitTextIntoChunks(docs, 1000); + return c.json({ chunks }); + } + catch (error) { + console.log(error); + return c.json({ error: "An error occurred while processing the file." }); + } +}); diff --git a/JS/edgechains/api/routes/uploadPdf/uploadPdf.ts b/JS/edgechains/api/routes/uploadPdf/uploadPdf.ts new file mode 100644 index 000000000..91a99b1c2 --- /dev/null +++ b/JS/edgechains/api/routes/uploadPdf/uploadPdf.ts @@ -0,0 +1,32 @@ +import { TextSplitter } from "../../../splitter/index.js"; +import { ArakooServer } from '../../../arakooserver/index.js'; +import { PdfLoader } from "../../../document-loader/index.js" +import TokenBucket from "../../utils/tokenBucket.js"; + +const server = new ArakooServer(); +const splitter = new TextSplitter(); +const loader = new PdfLoader(); +const bucket = new TokenBucket(10, 1, 1); + +export const UploadPdfRouter = server.createApp(); + + +UploadPdfRouter.post("/", async (c) => { + try { + if (!bucket.handleRequest("uploadPdf")) { + return c.json({ error: "Rate limit exceeded" }); + } + const body = await c.req.parseBody() + const text = body['file'] + + // @ts-ignore + const buffer = await text?.arrayBuffer(); + const docs = await loader.loadPdf(buffer) + const chunks = await splitter.splitTextIntoChunks(docs, 1000); + return c.json({ chunks }) + + } catch (error) { + console.log(error); + return c.json({ error: "An error occurred while processing the file." }); + } +}); diff --git a/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.d.ts b/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.d.ts new file mode 100644 index 000000000..5d6027ab3 --- /dev/null +++ b/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.d.ts @@ -0,0 +1 @@ +export declare const uploadToSupabaseRouter: import("hono").Hono; diff --git a/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.js b/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.js new file mode 100644 index 000000000..e313c2a11 --- /dev/null +++ b/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.js @@ -0,0 +1,46 @@ +import { ArakooServer } from "../../../arakooserver/index.js"; +import { Supabase } from "../../../supabase/index.js"; +import { config } from "dotenv"; +import TokenBucket from "../../utils/tokenBucket.js"; +config(); +const server = new ArakooServer(); +const bucket = new TokenBucket(4, 1, 1); +const supabase = new Supabase(process.env.SUPABASE_URL, process.env.SUPABASE_API_KEY); +const client = supabase.createClient(); +export const uploadToSupabaseRouter = server.createApp(); +uploadToSupabaseRouter.post("/", async (c) => { + try { + if (!bucket.handleRequest("uploadToSupabase")) { + return c.json({ statusCode: 429, message: "Rate limit exceeded" }); + } + const body = await JSON.parse(await c.req.text()); + const { embeddings, content, tableName } = body; + if (!embeddings) { + return c.json({ statusCode: 400, message: "embeddings is required" }); + } + if (!content) { + return c.json({ statusCode: 400, message: "content is required" }); + } + if (!tableName) { + return c.json({ statusCode: 400, message: "tableName is required" }); + } + for (let i = 0; i < embeddings.length; i++) { + if (content[i].length <= 1) { + continue; + } + const element = embeddings[i].embedding; + const res = await supabase.insertVectorData({ + client, + tableName, + content: content[i], + embedding: element + }); + console.log(res); + } + return c.json({ statusCode: 200, message: `successfully uploaded to ${tableName}` }); + } + catch (error) { + console.error("Error uploading to supabase:", error); + return c.json({ statusCode: 500, message: "Error uploading to supabase" }); + } +}); diff --git a/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.ts b/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.ts new file mode 100644 index 000000000..1904d4a65 --- /dev/null +++ b/JS/edgechains/api/routes/uploadToSupabase/uploadToSupabase.ts @@ -0,0 +1,62 @@ +import { ArakooServer } from "../../../arakooserver/index.js"; +import { Supabase } from "../../../supabase/index.js"; +import { config } from "dotenv" +import TokenBucket from "../../utils/tokenBucket.js"; +config() + +const server = new ArakooServer(); +const bucket = new TokenBucket(4, 1, 1); +const supabase = new Supabase(process.env.SUPABASE_URL!, process.env.SUPABASE_API_KEY!) + +const client = supabase.createClient() + +export const uploadToSupabaseRouter = server.createApp(); + +interface BodyType { + embeddings: Array<{ embedding: Array }>, + content: Array, + tableName: string +} + +uploadToSupabaseRouter.post("/", async (c) => { + try { + if (!bucket.handleRequest("uploadToSupabase")) { + return c.json({ statusCode: 429, message: "Rate limit exceeded" }) + } + const body = await JSON.parse(await c.req.text()); + const { embeddings, content, tableName } = body as BodyType + + if (!embeddings) { + return c.json({ statusCode: 400, message: "embeddings is required" }) + } + if (!content) { + return c.json({ statusCode: 400, message: "content is required" }) + } + if (!tableName) { + return c.json({ statusCode: 400, message: "tableName is required" }) + } + + for (let i = 0; i < embeddings.length; i++) { + if (content[i].length <= 1) { + continue; + } + + const element = embeddings[i].embedding; + + const res = await supabase.insertVectorData({ + client, + tableName, + content: content[i], + embedding: element + }) + console.log(res) + } + + return c.json({ statusCode: 200, message: `successfully uploaded to ${tableName}` }) + + } catch (error) { + console.error("Error uploading to supabase:", error); + return c.json({ statusCode: 500, message: "Error uploading to supabase" }) + } + +}) \ No newline at end of file diff --git a/JS/edgechains/api/tsconfig.json b/JS/edgechains/api/tsconfig.json new file mode 100644 index 000000000..98423f3ca --- /dev/null +++ b/JS/edgechains/api/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "types": ["dotenv/config", "jest", "node"], + "target": "ES2022", + "module": "NodeNext", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "jsx": "react-jsx", + "jsxImportSource": "hono/jsx", + "noImplicitAny": false, + "moduleResolution": "NodeNext", + "declaration": true + } +} diff --git a/JS/edgechains/api/utils/tokenBucket.d.ts b/JS/edgechains/api/utils/tokenBucket.d.ts new file mode 100644 index 000000000..72a5db2bb --- /dev/null +++ b/JS/edgechains/api/utils/tokenBucket.d.ts @@ -0,0 +1,21 @@ +export default class TokenBucket { + capacity: number; + refillTime: number; + refillAmount: number; + db: { + [key: string]: { + tokens: number; + ts: number; + }; + }; + constructor(capacity: number, refillAmount: number, refillTime: number); + refillBucket(key: any): { + tokens: number; + ts: number; + } | null; + createBucket(key: any): { + tokens: number; + ts: number; + }; + handleRequest(key: any): boolean; +} diff --git a/JS/edgechains/api/utils/tokenBucket.js b/JS/edgechains/api/utils/tokenBucket.js new file mode 100644 index 000000000..3a511e96d --- /dev/null +++ b/JS/edgechains/api/utils/tokenBucket.js @@ -0,0 +1,59 @@ +import chalk from 'chalk'; +export default class TokenBucket { + capacity; + refillTime; + refillAmount; + db; + constructor(capacity, refillAmount, refillTime) { + this.capacity = capacity; + this.refillTime = refillTime; // amount of time between refills (in sec) + this.refillAmount = refillAmount; // number of tokens to add per refill cycle + this.db = {}; + } + refillBucket(key) { + if (this.db[key] === undefined) + return null; + const { tokens, ts } = this.db[key]; + const currentTime = Date.now(); + const elapsedTime = Math.floor((currentTime - ts) / (this.refillTime * 1000) // convert to seconds + ); + const newTokens = elapsedTime * this.refillAmount; + this.db[key] = { + tokens: Math.min(this.capacity, tokens + newTokens), + ts: currentTime, + }; + return this.db[key]; + } + createBucket(key) { + if (this.db[key] === undefined) { + this.db[key] = { + tokens: this.capacity, + ts: Date.now(), + }; + } + return this.db[key]; + } + handleRequest(key) { + let bucket = this.createBucket(key); + const currentTime = Date.now(); + // check if the time elapsed since the (convert to seconds) + const elapsedTime = Math.floor((currentTime - bucket.ts) / 1000); + if (elapsedTime >= this.refillTime) { + //@ts-ignore + bucket = this.refillBucket(key); + } + else { + if (bucket?.tokens <= 0) { + console.log(chalk.red(`Request[REJECTED] for ${key} (tokens - ${bucket.tokens}) -- ${new Date().toLocaleTimeString()}\n`)); + return false; + } + } + if (!bucket) { + chalk.red(`Request[REJECTED] for ${key} -- ${new Date().toLocaleTimeString()} -- BUCKET NOT FOUND\n`); + return false; + } + console.log(chalk.green(`Request[ACCEPTED] for ${key} (tokens - ${bucket.tokens}) -- ${new Date().toLocaleTimeString()}\n`)); + bucket.tokens -= 1; + return true; + } +} diff --git a/JS/edgechains/api/utils/tokenBucket.ts b/JS/edgechains/api/utils/tokenBucket.ts new file mode 100644 index 000000000..e9b25a77f --- /dev/null +++ b/JS/edgechains/api/utils/tokenBucket.ts @@ -0,0 +1,84 @@ +import chalk from 'chalk'; + +export default class TokenBucket { + capacity: number; + refillTime: number; + refillAmount: number; + db: { [key: string]: { tokens: number; ts: number } }; + + constructor(capacity:number, refillAmount:number, refillTime:number) { + this.capacity = capacity; + this.refillTime = refillTime; // amount of time between refills (in sec) + this.refillAmount = refillAmount; // number of tokens to add per refill cycle + this.db = {}; + } + + refillBucket(key) { + if (this.db[key] === undefined) return null; + + const { tokens, ts } = this.db[key]; + const currentTime = Date.now(); + const elapsedTime = Math.floor( + (currentTime - ts) / (this.refillTime * 1000) // convert to seconds + ); + + const newTokens = elapsedTime * this.refillAmount; + + this.db[key] = { + tokens: Math.min(this.capacity, tokens + newTokens), + ts: currentTime, + }; + + return this.db[key]; + } + + createBucket(key) { + if (this.db[key] === undefined) { + this.db[key] = { + tokens: this.capacity, + ts: Date.now(), + }; + } + + return this.db[key]; + } + + handleRequest(key) { + let bucket = this.createBucket(key); + const currentTime = Date.now(); + + // check if the time elapsed since the (convert to seconds) + const elapsedTime = Math.floor((currentTime - bucket.ts) / 1000); + + if (elapsedTime >= this.refillTime) { + //@ts-ignore + bucket = this.refillBucket(key) + } else { + if (bucket?.tokens <= 0) { + console.log( + chalk.red( + `Request[REJECTED] for ${key} (tokens - ${bucket.tokens + }) -- ${new Date().toLocaleTimeString()}\n` + ) + ); + return false; + } + } + + if (!bucket) { + chalk.red( + `Request[REJECTED] for ${key} -- ${new Date().toLocaleTimeString()} -- BUCKET NOT FOUND\n` + ); + return false; + } + + console.log( + chalk.green( + `Request[ACCEPTED] for ${key} (tokens - ${bucket.tokens + }) -- ${new Date().toLocaleTimeString()}\n` + ) + ); + bucket.tokens -= 1; + return true; + } +} \ No newline at end of file From 45c7ac783ca16181daf72b80d16ea77a26d2f1e3 Mon Sep 17 00:00:00 2001 From: Shyam Raghuwanshi Date: Tue, 9 Apr 2024 15:24:46 +0530 Subject: [PATCH 2/2] Delete JS/edgechains/api/index.html --- JS/edgechains/api/index.html | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 JS/edgechains/api/index.html diff --git a/JS/edgechains/api/index.html b/JS/edgechains/api/index.html deleted file mode 100644 index c3ac73e41..000000000 --- a/JS/edgechains/api/index.html +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - Document - - - -
-
- - -
-
- - - \ No newline at end of file