From de7ab9231120686ac14020e2523c666133b8388e Mon Sep 17 00:00:00 2001 From: Hoshino-Yumetsuki Date: Wed, 1 Jan 2025 13:15:31 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E7=A7=BB=E9=99=A4=20MongoDB=20?= =?UTF-8?q?=E4=BD=9C=E4=B8=BA=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E7=9A=84=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/vector-store-service/src/index.ts | 17 +-- .../src/locales/en-US.schema.yml | 4 - .../src/locales/zh-CN.schema.yml | 4 - .../vector-store-service/src/vectorstore.ts | 3 +- .../src/vectorstore/mongodb.ts | 127 ------------------ 5 files changed, 2 insertions(+), 153 deletions(-) delete mode 100644 packages/vector-store-service/src/vectorstore/mongodb.ts diff --git a/packages/vector-store-service/src/index.ts b/packages/vector-store-service/src/index.ts index 291eb807..7117bcc1 100644 --- a/packages/vector-store-service/src/index.ts +++ b/packages/vector-store-service/src/index.ts @@ -26,10 +26,6 @@ export interface Config extends ChatLunaPlugin.Config { milvusUrl: string milvusUsername: string milvusPassword: string - - mongodbUrl: string - mongodbDbName: string - mongodbCollectionName: string } export const Config: Schema = Schema.intersect([ @@ -39,8 +35,7 @@ export const Config: Schema = Schema.intersect([ Schema.const('faiss').description('Faiss'), Schema.const('redis').description('Redis'), Schema.const('milvus').description('Milvus'), - Schema.const('luna-vdb').description('lunavdb'), - Schema.const('mongodb').description('MongoDB Atlas') + Schema.const('luna-vdb').description('lunavdb') ]) ) .default(['luna-vdb']) @@ -57,14 +52,6 @@ export const Config: Schema = Schema.intersect([ .default('http://127.0.0.1:19530'), milvusUsername: Schema.string().default(''), milvusPassword: Schema.string().role('secret').default('') - }), - - Schema.object({ - mongodbUrl: Schema.string() - .role('url') - .default('mongodb://localhost:27017'), - mongodbDbName: Schema.string().default('chatluna'), - mongodbCollectionName: Schema.string().default('chatluna_collection') }) ]).i18n({ 'zh-CN': require('./locales/zh-CN.schema.yml'), @@ -82,8 +69,6 @@ export const usage = ` 要查看如何配置 Milvus 数据库，看[这里](https://js.langchain.com/docs/integrations/vectorstores/milvus/) -要查看如何配置 MongoDB 数据库，看[这里](https://js.langchain.com/docs/integrations/vectorstores/mongodb_atlas/) - 目前配置 Faiss 数据库安装后可能会导致 koishi 环境不安全，如果安装完成后进行某些操作完成后出现了问题（如，升级 node 版本），开发者不对此负直接责任。 ` diff --git a/packages/vector-store-service/src/locales/en-US.schema.yml b/packages/vector-store-service/src/locales/en-US.schema.yml index 2379e9f7..940b8d4e 100644 --- a/packages/vector-store-service/src/locales/en-US.schema.yml +++ b/packages/vector-store-service/src/locales/en-US.schema.yml @@ -7,7 +7,3 @@ $inner: milvusUrl: Milvus URL Address milvusUsername: Milvus Username milvusPassword: Milvus Password - - $desc: MongoDB Database Settings - mongodbUrl: MongoDB URL Address - mongodbDbName: MongoDB Database Name - mongodbCollectionName: MongoDB Collection Name diff --git a/packages/vector-store-service/src/locales/zh-CN.schema.yml b/packages/vector-store-service/src/locales/zh-CN.schema.yml index 0415aaf7..3c60191d 100644 --- a/packages/vector-store-service/src/locales/zh-CN.schema.yml +++ b/packages/vector-store-service/src/locales/zh-CN.schema.yml @@ -7,7 +7,3 @@ $inner: milvusUrl: Milvus url 地址 milvusUsername: Milvus 用户名 milvusPassword: Milvus 密码 - - $desc: MongoDB 数据库设置 - mongodbUrl: MongoDB url 地址 - mongodbDbName: MongoDB 数据库名 - mongodbCollectionName: MongoDB 集合名 diff --git a/packages/vector-store-service/src/vectorstore.ts b/packages/vector-store-service/src/vectorstore.ts index def001fe..17cf2971 100644 --- a/packages/vector-store-service/src/vectorstore.ts +++ b/packages/vector-store-service/src/vectorstore.ts @@ -5,7 +5,6 @@ import { Config } from '.' import { apply as faiss } from './vectorstore/faiss' import { apply as lunavdb } from './vectorstore/lunavdb' import { apply as milvus } from './vectorstore/milvus' -import { apply as mongodb } from './vectorstore/mongodb' import { apply as redis } from './vectorstore/redis' // import end export async function vectorStore( @@ -21,7 +20,7 @@ export async function vectorStore( const middlewares: VectorStore[] = // middleware start - [faiss, lunavdb, milvus, mongodb, redis] // middleware end + [faiss, lunavdb, milvus, redis] // middleware end for (const middleware of middlewares) { try { diff --git a/packages/vector-store-service/src/vectorstore/mongodb.ts b/packages/vector-store-service/src/vectorstore/mongodb.ts deleted file mode 100644 index 7fc079f7..00000000 --- a/packages/vector-store-service/src/vectorstore/mongodb.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { Context, Logger } from 'koishi' -import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat' -import { createLogger } from 'koishi-plugin-chatluna/utils/logger' -import { Config } from '..' -import { ChatLunaSaveableVectorStore } from 'koishi-plugin-chatluna/llm-core/model/base' - -let logger: Logger - -export async function apply( - ctx: Context, - config: Config, - plugin: ChatLunaPlugin -) { - logger = createLogger(ctx, 'chatluna-vector-store-service') - - if (!config.vectorStore.includes('mongodb')) { - return - } - - const MongoDB = await importMongoDB() - const MongoDBAtlas = await importMongoDBAtlasVectorSearch() - - plugin.registerVectorStore('mongodb', async (params) => { - const embeddings = params.embeddings - - const client = new MongoDB.MongoClient(config.mongodbUrl) - await client.connect() - - ctx.on('dispose', async () => { - await client.close() - logger.info('MongoDB connection closed') - }) - - const collection = client - .db(config.mongodbDbName) - .collection(config.mongodbCollectionName) - - const vectorStore = new MongoDBAtlas.MongoDBAtlasVectorSearch( - embeddings, - { - collection, - indexName: params.key ?? 'vector_index', - textKey: 'text', - embeddingKey: 'embedding' - } - ) - - const wrapperStore = new ChatLunaSaveableVectorStore< - typeof vectorStore - >(vectorStore, { - async deletableFunction(_store, options) { - if (options.deleteAll) { - await collection.deleteMany({}) - return - } - - const ids: string[] = [] - if (options.ids) { - ids.push(...options.ids) - } - - if (options.documents) { - const documentIds = options.documents - ?.map( - (document) => - document.metadata?.raw_id as string | undefined - ) - .filter((id): id is string => id != null) - - ids.push(...documentIds) - } - - if (ids.length > 0) { - await collection.deleteMany({ - _id: { $in: ids.map((id) => new MongoDB.ObjectId(id)) } - }) - } - }, - async addDocumentsFunction( - store, - documents, - options: { ids?: string[] } - ) { - let keys = options?.ids ?? [] - - keys = documents.map((document, i) => { - const id = keys[i] ?? crypto.randomUUID() - document.metadata = { - ...document.metadata, - raw_id: id - } - return id - }) - - await store.addDocuments(documents) - }, - async saveableFunction(_store) { - /* await client.close() - logger.info('MongoDB connection closed during save') */ - } - }) - - return wrapperStore - }) -} - -async function importMongoDB() { - try { - return await import('mongodb') - } catch (err) { - logger.error(err) - throw new Error( - 'Please install mongodb as a dependency with, e.g. `npm install -S mongodb`' - ) - } -} - -async function importMongoDBAtlasVectorSearch() { - try { - return await import('@langchain/mongodb') - } catch (err) { - logger.error(err) - throw new Error( - 'Please install mongodb-atlas-vector-search as a dependency with, e.g. `npm install -S mongodb-atlas-vector-search`' - ) - } -}