From ccc0088d0db247e19ea62e278ab7495e49385304 Mon Sep 17 00:00:00 2001 From: Ryno Date: Fri, 8 Nov 2024 14:59:22 +0200 Subject: [PATCH] feat: Add BraveSearchAPI document loader (#3486) --- .../BraveSearchAPI/BraveSearchAPI.ts | 141 ++++++++++++++++++ .../documentloaders/BraveSearchAPI/brave.svg | 8 + 2 files changed, 149 insertions(+) create mode 100644 packages/components/nodes/documentloaders/BraveSearchAPI/BraveSearchAPI.ts create mode 100644 packages/components/nodes/documentloaders/BraveSearchAPI/brave.svg diff --git a/packages/components/nodes/documentloaders/BraveSearchAPI/BraveSearchAPI.ts b/packages/components/nodes/documentloaders/BraveSearchAPI/BraveSearchAPI.ts new file mode 100644 index 00000000000..98ce932d14c --- /dev/null +++ b/packages/components/nodes/documentloaders/BraveSearchAPI/BraveSearchAPI.ts @@ -0,0 +1,141 @@ +import { omit } from 'lodash' +import { ICommonObject, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { BraveSearch } from '@langchain/community/tools/brave_search' +import { getCredentialData, getCredentialParam } from '../../../src/utils' +import { Document } from '@langchain/core/documents' + +class BraveSearchAPI_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'BraveSearch API Document Loader' + this.name = 'braveSearchApiLoader' + this.version = 1.0 + this.type = 'Document' + this.icon = 'brave.svg' + this.category = 'Document Loaders' + this.description = 'Load and process data from BraveSearch results' + this.baseClasses = [this.type] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: false, + credentialNames: ['braveSearchApi'] + } + this.inputs = [ + { + label: 'Query', + name: 'query', + type: 'string' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Additional Metadata', + name: 'metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, separated by comma. Use * to omit all metadata keys except the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const query = nodeData.inputs?.query as string + const metadata = nodeData.inputs?.metadata + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + + let omitMetadataKeys: string[] = [] + if (_omitMetadataKeys) { + omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) + } + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const braveApiKey = getCredentialParam('braveApiKey', credentialData, nodeData) + const loader = new BraveSearch({ apiKey: braveApiKey }) + + let docs: IDocument[] = [] + const searchResults = await loader._call(query) // Use _call method for search + const parsedResults = JSON.parse(searchResults) // Parse the JSON string to get documents + + // Format the results to match the expected Document structure + docs = parsedResults.map( + (result: any) => + new Document({ + pageContent: result.snippet, // Assuming snippet is the content + metadata: { + title: result.title, + link: result.link + } + }) + ) + + if (textSplitter) { + docs = await textSplitter.splitDocuments(docs) + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) + })) + } else { + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) + })) + } + + return docs + } +} + +module.exports = { nodeClass: BraveSearchAPI_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/BraveSearchAPI/brave.svg b/packages/components/nodes/documentloaders/BraveSearchAPI/brave.svg new file mode 100644 index 00000000000..b1e233577b7 --- /dev/null +++ b/packages/components/nodes/documentloaders/BraveSearchAPI/brave.svg @@ -0,0 +1,8 @@ + + + + + + + +