diff --git a/docs/api_refs/blacklisted-entrypoints.json b/docs/api_refs/blacklisted-entrypoints.json
index 419d8800827d..e1b4fa28e3a6 100644
--- a/docs/api_refs/blacklisted-entrypoints.json
+++ b/docs/api_refs/blacklisted-entrypoints.json
@@ -15,6 +15,7 @@
"../../langchain/src/embeddings/tensorflow.ts",
"../../langchain/src/embeddings/hf.ts",
"../../langchain/src/embeddings/hf_transformers.ts",
+ "../../langchain/src/embeddings/huggingface_transformers.ts",
"../../langchain/src/embeddings/googlevertexai.ts",
"../../langchain/src/embeddings/googlepalm.ts",
"../../langchain/src/embeddings/minimax.ts",
diff --git a/docs/core_docs/docs/integrations/document_loaders/file_loaders/docx.mdx b/docs/core_docs/docs/integrations/document_loaders/file_loaders/docx.mdx
index baaf464a5e5b..8e46cde7a1b8 100644
--- a/docs/core_docs/docs/integrations/document_loaders/file_loaders/docx.mdx
+++ b/docs/core_docs/docs/integrations/document_loaders/file_loaders/docx.mdx
@@ -4,17 +4,38 @@ hide_table_of_contents: true
# Docx files
-This example goes over how to load data from docx files.
+The `DocxLoader` allows you to extract text data from Microsoft Word documents. It supports both the modern `.docx` format and the legacy `.doc` format. Depending on the file type, additional dependencies are required.
-# Setup
+---
+
+## Setup
+
+To use `DocxLoader`, you'll need the `@langchain/community` integration along with either `mammoth` or `word-extractor` package:
+
+- **`mammoth`**: For processing `.docx` files.
+- **`word-extractor`**: For handling `.doc` files.
+
+### Installation
+
+#### For `.docx` Files
```bash npm2yarn
npm install @langchain/community @langchain/core mammoth
```
-# Usage
+#### For `.doc` Files
+
+```bash npm2yarn
+npm install @langchain/community @langchain/core word-extractor
+```
+
+## Usage
+
+### Loading `.docx` Files
-```typescript
+For `.docx` files, there is no need to explicitly specify any parameters when initializing the loader:
+
+```javascript
import { DocxLoader } from "@langchain/community/document_loaders/fs/docx";
const loader = new DocxLoader(
@@ -23,3 +44,20 @@ const loader = new DocxLoader(
const docs = await loader.load();
```
+
+### Loading `.doc` Files
+
+For `.doc` files, you must explicitly specify the `type` as `doc` when initializing the loader:
+
+```javascript
+import { DocxLoader } from "@langchain/community/document_loaders/fs/docx";
+
+const loader = new DocxLoader(
+ "src/document_loaders/tests/example_data/attention.doc",
+ {
+ type: "doc",
+ }
+);
+
+const docs = await loader.load();
+```
diff --git a/docs/core_docs/docs/integrations/document_loaders/web_loaders/jira.mdx b/docs/core_docs/docs/integrations/document_loaders/web_loaders/jira.mdx
new file mode 100644
index 000000000000..fc77f4a14ab1
--- /dev/null
+++ b/docs/core_docs/docs/integrations/document_loaders/web_loaders/jira.mdx
@@ -0,0 +1,23 @@
+---
+sidebar_class_name: node-only
+---
+
+# Jira
+
+:::tip Compatibility
+Only available on Node.js.
+:::
+
+This covers how to load document objects from issues in a Jira projects.
+
+## Credentials
+
+- You'll need to set up an access token and provide it along with your Jira username in order to authenticate the request
+- You'll also need the project key and host URL for the project containing the issues to load as documents.
+
+## Usage
+
+import CodeBlock from "@theme/CodeBlock";
+import Example from "@examples/document_loaders/jira.ts";
+
+{Example}
diff --git a/docs/core_docs/docs/integrations/text_embedding/transformers.mdx b/docs/core_docs/docs/integrations/text_embedding/transformers.mdx
index dc75291a39de..97f08776ab6a 100644
--- a/docs/core_docs/docs/integrations/text_embedding/transformers.mdx
+++ b/docs/core_docs/docs/integrations/text_embedding/transformers.mdx
@@ -8,10 +8,15 @@ It runs locally and even works directly in the browser, allowing you to create w
## Setup
-You'll need to install the [@xenova/transformers](https://www.npmjs.com/package/@xenova/transformers) package as a peer dependency:
+You'll need to install the [@huggingface/transformers](https://www.npmjs.com/package/@huggingface/transformers) package as a peer dependency:
+
+:::tip Compatibility
+If you are using a version of community older than 0.3.21, install the older `@xenova/transformers` package and
+import the embeddings from `"@langchain/community/embeddings/hf_transformers"` below.
+:::
```bash npm2yarn
-npm install @xenova/transformers
+npm install @huggingface/transformers
```
import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx";
diff --git a/environment_tests/test-exports-cjs/src/import.js b/environment_tests/test-exports-cjs/src/import.js
index 752cfdea37ba..6723331f7dc9 100644
--- a/environment_tests/test-exports-cjs/src/import.js
+++ b/environment_tests/test-exports-cjs/src/import.js
@@ -3,7 +3,7 @@ async function test() {
const { OpenAI } = await import("@langchain/openai");
const { LLMChain } = await import("langchain/chains");
const { ChatPromptTemplate } = await import("@langchain/core/prompts");
- const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/hf_transformers");
+ const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/huggingface_transformers");
const { Document } = await import("@langchain/core/documents");
const { MemoryVectorStore } = await import("langchain/vectorstores/memory");
diff --git a/environment_tests/test-exports-cjs/src/index.mjs b/environment_tests/test-exports-cjs/src/index.mjs
index 632b8081fbaa..7f30afdc81d6 100644
--- a/environment_tests/test-exports-cjs/src/index.mjs
+++ b/environment_tests/test-exports-cjs/src/index.mjs
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
import { LLMChain } from "langchain/chains";
import { MemoryVectorStore } from "langchain/vectorstores/memory";
import { ChatPromptTemplate } from "@langchain/core/prompts";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
import { Document } from "@langchain/core/documents";
// Test exports
diff --git a/environment_tests/test-exports-cjs/src/index.ts b/environment_tests/test-exports-cjs/src/index.ts
index d2dcb9ebab0d..c4077382afd4 100644
--- a/environment_tests/test-exports-cjs/src/index.ts
+++ b/environment_tests/test-exports-cjs/src/index.ts
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
import { LLMChain } from "langchain/chains";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { MemoryVectorStore } from "langchain/vectorstores/memory";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
import { Document } from "@langchain/core/documents";
async function test(useAzure: boolean = false) {
@@ -25,7 +25,9 @@ async function test(useAzure: boolean = false) {
openAIApiKey: "sk-XXXX",
};
- const vs = new MemoryVectorStore(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }));
+ const vs = new MemoryVectorStore(
+ new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" })
+ );
await vs.addVectors(
[
diff --git a/environment_tests/test-exports-cjs/src/require.js b/environment_tests/test-exports-cjs/src/require.js
index 1343f8587f35..f9110d7cd71f 100644
--- a/environment_tests/test-exports-cjs/src/require.js
+++ b/environment_tests/test-exports-cjs/src/require.js
@@ -3,7 +3,7 @@ const { OpenAI } = require("@langchain/openai");
const { LLMChain } = require("langchain/chains");
const { ChatPromptTemplate } = require("@langchain/core/prompts");
const { MemoryVectorStore } = require("langchain/vectorstores/memory");
-const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/hf_transformers");
+const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/huggingface_transformers");
const { Document } = require("@langchain/core/documents");
async function test() {
diff --git a/environment_tests/test-exports-esm/src/import.cjs b/environment_tests/test-exports-esm/src/import.cjs
index 6837754c442c..6ab7bc73355e 100644
--- a/environment_tests/test-exports-esm/src/import.cjs
+++ b/environment_tests/test-exports-esm/src/import.cjs
@@ -4,7 +4,7 @@ async function test() {
const { LLMChain } = await import("langchain/chains");
const { ChatPromptTemplate } = await import("@langchain/core/prompts");
const { MemoryVectorStore } = await import("langchain/vectorstores/memory");
- const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/hf_transformers");
+ const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/huggingface_transformers");
const { Document } = await import("@langchain/core/documents");
// Test exports
diff --git a/environment_tests/test-exports-esm/src/index.js b/environment_tests/test-exports-esm/src/index.js
index 2347699ee1dc..0046911c1adb 100644
--- a/environment_tests/test-exports-esm/src/index.js
+++ b/environment_tests/test-exports-esm/src/index.js
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
import { LLMChain } from "langchain/chains";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { MemoryVectorStore } from "langchain/vectorstores/memory";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
import { Document } from "@langchain/core/documents";
import { CallbackManager } from "@langchain/core/callbacks/manager";
diff --git a/environment_tests/test-exports-esm/src/index.ts b/environment_tests/test-exports-esm/src/index.ts
index c29f419c07a8..2d55732c8758 100644
--- a/environment_tests/test-exports-esm/src/index.ts
+++ b/environment_tests/test-exports-esm/src/index.ts
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
import { LLMChain } from "langchain/chains";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { MemoryVectorStore } from "langchain/vectorstores/memory";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
import { Document } from "@langchain/core/documents";
async function test(useAzure: boolean = false) {
@@ -24,7 +24,9 @@ async function test(useAzure: boolean = false) {
openAIApiKey: "sk-XXXX",
};
- const vs = new MemoryVectorStore(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }));
+ const vs = new MemoryVectorStore(
+ new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" })
+ );
await vs.addVectors(
[
diff --git a/environment_tests/test-exports-esm/src/require.cjs b/environment_tests/test-exports-esm/src/require.cjs
index 31461c4b6c23..ab0db459e6c1 100644
--- a/environment_tests/test-exports-esm/src/require.cjs
+++ b/environment_tests/test-exports-esm/src/require.cjs
@@ -3,7 +3,7 @@ const { OpenAI } = require("@langchain/openai");
const { LLMChain } = require("langchain/chains");
const { ChatPromptTemplate } = require("@langchain/core/prompts");
const { MemoryVectorStore } = require("langchain/vectorstores/memory");
-const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/hf_transformers");
+const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/huggingface_transformers");
const { Document } = require("@langchain/core/documents");
async function test() {
diff --git a/examples/.env.example b/examples/.env.example
index 25730a68857e..83a320555978 100644
--- a/examples/.env.example
+++ b/examples/.env.example
@@ -84,4 +84,8 @@ HANA_HOST=HANA_DB_ADDRESS
HANA_PORT=HANA_DB_PORT
HANA_UID=HANA_DB_USER
HANA_PWD=HANA_DB_PASSWORD
-ARK_API_KEY=ADD_YOURS_HERE # https://console.volcengine.com/
\ No newline at end of file
+ARK_API_KEY=ADD_YOURS_HERE # https://console.volcengine.com/
+JIRA_HOST=ADD_YOURS_HERE
+JIRA_USERNAME=ADD_YOURS_HERE
+JIRA_ACCESS_TOKEN=ADD_YOURS_HERE
+JIRA_PROJECT_KEY=ADD_YOURS_HERE
diff --git a/examples/src/document_loaders/jira.ts b/examples/src/document_loaders/jira.ts
new file mode 100644
index 000000000000..50b2a9511c45
--- /dev/null
+++ b/examples/src/document_loaders/jira.ts
@@ -0,0 +1,26 @@
+import { JiraProjectLoader } from "@langchain/community/document_loaders/web/jira";
+
+const host = process.env.JIRA_HOST || "https://jira.example.com";
+const username = process.env.JIRA_USERNAME;
+const accessToken = process.env.JIRA_ACCESS_TOKEN;
+const projectKey = process.env.JIRA_PROJECT_KEY || "PROJ";
+
+if (username && accessToken) {
+ // Created within last 30 days
+ const createdAfter = new Date();
+ createdAfter.setDate(createdAfter.getDate() - 30);
+ const loader = new JiraProjectLoader({
+ host,
+ projectKey,
+ username,
+ accessToken,
+ createdAfter,
+ });
+
+ const documents = await loader.load();
+ console.log(`Loaded ${documents.length} Jira document(s)`);
+} else {
+ console.log(
+ "You must provide a username and access token to run this example."
+ );
+}
diff --git a/examples/src/models/embeddings/hf_transformers.ts b/examples/src/models/embeddings/hf_transformers.ts
index 2643eabe7c38..160530d9a30f 100644
--- a/examples/src/models/embeddings/hf_transformers.ts
+++ b/examples/src/models/embeddings/hf_transformers.ts
@@ -1,4 +1,4 @@
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
const model = new HuggingFaceTransformersEmbeddings({
model: "Xenova/all-MiniLM-L6-v2",
diff --git a/examples/src/use_cases/local_retrieval_qa/chain.ts b/examples/src/use_cases/local_retrieval_qa/chain.ts
index c5da25e3a6f9..38bbceb4199b 100644
--- a/examples/src/use_cases/local_retrieval_qa/chain.ts
+++ b/examples/src/use_cases/local_retrieval_qa/chain.ts
@@ -2,7 +2,7 @@ import { CheerioWebBaseLoader } from "@langchain/community/document_loaders/web/
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
import { Ollama } from "@langchain/community/llms/ollama";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
import { formatDocumentsAsString } from "langchain/util/document";
import { PromptTemplate } from "@langchain/core/prompts";
import {
diff --git a/examples/src/use_cases/local_retrieval_qa/load_documents.ts b/examples/src/use_cases/local_retrieval_qa/load_documents.ts
index 9e449b2cc532..7efea9a161ea 100644
--- a/examples/src/use_cases/local_retrieval_qa/load_documents.ts
+++ b/examples/src/use_cases/local_retrieval_qa/load_documents.ts
@@ -1,7 +1,7 @@
import { CheerioWebBaseLoader } from "@langchain/community/document_loaders/web/cheerio";
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
const loader = new CheerioWebBaseLoader(
"https://lilianweng.github.io/posts/2023-06-23-agent/"
diff --git a/examples/src/use_cases/local_retrieval_qa/qa_chain.ts b/examples/src/use_cases/local_retrieval_qa/qa_chain.ts
index 949918067369..6bb4711e6276 100644
--- a/examples/src/use_cases/local_retrieval_qa/qa_chain.ts
+++ b/examples/src/use_cases/local_retrieval_qa/qa_chain.ts
@@ -3,7 +3,7 @@ import { CheerioWebBaseLoader } from "@langchain/community/document_loaders/web/
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
import { Ollama } from "@langchain/community/llms/ollama";
-import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/huggingface_transformers";
import { PromptTemplate } from "@langchain/core/prompts";
const loader = new CheerioWebBaseLoader(
diff --git a/langchain-core/package.json b/langchain-core/package.json
index 32510ecc8c72..b938bb050122 100644
--- a/langchain-core/package.json
+++ b/langchain-core/package.json
@@ -1,6 +1,6 @@
{
"name": "@langchain/core",
- "version": "0.3.26",
+ "version": "0.3.27",
"description": "Core LangChain.js abstractions and schemas",
"type": "module",
"engines": {
diff --git a/libs/langchain-aws/package.json b/libs/langchain-aws/package.json
index dfa7745f56bc..50a419fb57e2 100644
--- a/libs/langchain-aws/package.json
+++ b/libs/langchain-aws/package.json
@@ -1,6 +1,6 @@
{
"name": "@langchain/aws",
- "version": "0.1.2",
+ "version": "0.1.3",
"description": "LangChain AWS integration",
"type": "module",
"engines": {
diff --git a/libs/langchain-community/.env.example b/libs/langchain-community/.env.example
new file mode 100644
index 000000000000..2c36f95558b3
--- /dev/null
+++ b/libs/langchain-community/.env.example
@@ -0,0 +1,4 @@
+JIRA_HOST=ADD_YOURS_HERE
+JIRA_USERNAME=ADD_YOURS_HERE
+JIRA_ACCESS_TOKEN=ADD_YOURS_HERE
+JIRA_PROJECT_KEY=ADD_YOURS_HERE
diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore
index 2f2abe352f2c..6eb6e9c7d9d3 100644
--- a/libs/langchain-community/.gitignore
+++ b/libs/langchain-community/.gitignore
@@ -178,6 +178,10 @@ embeddings/hf_transformers.cjs
embeddings/hf_transformers.js
embeddings/hf_transformers.d.ts
embeddings/hf_transformers.d.cts
+embeddings/huggingface_transformers.cjs
+embeddings/huggingface_transformers.js
+embeddings/huggingface_transformers.d.ts
+embeddings/huggingface_transformers.d.cts
embeddings/ibm.cjs
embeddings/ibm.js
embeddings/ibm.d.ts
@@ -930,6 +934,10 @@ document_loaders/web/imsdb.cjs
document_loaders/web/imsdb.js
document_loaders/web/imsdb.d.ts
document_loaders/web/imsdb.d.cts
+document_loaders/web/jira.cjs
+document_loaders/web/jira.js
+document_loaders/web/jira.d.ts
+document_loaders/web/jira.d.cts
document_loaders/web/figma.cjs
document_loaders/web/figma.js
document_loaders/web/figma.d.ts
diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js
index 496e015280e7..3883e56873f8 100644
--- a/libs/langchain-community/langchain.config.js
+++ b/libs/langchain-community/langchain.config.js
@@ -80,6 +80,7 @@ export const config = {
"embeddings/gradient_ai": "embeddings/gradient_ai",
"embeddings/hf": "embeddings/hf",
"embeddings/hf_transformers": "embeddings/hf_transformers",
+ "embeddings/huggingface_transformers": "embeddings/huggingface_transformers",
"embeddings/ibm": "embeddings/ibm",
"embeddings/jina": "embeddings/jina",
"embeddings/llama_cpp": "embeddings/llama_cpp",
@@ -288,6 +289,7 @@ export const config = {
"document_loaders/web/gitbook": "document_loaders/web/gitbook",
"document_loaders/web/hn": "document_loaders/web/hn",
"document_loaders/web/imsdb": "document_loaders/web/imsdb",
+ "document_loaders/web/jira": "document_loaders/web/jira",
"document_loaders/web/figma": "document_loaders/web/figma",
"document_loaders/web/firecrawl": "document_loaders/web/firecrawl",
"document_loaders/web/github": "document_loaders/web/github",
@@ -356,6 +358,7 @@ export const config = {
"embeddings/tensorflow",
"embeddings/hf",
"embeddings/hf_transformers",
+ "embeddings/huggingface_transformers",
"embeddings/ibm",
"embeddings/jina",
"embeddings/llama_cpp",
diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json
index 50795f2c3cef..a286ff97ff01 100644
--- a/libs/langchain-community/package.json
+++ b/libs/langchain-community/package.json
@@ -1,6 +1,6 @@
{
"name": "@langchain/community",
- "version": "0.3.20",
+ "version": "0.3.21",
"description": "Third-party integrations for LangChain.js",
"type": "module",
"engines": {
@@ -78,6 +78,7 @@
"@google-cloud/storage": "^7.7.0",
"@gradientai/nodejs-sdk": "^1.2.0",
"@huggingface/inference": "^2.6.4",
+ "@huggingface/transformers": "^3.2.3",
"@ibm-cloud/watsonx-ai": "^1.3.0",
"@jest/globals": "^29.5.0",
"@lancedb/lancedb": "^0.13.0",
@@ -124,6 +125,7 @@
"@types/pg": "^8.11.0",
"@types/pg-copy-streams": "^1.2.2",
"@types/uuid": "^9",
+ "@types/word-extractor": "^1",
"@types/ws": "^8",
"@typescript-eslint/eslint-plugin": "^5.58.0",
"@typescript-eslint/parser": "^5.58.0",
@@ -134,7 +136,6 @@
"@vercel/postgres": "^0.5.0",
"@writerai/writer-sdk": "^0.40.2",
"@xata.io/client": "^0.28.0",
- "@xenova/transformers": "^2.17.2",
"@zilliz/milvus2-sdk-node": ">=2.3.5",
"apify-client": "^2.7.1",
"assemblyai": "^4.6.0",
@@ -217,6 +218,7 @@
"voy-search": "0.6.2",
"weaviate-ts-client": "^1.4.0",
"web-auth-library": "^1.0.3",
+ "word-extractor": "^1.0.4",
"youtube-transcript": "^1.0.6",
"youtubei.js": "^9.1.0"
},
@@ -249,6 +251,7 @@
"@google-cloud/storage": "^6.10.1 || ^7.7.0",
"@gradientai/nodejs-sdk": "^1.2.0",
"@huggingface/inference": "^2.6.4",
+ "@huggingface/transformers": "^3.2.3",
"@ibm-cloud/watsonx-ai": "*",
"@lancedb/lancedb": "^0.12.0",
"@langchain/core": ">=0.2.21 <0.4.0",
@@ -282,7 +285,6 @@
"@vercel/postgres": "^0.5.0",
"@writerai/writer-sdk": "^0.40.2",
"@xata.io/client": "^0.28.0",
- "@xenova/transformers": "^2.17.2",
"@zilliz/milvus2-sdk-node": ">=2.3.5",
"apify-client": "^2.7.1",
"assemblyai": "^4.6.0",
@@ -344,6 +346,7 @@
"voy-search": "0.6.2",
"weaviate-ts-client": "*",
"web-auth-library": "^1.0.3",
+ "word-extractor": "*",
"ws": "^8.14.2",
"youtube-transcript": "^1.0.6",
"youtubei.js": "^9.1.0"
@@ -430,6 +433,9 @@
"@huggingface/inference": {
"optional": true
},
+ "@huggingface/transformers": {
+ "optional": true
+ },
"@lancedb/lancedb": {
"optional": true
},
@@ -523,9 +529,6 @@
"@xata.io/client": {
"optional": true
},
- "@xenova/transformers": {
- "optional": true
- },
"@zilliz/milvus2-sdk-node": {
"optional": true
},
@@ -703,6 +706,9 @@
"web-auth-library": {
"optional": true
},
+ "word-extractor": {
+ "optional": true
+ },
"ws": {
"optional": true
},
@@ -1122,6 +1128,15 @@
"import": "./embeddings/hf_transformers.js",
"require": "./embeddings/hf_transformers.cjs"
},
+ "./embeddings/huggingface_transformers": {
+ "types": {
+ "import": "./embeddings/huggingface_transformers.d.ts",
+ "require": "./embeddings/huggingface_transformers.d.cts",
+ "default": "./embeddings/huggingface_transformers.d.ts"
+ },
+ "import": "./embeddings/huggingface_transformers.js",
+ "require": "./embeddings/huggingface_transformers.cjs"
+ },
"./embeddings/ibm": {
"types": {
"import": "./embeddings/ibm.d.ts",
@@ -2814,6 +2829,15 @@
"import": "./document_loaders/web/imsdb.js",
"require": "./document_loaders/web/imsdb.cjs"
},
+ "./document_loaders/web/jira": {
+ "types": {
+ "import": "./document_loaders/web/jira.d.ts",
+ "require": "./document_loaders/web/jira.d.cts",
+ "default": "./document_loaders/web/jira.d.ts"
+ },
+ "import": "./document_loaders/web/jira.js",
+ "require": "./document_loaders/web/jira.cjs"
+ },
"./document_loaders/web/figma": {
"types": {
"import": "./document_loaders/web/figma.d.ts",
@@ -3349,6 +3373,10 @@
"embeddings/hf_transformers.js",
"embeddings/hf_transformers.d.ts",
"embeddings/hf_transformers.d.cts",
+ "embeddings/huggingface_transformers.cjs",
+ "embeddings/huggingface_transformers.js",
+ "embeddings/huggingface_transformers.d.ts",
+ "embeddings/huggingface_transformers.d.cts",
"embeddings/ibm.cjs",
"embeddings/ibm.js",
"embeddings/ibm.d.ts",
@@ -4101,6 +4129,10 @@
"document_loaders/web/imsdb.js",
"document_loaders/web/imsdb.d.ts",
"document_loaders/web/imsdb.d.cts",
+ "document_loaders/web/jira.cjs",
+ "document_loaders/web/jira.js",
+ "document_loaders/web/jira.d.ts",
+ "document_loaders/web/jira.d.cts",
"document_loaders/web/figma.cjs",
"document_loaders/web/figma.js",
"document_loaders/web/figma.d.ts",
diff --git a/libs/langchain-community/src/document_loaders/fs/docx.ts b/libs/langchain-community/src/document_loaders/fs/docx.ts
index 72518aec3b2e..e1edef2fc8e7 100644
--- a/libs/langchain-community/src/document_loaders/fs/docx.ts
+++ b/libs/langchain-community/src/document_loaders/fs/docx.ts
@@ -1,19 +1,33 @@
import { Document } from "@langchain/core/documents";
import { BufferLoader } from "langchain/document_loaders/fs/buffer";
+type DocxLoaderOptions = {
+ type: "docx" | "doc";
+};
/**
* A class that extends the `BufferLoader` class. It represents a document
* loader that loads documents from DOCX files.
+ * It has a constructor that takes a `filePathOrBlob` parameter representing the path to the word
+ * file or a Blob object, and an optional `options` parameter of type
+ * `DocxLoaderOptions`
*/
export class DocxLoader extends BufferLoader {
- constructor(filePathOrBlob: string | Blob) {
+ protected options: DocxLoaderOptions = { type: "docx" };
+
+ constructor(filePathOrBlob: string | Blob, options?: DocxLoaderOptions) {
super(filePathOrBlob);
+ if (options) {
+ this.options = {
+ ...options,
+ };
+ }
}
/**
* A method that takes a `raw` buffer and `metadata` as parameters and
* returns a promise that resolves to an array of `Document` instances. It
- * uses the `extractRawText` function from the `mammoth` module to extract
+ * uses the `extractRawText` function from the `mammoth` module or
+ * `extract` method from the `word-extractor` module to extract
* the raw text content from the buffer. If the extracted text content is
* empty, it returns an empty array. Otherwise, it creates a new
* `Document` instance with the extracted text content and the provided
@@ -26,6 +40,31 @@ export class DocxLoader extends BufferLoader {
raw: Buffer,
metadata: Document["metadata"]
): Promise {
+ if (this.options.type === "doc") {
+ return this.parseDoc(raw, metadata);
+ }
+ return this.parseDocx(raw, metadata);
+ }
+
+ /**
+ * A private method that takes a `raw` buffer and `metadata` as parameters and
+ * returns a promise that resolves to an array of `Document` instances. It
+ * uses the `extractRawText` function from the `mammoth` module to extract
+ * the raw text content from the buffer. If the extracted text content is
+ * empty, it returns an empty array. Otherwise, it creates a new
+ * `Document` instance with the extracted text content and the provided
+ * metadata, and returns it as an array.
+ * @param raw The raw buffer from which to extract text content.
+ * @param metadata The metadata to be associated with the created `Document` instance.
+ * @returns A promise that resolves to an array of `Document` instances.
+ */
+ private async parseDocx(
+ raw: Buffer,
+ metadata: Document["metadata"]
+ ): Promise {
+ if (this.options.type === "doc") {
+ return this.parseDoc(raw, metadata);
+ }
const { extractRawText } = await DocxLoaderImports();
const docx = await extractRawText({
buffer: raw,
@@ -40,6 +79,33 @@ export class DocxLoader extends BufferLoader {
}),
];
}
+
+ /**
+ * A private method that takes a `raw` buffer and `metadata` as parameters and
+ * returns a promise that resolves to an array of `Document` instances. It
+ * uses the `extract` method from the `word-extractor` module to extract
+ * the raw text content from the buffer. If the extracted text content is
+ * empty, it returns an empty array. Otherwise, it creates a new
+ * `Document` instance with the extracted text content and the provided
+ * metadata, and returns it as an array.
+ * @param raw The raw buffer from which to extract text content.
+ * @param metadata The metadata to be associated with the created `Document` instance.
+ * @returns A promise that resolves to an array of `Document` instances.
+ */
+ private async parseDoc(
+ raw: Buffer,
+ metadata: Document["metadata"]
+ ): Promise {
+ const WordExtractor = await DocLoaderImports();
+ const extractor = new WordExtractor();
+ const doc = await extractor.extract(raw);
+ return [
+ new Document({
+ pageContent: doc.getBody(),
+ metadata,
+ }),
+ ];
+ }
}
async function DocxLoaderImports() {
@@ -53,3 +119,15 @@ async function DocxLoaderImports() {
);
}
}
+
+async function DocLoaderImports() {
+ try {
+ const WordExtractor = await import("word-extractor");
+ return WordExtractor.default;
+ } catch (e) {
+ console.error(e);
+ throw new Error(
+ "Failed to load word-extractor. Please install it with eg. `npm install word-extractor`."
+ );
+ }
+}
diff --git a/libs/langchain-community/src/document_loaders/tests/docx.test.ts b/libs/langchain-community/src/document_loaders/tests/docx.test.ts
index 63395bb51bc0..82e66aa91907 100644
--- a/libs/langchain-community/src/document_loaders/tests/docx.test.ts
+++ b/libs/langchain-community/src/document_loaders/tests/docx.test.ts
@@ -3,7 +3,7 @@ import * as url from "node:url";
import * as path from "node:path";
import { DocxLoader } from "../fs/docx.js";
-test("Test Word doc loader from file", async () => {
+test("Test Word doc loader from .docx file", async () => {
const filePath = path.resolve(
path.dirname(url.fileURLToPath(import.meta.url)),
"./example_data/attention.docx"
@@ -14,3 +14,17 @@ test("Test Word doc loader from file", async () => {
expect(docs.length).toBe(1); // not much text in the example
expect(docs[0].pageContent).toContain("an interesting activity");
});
+
+test("Test Word doc loader from .doc file", async () => {
+ const filePath = path.resolve(
+ path.dirname(url.fileURLToPath(import.meta.url)),
+ "./example_data/attention.doc"
+ );
+ const loader = new DocxLoader(filePath, {
+ type: "doc",
+ });
+ const docs = await loader.load();
+
+ expect(docs.length).toBe(1); // not much text in the example
+ expect(docs[0].pageContent).toContain("an interesting activity");
+});
diff --git a/libs/langchain-community/src/document_loaders/tests/example_data/attention.doc b/libs/langchain-community/src/document_loaders/tests/example_data/attention.doc
new file mode 100644
index 000000000000..e68399c5a7c8
Binary files /dev/null and b/libs/langchain-community/src/document_loaders/tests/example_data/attention.doc differ
diff --git a/libs/langchain-community/src/document_loaders/tests/jira.int.test.ts b/libs/langchain-community/src/document_loaders/tests/jira.int.test.ts
new file mode 100644
index 000000000000..e01d1d65663b
--- /dev/null
+++ b/libs/langchain-community/src/document_loaders/tests/jira.int.test.ts
@@ -0,0 +1,209 @@
+/**
+ * NOTE: Env var should be set, and configured project should exist
+ */
+import { Document } from "@langchain/core/documents";
+import { expect, test } from "@jest/globals";
+import {
+ JiraIssue,
+ JiraProjectLoader,
+ JiraProjectLoaderParams,
+} from "../web/jira.js";
+
+describe("JiraProjectLoader Integration Tests", () => {
+ const JIRA_HOST = requireEnvVar("JIRA_HOST");
+ const JIRA_USERNAME = requireEnvVar("JIRA_USERNAME");
+ const JIRA_ACCESS_TOKEN = requireEnvVar("JIRA_ACCESS_TOKEN");
+ const JIRA_PROJECT_KEY = requireEnvVar("JIRA_PROJECT_KEY");
+ const jiraConf: JiraProjectLoaderParams = {
+ host: JIRA_HOST,
+ projectKey: JIRA_PROJECT_KEY,
+ username: JIRA_USERNAME,
+ accessToken: JIRA_ACCESS_TOKEN,
+ limitPerRequest: 20,
+ };
+
+ test("should load Jira project issues as documents successfully", async () => {
+ const docs = await loadJiraDocsUntil((docs) => docs.length > 0);
+
+ expect(docs).toBeDefined();
+ expect(Array.isArray(docs)).toBe(true);
+
+ if (docs.length < 1) {
+ // Skip test if not enough issues available
+ return;
+ }
+ const firstDoc = docs[0];
+
+ // Check document structure
+ expect(firstDoc).toHaveProperty("pageContent");
+ expect(firstDoc).toHaveProperty("metadata");
+
+ // Check metadata
+ expect(firstDoc.metadata).toHaveProperty("id");
+ expect(firstDoc.metadata).toHaveProperty("host", JIRA_HOST);
+ expect(firstDoc.metadata).toHaveProperty("projectKey", JIRA_PROJECT_KEY);
+
+ // Check pageContent contains essential Jira issue information
+ const content = firstDoc.pageContent;
+ expect(content).toContain("Issue:");
+ expect(content).toContain("Project:");
+ expect(content).toContain("Status:");
+ expect(content).toContain("Priority:");
+ expect(content).toContain("Type:");
+ expect(content).toContain("Creator:");
+ });
+
+ test("should filter issues based on createdAfter date", async () => {
+ // First load at least 2 issues with different creation dates (ignoring time)
+ const baseIssues = await loadJiraIssuesUntil(haveTwoDifferentCreationDates);
+ if (baseIssues.length < 2) {
+ // Skip test if not enough issues available
+ return;
+ }
+
+ // Create a map from date string without time to list of issues
+ const dateToIssueMap = new Map();
+ baseIssues.forEach((issue) => {
+ const date = asStringWithoutTime(new Date(issue.fields.created));
+ dateToIssueMap.set(date, (dateToIssueMap.get(date) ?? []).concat(issue));
+ });
+ // Convert map to list of {date, issues}
+ const issuesGroupedByDate = Array.from(
+ dateToIssueMap,
+ ([date, issues]) => ({ date, issues })
+ );
+ issuesGroupedByDate.sort((a, b) => a.date.localeCompare(b.date));
+
+ // Pick middle date to split issues in two groups
+ const middleIndex = Math.floor(issuesGroupedByDate.length / 2);
+ const middleDate = new Date(issuesGroupedByDate[middleIndex].date);
+ const issuesAfterMiddle = issuesGroupedByDate
+ .slice(middleIndex)
+ .flatMap(({ issues }) => issues);
+
+ // Load issues created after middle date
+ const loader = new JiraProjectLoader({
+ ...jiraConf,
+ createdAfter: middleDate,
+ });
+
+ const filteredDocs = await loader.load();
+
+ // Verify we got the expected issues
+ expect(filteredDocs.length).toBeGreaterThan(0);
+ expect(filteredDocs.length).toBeLessThan(baseIssues.length);
+
+ // Verify all returned issues are created after our cutoff date
+ const middleDateTimestamp = middleDate.getTime();
+ filteredDocs.forEach((doc) => {
+ const issueDateString = doc.pageContent
+ .split("\n")
+ .filter((line) => /^Created: /.test(line))[0]
+ .replace("Created: ", "");
+ const issueDateTimestamp = new Date(
+ asStringWithoutTime(new Date(issueDateString))
+ ).getTime();
+ expect(issueDateTimestamp).toBeGreaterThanOrEqual(middleDateTimestamp);
+ });
+
+ // Verify we got the same issues as in our original set
+ const filteredIds = new Set(filteredDocs.map((d) => d.metadata.id));
+ const expectedIds = new Set(issuesAfterMiddle.map((issue) => issue.id));
+ expect(filteredIds).toEqual(expectedIds);
+ });
+
+ test("should handle invalid credentials", async () => {
+ const loader = new JiraProjectLoader({
+ ...jiraConf,
+ username: "invalid_username",
+ accessToken: "invalid_token",
+ });
+
+ const docs = await loader.load();
+ expect(docs).toEqual([]);
+ });
+
+ test("should handle invalid project key", async () => {
+ const loader = new JiraProjectLoader({
+ ...jiraConf,
+ projectKey: "INVALID_PROJECT_KEY",
+ });
+
+ const docs = await loader.load();
+ expect(docs).toEqual([]);
+ });
+
+ function requireEnvVar(name: string): string {
+ // eslint-disable-next-line no-process-env
+ const value = process.env[name];
+ if (!value) {
+ throw new Error(`environment variable "${name}" must be set`);
+ }
+ return value;
+ }
+
+ function asStringWithoutTime(date: Date): string {
+ return date.toISOString().split("T")[0];
+ }
+
+ function sameDate(a: Date, b: Date) {
+ return asStringWithoutTime(a) === asStringWithoutTime(b);
+ }
+
+ function haveTwoDifferentCreationDates(issues: JiraIssue[]): boolean {
+ return (
+ issues.length >= 2 &&
+ issues
+ .slice(1)
+ .some(
+ (issue) =>
+ !sameDate(
+ new Date(issue.fields.created),
+ new Date(issues[0].fields.created)
+ )
+ )
+ );
+ }
+
+ async function loadJiraDocsUntil(predicate: (docs: Document[]) => boolean) {
+ const load = (createdAfter: Date) =>
+ new JiraProjectLoader({
+ ...jiraConf,
+ createdAfter,
+ }).load();
+ return loadUntil(load, predicate);
+ }
+
+ async function loadJiraIssuesUntil(
+ predicate: (docs: JiraIssue[]) => boolean
+ ) {
+ const load = (createdAfter: Date) =>
+ new JiraProjectLoader({
+ ...jiraConf,
+ createdAfter,
+ }).loadAsIssues();
+ return loadUntil(load, predicate);
+ }
+
+ async function loadUntil(
+ loadCreatedAfter: (date: Date) => Promise,
+ predicate: (loaded: T[]) => boolean
+ ): Promise {
+ const now = new Date();
+ let months = 1;
+ const maxMonths = 120;
+
+ let loaded: T[] = [];
+ while (!predicate(loaded) && months < maxMonths) {
+ const createdAfter = new Date(now);
+ createdAfter.setDate(now.getDate() - months * 30);
+ loaded = await loadCreatedAfter(createdAfter);
+ months *= 1.2;
+ }
+
+ if (months >= maxMonths) {
+ return [];
+ }
+ return loaded;
+ }
+});
diff --git a/libs/langchain-community/src/document_loaders/tests/jira.test.ts b/libs/langchain-community/src/document_loaders/tests/jira.test.ts
new file mode 100644
index 000000000000..92b1224446e0
--- /dev/null
+++ b/libs/langchain-community/src/document_loaders/tests/jira.test.ts
@@ -0,0 +1,267 @@
+import { faker } from "@faker-js/faker";
+import {
+ JiraDocumentConverter,
+ JiraIssue,
+ JiraUser,
+ JiraIssueType,
+ JiraPriority,
+ JiraProgress,
+ JiraProject,
+ JiraStatus,
+ JiraStatusCategory,
+} from "../web/jira.js";
+
+describe("JiraDocumentConverter Unit Tests", () => {
+ function getConverter() {
+ return new JiraDocumentConverter({
+ projectKey: "PROJ",
+ host: "https://example.com",
+ });
+ }
+
+ it("should handle missing optional fields", () => {
+ const issue: JiraIssue = someJiraIssue();
+ delete issue.fields.assignee;
+ delete issue.fields.duedate;
+
+ const converter = getConverter();
+ const document = converter.convertToDocuments([issue])[0];
+
+ expect(document).toBeDefined();
+ expect(document.pageContent).toContain(issue.fields.summary);
+ expect(document.pageContent).toContain("Assignee: Unassigned");
+ expect(document.pageContent).not.toMatch(/.*^Due Date: .*/m);
+ expect(document.metadata).toEqual({
+ id: issue.id,
+ host: converter.host,
+ projectKey: converter.projectKey,
+ });
+ });
+
+ it("should format the document content properly", () => {
+ const converter = getConverter();
+ const issue = someJiraIssue();
+ const document = converter.convertToDocuments([issue])[0];
+
+ expect(document.pageContent).toContain(issue.fields.summary);
+ expect(document.pageContent).toContain(issue.fields.description);
+ expect(document.pageContent).toContain(
+ issue.fields.labels?.join(", ") || ""
+ );
+ expect(document.pageContent).toContain(
+ issue.fields.reporter?.displayName || ""
+ );
+ expect(document.pageContent).toContain(
+ issue.fields.assignee?.displayName || "Unassigned"
+ );
+ expect(document.pageContent).toContain(issue.fields.duedate || "");
+ expect(document.pageContent).toContain(
+ issue.fields.timeestimate?.toString() || ""
+ );
+ expect(document.pageContent).toContain(
+ issue.fields.timespent?.toString() || ""
+ );
+ expect(document.pageContent).toContain(issue.fields.resolutiondate || "");
+ expect(document.pageContent).toContain(
+ issue.fields.progress.percent?.toString() || ""
+ );
+ });
+});
+
+export function someJiraIssueType(
+ overrides: Partial = {}
+): JiraIssueType {
+ const baseIssueType: JiraIssueType = {
+ avatarId: faker.number.int({ min: 1, max: 100 }),
+ description: faker.lorem.sentence(),
+ entityId: faker.string.uuid(),
+ hierarchyLevel: faker.number.int({ min: 1, max: 5 }),
+ iconUrl: faker.image.url(),
+ id: faker.string.numeric(5),
+ name: faker.helpers.arrayElement(["Bug", "Task", "Story", "Epic"]),
+ self: faker.internet.url(),
+ subtask: false,
+ };
+
+ return {
+ ...baseIssueType,
+ ...overrides,
+ };
+}
+
+export function someJiraUser(overrides: Partial = {}): JiraUser {
+ const baseUser = {
+ accountId: faker.string.uuid(),
+ accountType: "atlassian",
+ active: true,
+ avatarUrls: {
+ "16x16": faker.image.avatar(),
+ "24x24": faker.image.avatar(),
+ "32x32": faker.image.avatar(),
+ "48x48": faker.image.avatar(),
+ },
+ displayName: faker.person.fullName(),
+ emailAddress: faker.internet.email(),
+ self: faker.internet.url(),
+ timeZone: faker.location.timeZone(),
+ };
+
+ return {
+ ...baseUser,
+ ...overrides,
+ };
+}
+
+export function someJiraPriority(
+ overrides: Partial = {}
+): JiraPriority {
+ const basePriority: JiraPriority = {
+ iconUrl: faker.image.url(),
+ id: faker.string.numeric(2),
+ name: faker.helpers.arrayElement([
+ "Highest",
+ "High",
+ "Medium",
+ "Low",
+ "Lowest",
+ ]),
+ self: faker.internet.url(),
+ };
+
+ return {
+ ...basePriority,
+ ...overrides,
+ };
+}
+
+export function someJiraProgress(
+ overrides: Partial = {}
+): JiraProgress {
+ const baseProgress: JiraProgress = {
+ progress: faker.number.int({ min: 0, max: 100 }),
+ total: 100,
+ percent: faker.number.int({ min: 0, max: 100 }),
+ };
+
+ return {
+ ...baseProgress,
+ ...overrides,
+ };
+}
+
+export function someJiraProject(
+ overrides: Partial = {}
+): JiraProject {
+ const baseProject: JiraProject = {
+ avatarUrls: {
+ "16x16": faker.image.avatar(),
+ "24x24": faker.image.avatar(),
+ "32x32": faker.image.avatar(),
+ "48x48": faker.image.avatar(),
+ },
+ id: faker.string.numeric(5),
+ key: faker.string.alpha(4).toUpperCase(),
+ name: faker.company.name(),
+ projectTypeKey: "software",
+ self: faker.internet.url(),
+ simplified: false,
+ };
+
+ return {
+ ...baseProject,
+ ...overrides,
+ };
+}
+
+export function someJiraStatusCategory(
+ overrides: Partial = {}
+): JiraStatusCategory {
+ const baseStatusCategory: JiraStatusCategory = {
+ self: faker.internet.url(),
+ id: faker.number.int({ min: 1, max: 5 }),
+ key: faker.helpers.arrayElement(["new", "indeterminate", "done"]),
+ colorName: faker.helpers.arrayElement(["blue-gray", "yellow", "green"]),
+ name: faker.helpers.arrayElement(["To Do", "In Progress", "Done"]),
+ };
+
+ return {
+ ...baseStatusCategory,
+ ...overrides,
+ };
+}
+
+export function someJiraStatus(
+ overrides: Partial = {}
+): JiraStatus {
+ const baseStatus: JiraStatus = {
+ self: faker.internet.url(),
+ description: faker.lorem.sentence(),
+ iconUrl: faker.image.url(),
+ name: faker.helpers.arrayElement([
+ "To Do",
+ "In Progress",
+ "Done",
+ "Blocked",
+ ]),
+ id: faker.string.numeric(2),
+ statusCategory: someJiraStatusCategory(),
+ };
+
+ return {
+ ...baseStatus,
+ ...overrides,
+ };
+}
+
+export function someJiraIssue(overrides: Partial = {}): JiraIssue {
+ const issueKey = `${faker.string.alpha(4).toUpperCase()}-${faker.number.int({
+ min: 1,
+ max: 9999,
+ })}`;
+
+ const baseIssue: JiraIssue = {
+ expand: "renderedFields",
+ id: faker.string.numeric(5),
+ self: `https://${faker.internet.domainName()}/rest/api/2/issue/${issueKey}`,
+ key: issueKey,
+ fields: {
+ assignee: faker.datatype.boolean() ? someJiraUser() : undefined,
+ created: faker.date.past().toISOString(),
+ description: faker.lorem.paragraph(),
+ issuelinks: [],
+ issuetype: someJiraIssueType(),
+ labels: faker.datatype.boolean()
+ ? Array.from({ length: faker.number.int({ min: 1, max: 5 }) }, () =>
+ faker.word.noun()
+ )
+ : undefined,
+ priority: someJiraPriority(),
+ progress: someJiraProgress(),
+ project: someJiraProject(),
+ reporter: faker.datatype.boolean() ? someJiraUser() : undefined,
+ creator: someJiraUser(),
+ resolutiondate: faker.datatype.boolean()
+ ? faker.date.recent().toISOString()
+ : undefined,
+ status: someJiraStatus(),
+ subtasks: [],
+ summary: faker.lorem.sentence(),
+ timeestimate: faker.datatype.boolean()
+ ? faker.number.int({ min: 1, max: 100 }) * 3600
+ : undefined,
+ timespent: faker.datatype.boolean()
+ ? faker.number.int({ min: 1, max: 100 }) * 3600
+ : undefined,
+ updated: faker.date.recent().toISOString(),
+ duedate: faker.datatype.boolean()
+ ? faker.date.future().toISOString()
+ : undefined,
+ },
+ };
+ console.log(baseIssue.fields.duedate);
+
+ return {
+ ...baseIssue,
+ ...overrides,
+ };
+}
diff --git a/libs/langchain-community/src/document_loaders/web/cheerio.ts b/libs/langchain-community/src/document_loaders/web/cheerio.ts
index 106b1ffe9d33..72d06aed2d10 100644
--- a/libs/langchain-community/src/document_loaders/web/cheerio.ts
+++ b/libs/langchain-community/src/document_loaders/web/cheerio.ts
@@ -130,8 +130,9 @@ export class CheerioWebBaseLoader
*/
async load(): Promise {
const $ = await this.scrape();
+ const title = $("title").text();
const text = $(this.selector).text();
- const metadata = { source: this.webPath };
+ const metadata = { source: this.webPath, title };
return [new Document({ pageContent: text, metadata })];
}
diff --git a/libs/langchain-community/src/document_loaders/web/jira.ts b/libs/langchain-community/src/document_loaders/web/jira.ts
new file mode 100644
index 000000000000..59e0879d2ab9
--- /dev/null
+++ b/libs/langchain-community/src/document_loaders/web/jira.ts
@@ -0,0 +1,441 @@
+import { Document } from "@langchain/core/documents";
+import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
+
+export type JiraStatusCategory = {
+ self: string;
+ id: number;
+ key: string;
+ colorName: string;
+ name: string;
+};
+
+export type JiraStatus = {
+ self: string;
+ description: string;
+ iconUrl: string;
+ name: string;
+ id: string;
+ statusCategory: JiraStatusCategory;
+};
+
+export type JiraUser = {
+ accountId: string;
+ accountType: string;
+ active: boolean;
+ avatarUrls: {
+ "16x16": string;
+ "24x24": string;
+ "32x32": string;
+ "48x48": string;
+ };
+ displayName: string;
+ emailAddress: string;
+ self: string;
+ timeZone: string;
+};
+
+export type JiraIssueType = {
+ avatarId: number;
+ description: string;
+ entityId: string;
+ hierarchyLevel: number;
+ iconUrl: string;
+ id: string;
+ name: string;
+ self: string;
+ subtask: boolean;
+};
+
+export type JiraPriority = {
+ iconUrl: string;
+ id: string;
+ name: string;
+ self: string;
+};
+
+export type JiraProgress = {
+ progress: number;
+ total: number;
+ percent?: number;
+};
+
+export type JiraProject = {
+ avatarUrls: {
+ "16x16": string;
+ "24x24": string;
+ "32x32": string;
+ "48x48": string;
+ };
+ id: string;
+ key: string;
+ name: string;
+ projectTypeKey: string;
+ self: string;
+ simplified: boolean;
+};
+
+export type JiraSubTask = {
+ id: string;
+ key: string;
+ self: string;
+ fields: {
+ issuetype: JiraIssueType;
+ priority: JiraPriority;
+ status: JiraStatus;
+ summary: string;
+ };
+};
+
+export type JiraIssueLinkType = {
+ id: string;
+ name: string;
+ inward: string;
+ outward: string;
+ self: string;
+};
+
+export type JiraBriefIssue = {
+ id: string;
+ key: string;
+ self: string;
+ fields: {
+ summary: string;
+ status: JiraStatus;
+ priority: JiraPriority;
+ issuetype: JiraIssueType;
+ };
+};
+
+export type JiraIssueLink = {
+ id: string;
+ self: string;
+ type: JiraIssueLinkType;
+ inwardIssue?: JiraBriefIssue;
+ outwardIssue?: JiraBriefIssue;
+};
+
+export type JiraIssue = {
+ expand: string;
+ id: string;
+ self: string;
+ key: string;
+ fields: {
+ assignee?: JiraUser;
+ created: string;
+ description: string;
+ issuelinks: JiraIssueLink[];
+ issuetype: JiraIssueType;
+ labels?: string[];
+ priority: JiraPriority;
+ progress: JiraProgress;
+ project: JiraProject;
+ reporter?: JiraUser;
+ creator: JiraUser;
+ resolutiondate?: string;
+ status: JiraStatus;
+ subtasks: JiraSubTask[];
+ summary: string;
+ timeestimate?: number;
+ timespent?: number;
+ updated: string;
+ duedate?: string;
+ parent?: JiraBriefIssue;
+ };
+};
+
+export type JiraAPIResponse = {
+ expand: string;
+ startAt: number;
+ maxResults: number;
+ total: number;
+ issues: JiraIssue[];
+};
+
+/**
+ * Interface representing the parameters for configuring the
+ * JiraDocumentConverter.
+ */
+export interface JiraDocumentConverterParams {
+ host: string;
+ projectKey: string;
+}
+
+/**
+ * Class responsible for converting Jira issues to Document objects
+ */
+export class JiraDocumentConverter {
+ public readonly host: string;
+
+ public readonly projectKey: string;
+
+ constructor({ host, projectKey }: JiraDocumentConverterParams) {
+ this.host = host;
+ this.projectKey = projectKey;
+ }
+
+ public convertToDocuments(issues: JiraIssue[]): Document[] {
+ return issues.map((issue) => this.documentFromIssue(issue));
+ }
+
+ private documentFromIssue(issue: JiraIssue): Document {
+ return new Document({
+ pageContent: this.formatIssueInfo({
+ issue,
+ host: this.host,
+ }),
+ metadata: {
+ id: issue.id,
+ host: this.host,
+ projectKey: this.projectKey,
+ },
+ });
+ }
+
+ private formatIssueInfo({
+ issue,
+ host,
+ }: {
+ issue: JiraIssue;
+ host: string;
+ }): string {
+ let text = `Issue: ${this.formatMainIssueInfoText({ issue, host })}\n`;
+ text += `Project: ${issue.fields.project.name} (${issue.fields.project.key}, ID ${issue.fields.project.id})\n`;
+ text += `Status: ${issue.fields.status.name}\n`;
+ text += `Priority: ${issue.fields.priority.name}\n`;
+ text += `Type: ${issue.fields.issuetype.name}\n`;
+ text += `Creator: ${issue.fields.creator.displayName}\n`;
+
+ if (issue.fields.labels && issue.fields.labels.length > 0) {
+ text += `Labels: ${issue.fields.labels.join(", ")}\n`;
+ }
+
+ text += `Created: ${issue.fields.created}\n`;
+ text += `Updated: ${issue.fields.updated}\n`;
+
+ if (issue.fields.reporter) {
+ text += `Reporter: ${issue.fields.reporter.displayName}\n`;
+ }
+
+ text += `Assignee: ${issue.fields.assignee?.displayName ?? "Unassigned"}\n`;
+
+ if (issue.fields.duedate) {
+ text += `Due Date: ${issue.fields.duedate}\n`;
+ }
+
+ if (issue.fields.timeestimate) {
+ text += `Time Estimate: ${issue.fields.timeestimate}\n`;
+ }
+
+ if (issue.fields.timespent) {
+ text += `Time Spent: ${issue.fields.timespent}\n`;
+ }
+
+ if (issue.fields.resolutiondate) {
+ text += `Resolution Date: ${issue.fields.resolutiondate}\n`;
+ }
+
+ if (issue.fields.description) {
+ text += `Description: ${issue.fields.description}\n`;
+ }
+
+ if (issue.fields.progress.percent) {
+ text += `Progress: ${issue.fields.progress.percent}%\n`;
+ }
+
+ if (issue.fields.parent) {
+ text += `Parent Issue: ${this.formatMainIssueInfoText({
+ issue: issue.fields.parent,
+ host,
+ })}\n`;
+ }
+
+ if (issue.fields.subtasks.length > 0) {
+ text += `Subtasks:\n`;
+ issue.fields.subtasks.forEach((subtask) => {
+ text += ` - ${this.formatMainIssueInfoText({
+ issue: subtask,
+ host,
+ })}\n`;
+ });
+ }
+
+ if (issue.fields.issuelinks.length > 0) {
+ text += `Issue Links:\n`;
+ issue.fields.issuelinks.forEach((link) => {
+ text += ` - ${link.type.name}\n`;
+ if (link.inwardIssue) {
+ text += ` - ${this.formatMainIssueInfoText({
+ issue: link.inwardIssue,
+ host,
+ })}\n`;
+ }
+ if (link.outwardIssue) {
+ text += ` - ${this.formatMainIssueInfoText({
+ issue: link.outwardIssue,
+ host,
+ })}\n`;
+ }
+ });
+ }
+
+ return text;
+ }
+
+ private getLinkToIssue({
+ issueKey,
+ host,
+ }: {
+ issueKey: string;
+ host: string;
+ }): string {
+ return `${host}/browse/${issueKey}`;
+ }
+
+ private formatMainIssueInfoText({
+ issue,
+ host,
+ }: {
+ issue: JiraIssue | JiraBriefIssue;
+ host: string;
+ }): string {
+ const link = this.getLinkToIssue({
+ issueKey: issue.key,
+ host,
+ });
+
+ const text = `${issue.key} (ID ${issue.id}) - ${issue.fields.summary} (${link})`;
+
+ return text;
+ }
+}
+
+/**
+ * Interface representing the parameters for configuring the
+ * JiraProjectLoader.
+ */
+export interface JiraProjectLoaderParams {
+ host: string;
+ projectKey: string;
+ username: string;
+ accessToken: string;
+ limitPerRequest?: number;
+ createdAfter?: Date;
+}
+
+const API_ENDPOINTS = {
+ SEARCH: "/rest/api/2/search",
+};
+
+/**
+ * Class representing a document loader for loading pages from Confluence.
+ */
+export class JiraProjectLoader extends BaseDocumentLoader {
+ private readonly accessToken: string;
+
+ public readonly host: string;
+
+ public readonly projectKey: string;
+
+ public readonly username: string;
+
+ public readonly limitPerRequest: number;
+
+ private readonly createdAfter?: Date;
+
+ private readonly documentConverter: JiraDocumentConverter;
+
+ constructor({
+ host,
+ projectKey,
+ username,
+ accessToken,
+ limitPerRequest = 100,
+ createdAfter,
+ }: JiraProjectLoaderParams) {
+ super();
+ this.host = host;
+ this.projectKey = projectKey;
+ this.username = username;
+ this.accessToken = accessToken;
+ this.limitPerRequest = limitPerRequest;
+ this.createdAfter = createdAfter;
+ this.documentConverter = new JiraDocumentConverter({ host, projectKey });
+ }
+
+ private buildAuthorizationHeader(): string {
+ return `Basic ${Buffer.from(
+ `${this.username}:${this.accessToken}`
+ ).toString("base64")}`;
+ }
+
+ public async load(): Promise {
+ try {
+ const allJiraIssues = await this.loadAsIssues();
+ return this.documentConverter.convertToDocuments(allJiraIssues);
+ } catch (error) {
+ console.error("Error:", error);
+ return [];
+ }
+ }
+
+ public async loadAsIssues(): Promise {
+ const allIssues: JiraIssue[] = [];
+
+ for await (const issues of this.fetchIssues()) {
+ allIssues.push(...issues);
+ }
+
+ return allIssues;
+ }
+
+ protected toJiraDateString(date: Date | undefined): string | undefined {
+ if (!date) {
+ return undefined;
+ }
+ const year = date.getFullYear();
+ const month = String(date.getMonth() + 1).padStart(2, "0");
+ const dayOfMonth = String(date.getDate()).padStart(2, "0");
+ return `${year}-${month}-${dayOfMonth}`;
+ }
+
+ protected async *fetchIssues(): AsyncIterable {
+ const authorizationHeader = this.buildAuthorizationHeader();
+ const url = `${this.host}${API_ENDPOINTS.SEARCH}`;
+ const createdAfterAsString = this.toJiraDateString(this.createdAfter);
+ let startAt = 0;
+
+ while (true) {
+ try {
+ const jqlProps = [
+ `project=${this.projectKey}`,
+ ...(createdAfterAsString ? [`created>=${createdAfterAsString}`] : []),
+ ];
+ const params = new URLSearchParams({
+ jql: jqlProps.join(" AND "),
+ startAt: `${startAt}`,
+ maxResults: `${this.limitPerRequest}`,
+ });
+ const pageUrl = `${url}?${params}`;
+
+ const options = {
+ method: "GET",
+ headers: {
+ Authorization: authorizationHeader,
+ Accept: "application/json",
+ },
+ };
+
+ const response = await fetch(pageUrl, options);
+ const data: JiraAPIResponse = await response.json();
+
+ if (!data.issues || data.issues.length === 0) break;
+
+ yield data.issues;
+ startAt += this.limitPerRequest;
+ } catch (error) {
+ console.error(error);
+ yield [];
+ }
+ }
+ }
+}
diff --git a/libs/langchain-community/src/embeddings/hf_transformers.ts b/libs/langchain-community/src/embeddings/hf_transformers.ts
index 08175dccbb7a..dba24638cf28 100644
--- a/libs/langchain-community/src/embeddings/hf_transformers.ts
+++ b/libs/langchain-community/src/embeddings/hf_transformers.ts
@@ -1,3 +1,5 @@
+/* eslint-disable */
+// @ts-nocheck
import type {
PretrainedOptions,
FeatureExtractionPipelineOptions,
@@ -6,6 +8,11 @@ import type {
import { Embeddings, type EmbeddingsParams } from "@langchain/core/embeddings";
import { chunkArray } from "@langchain/core/utils/chunk_array";
+/**
+ * @deprecated Import from
+ * "@langchain/community/embeddings/huggingface_transformers"
+ * instead and use the new "@huggingface/transformers" peer dependency.
+ */
export interface HuggingFaceTransformersEmbeddingsParams
extends EmbeddingsParams {
/**
@@ -13,6 +20,7 @@ export interface HuggingFaceTransformersEmbeddingsParams
* Alias for `model`
*/
modelName: string;
+
/** Model name to use */
model: string;
@@ -42,24 +50,10 @@ export interface HuggingFaceTransformersEmbeddingsParams
*/
pipelineOptions?: FeatureExtractionPipelineOptions;
}
-
/**
- * @example
- * ```typescript
- * const model = new HuggingFaceTransformersEmbeddings({
- * model: "Xenova/all-MiniLM-L6-v2",
- * });
- *
- * // Embed a single query
- * const res = await model.embedQuery(
- * "What would be a good company name for a company that makes colorful socks?"
- * );
- * console.log({ res });
- *
- * // Embed multiple documents
- * const documentRes = await model.embedDocuments(["Hello world", "Bye bye"]);
- * console.log({ documentRes });
- * ```
+ * @deprecated Import from
+ * "@langchain/community/embeddings/huggingface_transformers"
+ * instead and use the new "@huggingface/transformers" peer dependency.
*/
export class HuggingFaceTransformersEmbeddings
extends Embeddings
@@ -83,7 +77,6 @@ export class HuggingFaceTransformersEmbeddings
constructor(fields?: Partial) {
super(fields ?? {});
-
this.modelName = fields?.model ?? fields?.modelName ?? this.model;
this.model = this.modelName;
this.stripNewLines = fields?.stripNewLines ?? this.stripNewLines;
@@ -95,27 +88,22 @@ export class HuggingFaceTransformersEmbeddings
...fields?.pipelineOptions,
};
}
-
async embedDocuments(texts: string[]): Promise {
const batches = chunkArray(
this.stripNewLines ? texts.map((t) => t.replace(/\n/g, " ")) : texts,
this.batchSize
);
-
const batchRequests = batches.map((batch) => this.runEmbedding(batch));
const batchResponses = await Promise.all(batchRequests);
const embeddings: number[][] = [];
-
for (let i = 0; i < batchResponses.length; i += 1) {
const batchResponse = batchResponses[i];
for (let j = 0; j < batchResponse.length; j += 1) {
embeddings.push(batchResponse[j]);
}
}
-
return embeddings;
}
-
async embedQuery(text: string): Promise {
const data = await this.runEmbedding([
this.stripNewLines ? text.replace(/\n/g, " ") : text,
diff --git a/libs/langchain-community/src/embeddings/huggingface_transformers.ts b/libs/langchain-community/src/embeddings/huggingface_transformers.ts
new file mode 100644
index 000000000000..03f458c51f38
--- /dev/null
+++ b/libs/langchain-community/src/embeddings/huggingface_transformers.ts
@@ -0,0 +1,128 @@
+import type {
+ PretrainedOptions,
+ FeatureExtractionPipelineOptions,
+ FeatureExtractionPipeline,
+} from "@huggingface/transformers";
+import { Embeddings, type EmbeddingsParams } from "@langchain/core/embeddings";
+import { chunkArray } from "@langchain/core/utils/chunk_array";
+
+export interface HuggingFaceTransformersEmbeddingsParams
+ extends EmbeddingsParams {
+ /** Model name to use */
+ model: string;
+
+ /**
+ * Timeout to use when making requests to OpenAI.
+ */
+ timeout?: number;
+
+ /**
+ * The maximum number of documents to embed in a single request.
+ */
+ batchSize?: number;
+
+ /**
+ * Whether to strip new lines from the input text. This is recommended by
+ * OpenAI, but may not be suitable for all use cases.
+ */
+ stripNewLines?: boolean;
+
+ /**
+ * Optional parameters for the pretrained model.
+ */
+ pretrainedOptions?: PretrainedOptions;
+
+ /**
+ * Optional parameters for the pipeline.
+ */
+ pipelineOptions?: FeatureExtractionPipelineOptions;
+}
+
+/**
+ * @example
+ * ```typescript
+ * const model = new HuggingFaceTransformersEmbeddings({
+ * model: "Xenova/all-MiniLM-L6-v2",
+ * });
+ *
+ * // Embed a single query
+ * const res = await model.embedQuery(
+ * "What would be a good company name for a company that makes colorful socks?"
+ * );
+ * console.log({ res });
+ *
+ * // Embed multiple documents
+ * const documentRes = await model.embedDocuments(["Hello world", "Bye bye"]);
+ * console.log({ documentRes });
+ * ```
+ */
+export class HuggingFaceTransformersEmbeddings
+ extends Embeddings
+ implements HuggingFaceTransformersEmbeddingsParams
+{
+ model = "Xenova/all-MiniLM-L6-v2";
+
+ batchSize = 512;
+
+ stripNewLines = true;
+
+ timeout?: number;
+
+ pretrainedOptions?: PretrainedOptions;
+
+ pipelineOptions?: FeatureExtractionPipelineOptions;
+
+ private pipelinePromise: Promise;
+
+ constructor(fields?: Partial) {
+ super(fields ?? {});
+
+ this.model = fields?.model ?? this.model;
+ this.stripNewLines = fields?.stripNewLines ?? this.stripNewLines;
+ this.timeout = fields?.timeout;
+ this.pretrainedOptions = fields?.pretrainedOptions ?? {};
+ this.pipelineOptions = {
+ pooling: "mean",
+ normalize: true,
+ ...fields?.pipelineOptions,
+ };
+ }
+
+ async embedDocuments(texts: string[]): Promise {
+ const batches = chunkArray(
+ this.stripNewLines ? texts.map((t) => t.replace(/\n/g, " ")) : texts,
+ this.batchSize
+ );
+
+ const batchRequests = batches.map((batch) => this.runEmbedding(batch));
+ const batchResponses = await Promise.all(batchRequests);
+ const embeddings: number[][] = [];
+
+ for (let i = 0; i < batchResponses.length; i += 1) {
+ const batchResponse = batchResponses[i];
+ for (let j = 0; j < batchResponse.length; j += 1) {
+ embeddings.push(batchResponse[j]);
+ }
+ }
+
+ return embeddings;
+ }
+
+ async embedQuery(text: string): Promise {
+ const data = await this.runEmbedding([
+ this.stripNewLines ? text.replace(/\n/g, " ") : text,
+ ]);
+ return data[0];
+ }
+
+ private async runEmbedding(texts: string[]) {
+ const pipe = await (this.pipelinePromise ??= (
+ await import("@huggingface/transformers")
+ ).pipeline("feature-extraction", this.model, this.pretrainedOptions));
+
+ return this.caller.call(async () => {
+ const output = await pipe(texts, this.pipelineOptions);
+ return output.tolist();
+ });
+ }
+}
diff --git a/libs/langchain-community/src/embeddings/tests/hf_transformers.int.test.ts b/libs/langchain-community/src/embeddings/tests/huggingface_transformers.int.test.ts
similarity index 91%
rename from libs/langchain-community/src/embeddings/tests/hf_transformers.int.test.ts
rename to libs/langchain-community/src/embeddings/tests/huggingface_transformers.int.test.ts
index de67cc9a63b0..e10caf1e7032 100644
--- a/libs/langchain-community/src/embeddings/tests/hf_transformers.int.test.ts
+++ b/libs/langchain-community/src/embeddings/tests/huggingface_transformers.int.test.ts
@@ -1,5 +1,5 @@
import { test, expect } from "@jest/globals";
-import { HuggingFaceTransformersEmbeddings } from "../hf_transformers.js";
+import { HuggingFaceTransformersEmbeddings } from "../huggingface_transformers.js";
import { HNSWLib } from "../../vectorstores/hnswlib.js";
test("HuggingFaceTransformersEmbeddings", async () => {
diff --git a/libs/langchain-community/src/load/import_constants.ts b/libs/langchain-community/src/load/import_constants.ts
index 014d418e872d..6ac412ca9543 100644
--- a/libs/langchain-community/src/load/import_constants.ts
+++ b/libs/langchain-community/src/load/import_constants.ts
@@ -15,6 +15,7 @@ export const optionalImportEntrypoints: string[] = [
"langchain_community/embeddings/gradient_ai",
"langchain_community/embeddings/hf",
"langchain_community/embeddings/hf_transformers",
+ "langchain_community/embeddings/huggingface_transformers",
"langchain_community/embeddings/ibm",
"langchain_community/embeddings/jina",
"langchain_community/embeddings/llama_cpp",
diff --git a/libs/langchain-community/src/load/import_map.ts b/libs/langchain-community/src/load/import_map.ts
index 6d00b2f172f9..624b789eb34a 100644
--- a/libs/langchain-community/src/load/import_map.ts
+++ b/libs/langchain-community/src/load/import_map.ts
@@ -78,6 +78,7 @@ export * as indexes__base from "../indexes/base.js";
export * as indexes__memory from "../indexes/memory.js";
export * as document_loaders__web__airtable from "../document_loaders/web/airtable.js";
export * as document_loaders__web__html from "../document_loaders/web/html.js";
+export * as document_loaders__web__jira from "../document_loaders/web/jira.js";
export * as document_loaders__web__searchapi from "../document_loaders/web/searchapi.js";
export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js";
export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js";
diff --git a/libs/langchain-google-common/package.json b/libs/langchain-google-common/package.json
index f5d32045f91c..5f32db49fc6a 100644
--- a/libs/langchain-google-common/package.json
+++ b/libs/langchain-google-common/package.json
@@ -1,6 +1,6 @@
{
"name": "@langchain/google-common",
- "version": "0.1.5",
+ "version": "0.1.6",
"description": "Core types and classes for Google services.",
"type": "module",
"engines": {
diff --git a/libs/langchain-google-common/src/index.ts b/libs/langchain-google-common/src/index.ts
index 373617c72e55..d1a37b6f5c70 100644
--- a/libs/langchain-google-common/src/index.ts
+++ b/libs/langchain-google-common/src/index.ts
@@ -2,6 +2,8 @@ export * from "./chat_models.js";
export * from "./llms.js";
export * from "./embeddings.js";
+export * from "./output_parsers.js";
+
export * from "./auth.js";
export * from "./connection.js";
export * from "./types.js";
diff --git a/libs/langchain-google-common/src/output_parsers.ts b/libs/langchain-google-common/src/output_parsers.ts
new file mode 100644
index 000000000000..8ca9377011ba
--- /dev/null
+++ b/libs/langchain-google-common/src/output_parsers.ts
@@ -0,0 +1,271 @@
+import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
+import { Callbacks } from "@langchain/core/callbacks/manager";
+import { ChatGeneration, Generation } from "@langchain/core/outputs";
+import { MessageContent } from "@langchain/core/messages";
+import {
+ GeminiGroundingChunk,
+ GeminiGroundingMetadata,
+ GeminiGroundingSupport,
+} from "./types.js";
+
+type Generations = Generation[] | ChatGeneration[];
+
+type GroundingInfo = {
+ metadata: GeminiGroundingMetadata;
+ supports: GeminiGroundingSupport[];
+};
+
+export abstract class BaseGoogleSearchOutputParser extends BaseLLMOutputParser {
+ lc_namespace: string[] = ["google_common", "output_parsers"];
+
+ protected generationToGroundingInfo(
+ generation: Generation | ChatGeneration
+ ): GroundingInfo | undefined {
+ if ("message" in generation) {
+ const responseMetadata = generation?.message?.response_metadata;
+ const metadata = responseMetadata.groundingMetadata;
+ const supports =
+ responseMetadata.groundingSupport ?? metadata.groundingSupports ?? [];
+ if (metadata) {
+ return {
+ metadata,
+ supports,
+ };
+ }
+ }
+ return undefined;
+ }
+
+ protected generationsToGroundingInfo(
+ generations: Generations
+ ): GroundingInfo | undefined {
+ for (const generation of generations) {
+ const info = this.generationToGroundingInfo(generation);
+ if (info !== undefined) {
+ return info;
+ }
+ }
+ return undefined;
+ }
+
+ protected generationToString(
+ generation: Generation | ChatGeneration
+ ): string {
+ if ("message" in generation) {
+ const content: MessageContent = generation?.message?.content;
+ if (typeof content === "string") {
+ return content;
+ } else {
+ return content
+ .map((c) => {
+ if (c?.type === "text") {
+ return c?.text ?? "";
+ } else {
+ return "";
+ }
+ })
+ .reduce(
+ (previousValue, currentValue) => `${previousValue}${currentValue}`
+ );
+ }
+ }
+ return generation.text;
+ }
+
+ protected generationsToString(generations: Generations): string {
+ return generations
+ .map((generation) => this.generationToString(generation))
+ .reduce(
+ (previousValue, currentValue) => `${previousValue}${currentValue}`
+ );
+ }
+
+ protected abstract segmentPrefix(
+ grounding: GroundingInfo,
+ support: GeminiGroundingSupport,
+ index: number
+ ): string | undefined;
+
+ protected abstract segmentSuffix(
+ grounding: GroundingInfo,
+ support: GeminiGroundingSupport,
+ index: number
+ ): string | undefined;
+
+ protected annotateSegment(
+ text: string,
+ grounding: GroundingInfo,
+ support: GeminiGroundingSupport,
+ index: number
+ ): string {
+ const start = support.segment.startIndex ?? 0;
+ const end = support.segment.endIndex;
+
+ const textBefore = text.substring(0, start);
+ const textSegment = text.substring(start, end);
+ const textAfter = text.substring(end);
+
+ const textPrefix = this.segmentPrefix(grounding, support, index) ?? "";
+ const textSuffix = this.segmentSuffix(grounding, support, index) ?? "";
+
+ return `${textBefore}${textPrefix}${textSegment}${textSuffix}${textAfter}`;
+ }
+
+ protected annotateTextSegments(
+ text: string,
+ grounding: GroundingInfo
+ ): string {
+ // Go through each support info in reverse, since the segment info
+ // is sorted, and we won't need to adjust string indexes this way.
+ let ret = text;
+ for (let co = grounding.supports.length - 1; co >= 0; co -= 1) {
+ const support = grounding.supports[co];
+ ret = this.annotateSegment(ret, grounding, support, co);
+ }
+ return ret;
+ }
+
+ protected abstract textPrefix(
+ text: string,
+ grounding: GroundingInfo
+ ): string | undefined;
+
+ protected abstract textSuffix(
+ text: string,
+ grounding: GroundingInfo
+ ): string | undefined;
+
+ /**
+ * Google requires us to
+ * "Display the Search Suggestion exactly as provided without any modifications"
+ * So this will typically be called from the textSuffix() method to get
+ * a string that renders HTML.
+ * See https://ai.google.dev/gemini-api/docs/grounding/search-suggestions
+ * @param grounding
+ */
+ protected searchSuggestion(grounding: GroundingInfo): string {
+ return grounding.metadata.searchEntryPoint?.renderedContent ?? "";
+ }
+
+ protected annotateText(text: string, grounding: GroundingInfo): string {
+ const prefix = this.textPrefix(text, grounding) ?? "";
+ const suffix = this.textSuffix(text, grounding) ?? "";
+ const body = this.annotateTextSegments(text, grounding);
+ return `${prefix}${body}${suffix}`;
+ }
+
+ async parseResult(
+ generations: Generations,
+ _callbacks?: Callbacks
+ ): Promise {
+ const text = this.generationsToString(generations);
+
+ const grounding = this.generationsToGroundingInfo(generations);
+ if (!grounding) {
+ return text;
+ }
+
+ return this.annotateText(text, grounding);
+ }
+}
+
+export class SimpleGoogleSearchOutputParser extends BaseGoogleSearchOutputParser {
+ protected segmentPrefix(
+ _grounding: GroundingInfo,
+ _support: GeminiGroundingSupport,
+ _index: number
+ ): string | undefined {
+ return undefined;
+ }
+
+ protected segmentSuffix(
+ _grounding: GroundingInfo,
+ support: GeminiGroundingSupport,
+ _index: number
+ ): string | undefined {
+ const indices: number[] = support.groundingChunkIndices.map((i) => i + 1);
+ return ` [${indices.join(", ")}]`;
+ }
+
+ protected textPrefix(_text: string, _grounding: GroundingInfo): string {
+ return "Google Says:\n";
+ }
+
+ protected chunkToString(chunk: GeminiGroundingChunk, index: number): string {
+ const info = chunk.retrievedContext ?? chunk.web;
+ return `${index + 1}. ${info.title} - ${info.uri}`;
+ }
+
+ protected textSuffix(_text: string, grounding: GroundingInfo): string {
+ let ret = "\n";
+ const chunks: GeminiGroundingChunk[] = grounding.metadata.groundingChunks;
+ chunks.forEach((chunk, index) => {
+ ret = `${ret}${this.chunkToString(chunk, index)}\n`;
+ });
+ return ret;
+ }
+}
+
+export class MarkdownGoogleSearchOutputParser extends BaseGoogleSearchOutputParser {
+ protected segmentPrefix(
+ _grounding: GroundingInfo,
+ _support: GeminiGroundingSupport,
+ _index: number
+ ): string | undefined {
+ return undefined;
+ }
+
+ protected chunkLink(grounding: GroundingInfo, index: number): string {
+ const chunk = grounding.metadata.groundingChunks[index];
+ const url = chunk.retrievedContext?.uri ?? chunk.web?.uri;
+ const num = index + 1;
+ return `[[${num}](${url})]`;
+ }
+
+ protected segmentSuffix(
+ grounding: GroundingInfo,
+ support: GeminiGroundingSupport,
+ _index: number
+ ): string | undefined {
+ let ret = "";
+ support.groundingChunkIndices.forEach((chunkIndex) => {
+ const link = this.chunkLink(grounding, chunkIndex);
+ ret = `${ret}${link}`;
+ });
+ return ret;
+ }
+
+ protected textPrefix(
+ _text: string,
+ _grounding: GroundingInfo
+ ): string | undefined {
+ return undefined;
+ }
+
+ protected chunkSuffixLink(
+ chunk: GeminiGroundingChunk,
+ index: number
+ ): string {
+ const num = index + 1;
+ const info = chunk.retrievedContext ?? chunk.web;
+ const url = info.uri;
+ const site = info.title;
+ return `${num}. [${site}](${url})`;
+ }
+
+ protected textSuffix(
+ _text: string,
+ grounding: GroundingInfo
+ ): string | undefined {
+ let ret = "\n**Search Sources**\n";
+ const chunks: GeminiGroundingChunk[] = grounding.metadata.groundingChunks;
+ chunks.forEach((chunk, index) => {
+ ret = `${ret}${this.chunkSuffixLink(chunk, index)}\n`;
+ });
+
+ const search = this.searchSuggestion(grounding);
+ ret = `${ret}\n${search}`;
+
+ return ret;
+ }
+}
diff --git a/libs/langchain-google-common/src/tests/chat_models.test.ts b/libs/langchain-google-common/src/tests/chat_models.test.ts
index 5726d9fd445e..0192c5b67710 100644
--- a/libs/langchain-google-common/src/tests/chat_models.test.ts
+++ b/libs/langchain-google-common/src/tests/chat_models.test.ts
@@ -35,7 +35,7 @@ import {
import { removeAdditionalProperties } from "../utils/zod_to_gemini_parameters.js";
import { MessageGeminiSafetyHandler } from "../utils/index.js";
-class ChatGoogle extends ChatGoogleBase {
+export class ChatGoogle extends ChatGoogleBase {
constructor(fields?: ChatGoogleBaseInput) {
super(fields);
}
diff --git a/libs/langchain-google-common/src/tests/data/chat-6-mock.json b/libs/langchain-google-common/src/tests/data/chat-6-mock.json
index 796fdcf9bcee..f4966c2e2f94 100644
--- a/libs/langchain-google-common/src/tests/data/chat-6-mock.json
+++ b/libs/langchain-google-common/src/tests/data/chat-6-mock.json
@@ -67,7 +67,7 @@
"endIndex": 611,
"text": "Shohei Ohtani, in his first year with the Dodgers, also experienced his first post-season appearance."
},
- "groundingChunkIndices": [0],
+ "groundingChunkIndices": [0, 2],
"confidenceScores": [0.95767003]
}
],
diff --git a/libs/langchain-google-common/src/tests/output_parsers.test.ts b/libs/langchain-google-common/src/tests/output_parsers.test.ts
new file mode 100644
index 000000000000..ac3c135f5279
--- /dev/null
+++ b/libs/langchain-google-common/src/tests/output_parsers.test.ts
@@ -0,0 +1,217 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { test } from "@jest/globals";
+import { MockClientAuthInfo, mockId } from "./mock.js";
+import { ChatGoogle } from "./chat_models.test.js";
+import {
+ MarkdownGoogleSearchOutputParser,
+ SimpleGoogleSearchOutputParser,
+} from "../output_parsers.js";
+
+describe("GoogleSearchOutputParsers", () => {
+ test("Simple", async () => {
+ const record: Record = {};
+ const projectId = mockId();
+ const authOptions: MockClientAuthInfo = {
+ record,
+ projectId,
+ resultFile: "chat-6-mock.json",
+ };
+
+ const searchRetrievalTool = {
+ googleSearchRetrieval: {
+ dynamicRetrievalConfig: {
+ mode: "MODE_DYNAMIC",
+ dynamicThreshold: 0.7, // default is 0.7
+ },
+ },
+ };
+ const model = new ChatGoogle({
+ authOptions,
+ modelName: "gemini-1.5-pro-002",
+ temperature: 0,
+ maxRetries: 0,
+ }).bindTools([searchRetrievalTool]);
+
+ const parser = new SimpleGoogleSearchOutputParser();
+
+ const chain = model.pipe(parser);
+
+ const result = await chain.invoke("Who won the 2024 MLB World Series?");
+
+ const expectation =
+ "Google Says:\n" +
+ "The Los Angeles Dodgers won the 2024 World Series, defeating the New York Yankees 4-1 in the series. [1] The Dodgers clinched the title with a 7-6 comeback victory in Game 5 at Yankee Stadium on Wednesday, October 30th. This was their eighth World Series title overall and their second in the past five years. It was also their first World Series win in a full season since 1988. [2] Mookie Betts earned his third World Series ring (2018, 2020, and 2024), becoming the only active player with three championships. [3] Shohei Ohtani, in his first year with the Dodgers, also experienced his first post-season appearance. [1, 3]\n" +
+ "\n" +
+ "1. bbc.com - https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcTYmdnM71OvWYUTG4JggmRj8cIIgA2KtKas5RPj09CiALB4n8hl-SfCD6r8WnimL2psBoYmEN9ng9sENjpeP5VxgLMTlm0zgxhrWFfx3yA6B_n0N9j-BgHLISAUi-_Ql4_Buyw68Svq-3v6BgrXzn9hLOtK\n" +
+ "2. mlb.com - https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcQRhhvHTdpb8OMOEMVxv9fkevPoMWMnhrpuC7E0E0R94xmFxT9Vv5na1hMrfHGKxVZ9aE3PgCAs5nftC3iAkeD7B6ZTfKGH2Im1CqssMM7zorGx1Ds5_7QPPBDQps_JvpkOuvRluGCVg8KwNaIU-hm3Kg==\n" +
+ "3. youtube.com - https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcSwvb2t622A2ZpKxqOWKy16L1mEUvmsAJoHjaR7uffKO71SeZkpdRXRsST9HJzJkGSkMF9kOaXGoDtcvUrttqKYOQHvHSUBYO7LWMlU00KyNlSoQzrBsgN4KuJ4O4acnNyNCSVX3-E=\n";
+
+ expect(result).toEqual(expectation);
+ });
+
+ test("Markdown", async () => {
+ const record: Record = {};
+ const projectId = mockId();
+ const authOptions: MockClientAuthInfo = {
+ record,
+ projectId,
+ resultFile: "chat-6-mock.json",
+ };
+
+ const searchRetrievalTool = {
+ googleSearchRetrieval: {
+ dynamicRetrievalConfig: {
+ mode: "MODE_DYNAMIC",
+ dynamicThreshold: 0.7, // default is 0.7
+ },
+ },
+ };
+ const model = new ChatGoogle({
+ authOptions,
+ modelName: "gemini-1.5-pro-002",
+ temperature: 0,
+ maxRetries: 0,
+ }).bindTools([searchRetrievalTool]);
+
+ const parser = new MarkdownGoogleSearchOutputParser();
+
+ const chain = model.pipe(parser);
+
+ const result = await chain.invoke("Who won the 2024 MLB World Series?");
+
+ const expectation =
+ "The Los Angeles Dodgers won the 2024 World Series, defeating the New York Yankees 4-1 in the series.[[1](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcTYmdnM71OvWYUTG4JggmRj8cIIgA2KtKas5RPj09CiALB4n8hl-SfCD6r8WnimL2psBoYmEN9ng9sENjpeP5VxgLMTlm0zgxhrWFfx3yA6B_n0N9j-BgHLISAUi-_Ql4_Buyw68Svq-3v6BgrXzn9hLOtK)] The Dodgers clinched the title with a 7-6 comeback victory in Game 5 at Yankee Stadium on Wednesday, October 30th. This was their eighth World Series title overall and their second in the past five years. It was also their first World Series win in a full season since 1988.[[2](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcQRhhvHTdpb8OMOEMVxv9fkevPoMWMnhrpuC7E0E0R94xmFxT9Vv5na1hMrfHGKxVZ9aE3PgCAs5nftC3iAkeD7B6ZTfKGH2Im1CqssMM7zorGx1Ds5_7QPPBDQps_JvpkOuvRluGCVg8KwNaIU-hm3Kg==)] Mookie Betts earned his third World Series ring (2018, 2020, and 2024), becoming the only active player with three championships.[[3](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcSwvb2t622A2ZpKxqOWKy16L1mEUvmsAJoHjaR7uffKO71SeZkpdRXRsST9HJzJkGSkMF9kOaXGoDtcvUrttqKYOQHvHSUBYO7LWMlU00KyNlSoQzrBsgN4KuJ4O4acnNyNCSVX3-E=)] Shohei Ohtani, in his first year with the Dodgers, also experienced his first post-season appearance.[[1](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcTYmdnM71OvWYUTG4JggmRj8cIIgA2KtKas5RPj09CiALB4n8hl-SfCD6r8WnimL2psBoYmEN9ng9sENjpeP5VxgLMTlm0zgxhrWFfx3yA6B_n0N9j-BgHLISAUi-_Ql4_Buyw68Svq-3v6BgrXzn9hLOtK)][[3](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcSwvb2t622A2ZpKxqOWKy16L1mEUvmsAJoHjaR7uffKO71SeZkpdRXRsST9HJzJkGSkMF9kOaXGoDtcvUrttqKYOQHvHSUBYO7LWMlU00KyNlSoQzrBsgN4KuJ4O4acnNyNCSVX3-E=)]\n" +
+ "\n" +
+ "**Search Sources**\n" +
+ "1. [bbc.com](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcTYmdnM71OvWYUTG4JggmRj8cIIgA2KtKas5RPj09CiALB4n8hl-SfCD6r8WnimL2psBoYmEN9ng9sENjpeP5VxgLMTlm0zgxhrWFfx3yA6B_n0N9j-BgHLISAUi-_Ql4_Buyw68Svq-3v6BgrXzn9hLOtK)\n" +
+ "2. [mlb.com](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcQRhhvHTdpb8OMOEMVxv9fkevPoMWMnhrpuC7E0E0R94xmFxT9Vv5na1hMrfHGKxVZ9aE3PgCAs5nftC3iAkeD7B6ZTfKGH2Im1CqssMM7zorGx1Ds5_7QPPBDQps_JvpkOuvRluGCVg8KwNaIU-hm3Kg==)\n" +
+ "3. [youtube.com](https://vertexaisearch.cloud.google.com/grounding-api-redirect/AYygrcSwvb2t622A2ZpKxqOWKy16L1mEUvmsAJoHjaR7uffKO71SeZkpdRXRsST9HJzJkGSkMF9kOaXGoDtcvUrttqKYOQHvHSUBYO7LWMlU00KyNlSoQzrBsgN4KuJ4O4acnNyNCSVX3-E=)\n" +
+ "\n" +
+ "\n" +
+ '\n' +
+ '
\n' +
+ '
\n" +
+ '
\n" +
+ '
\n' +
+ "
\n" +
+ '
\n" +
+ "