diff --git a/docs/docs/integrations/document_loaders/pdfplumber.ipynb b/docs/docs/integrations/document_loaders/pdfplumber.ipynb
index cfa43817f1076f..3b353dc48e2036 100644
--- a/docs/docs/integrations/document_loaders/pdfplumber.ipynb
+++ b/docs/docs/integrations/document_loaders/pdfplumber.ipynb
@@ -4,44 +4,55 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# PDFPlumber\n",
+ "# PDFPlumberLoader\n",
"\n",
- "Like PyMuPDF, the output Documents contain detailed metadata about the PDF and its pages, and returns one document per page.\n",
+ "This notebook provides a quick overview for getting started with `PDFMiner` [document loader](https://python.langchain.com/docs/concepts/document_loaders). For detailed documentation of all __ModuleName__Loader features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PDFPlumberLoader.html).\n",
+ "\n",
+ " \n",
"\n",
"## Overview\n",
"### Integration details\n",
"\n",
- "| Class | Package | Local | Serializable | JS support|\n",
- "| :--- | :--- | :---: | :---: | :---: |\n",
- "| [PDFPlumberLoader](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PDFPlumberLoader.html) | [langchain_community](https://python.langchain.com/api_reference/community/index.html) | ✅ | ❌ | ❌ | \n",
+ "| Class | Package | Local | Serializable | JS support|\n",
+ "|:---------------------------------------------------------------------------------------------------------------------------------------------------------| :--- | :---: | :---: | :---: |\n",
+ "| [PDFPlumberLoader](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PDFPlumberLoader.html) | [langchain_community](https://python.langchain.com/api_reference/community/index.html) | ✅ | ❌ | ❌ |\n",
+ "\n",
+ "--------- \n",
+ "\n",
"### Loader features\n",
- "| Source | Document Lazy Loading | Native Async Support\n",
- "| :---: | :---: | :---: | \n",
- "| PDFPlumberLoader | ✅ | ❌ | \n",
+ "\n",
+ "| Source | Document Lazy Loading | Native Async Support | Extract Images | Extract Tables |\n",
+ "|:----------------:| :---: | :---: | :---: |:---: |\n",
+ "| PDFPlumberLoader | ✅ | ❌ | ✅ | ✅ |\n",
+ "\n",
+ " \n",
"\n",
"## Setup\n",
"\n",
"### Credentials\n",
"\n",
- "No credentials are needed to use this loader."
+ "No credentials are required to use PyMuPDFLoader"
]
},
{
"cell_type": "markdown",
"metadata": {},
- "source": [
- "If you want to get automated best in-class tracing of your model calls you can also set your [LangSmith](https://docs.smith.langchain.com/) API key by uncommenting below:"
- ]
+ "source": "If you want to get automated best in-class tracing of your model calls you can also set your [LangSmith](https://docs.smith.langchain.com/) API key by uncommenting below:"
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:39:37.440900Z",
+ "start_time": "2025-02-10T08:39:37.438441Z"
+ }
+ },
"source": [
"# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n",
"# os.environ[\"LANGSMITH_TRACING\"] = \"true\""
- ]
+ ],
+ "outputs": [],
+ "execution_count": 1
},
{
"cell_type": "markdown",
@@ -49,17 +60,28 @@
"source": [
"### Installation\n",
"\n",
- "Install **langchain_community**."
+ "Install **langchain_community** and **pymupdf**."
]
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%pip install -qU langchain_community"
- ]
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:39:41.487372Z",
+ "start_time": "2025-02-10T08:39:39.209073Z"
+ }
+ },
+ "source": "%pip install -qU langchain_community pdfplumber",
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
+ "execution_count": 2
},
{
"cell_type": "markdown",
@@ -72,14 +94,20 @@
},
{
"cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:39:45.246502Z",
+ "start_time": "2025-02-10T08:39:44.229183Z"
+ }
+ },
"source": [
"from langchain_community.document_loaders import PDFPlumberLoader\n",
"\n",
- "loader = PDFPlumberLoader(\"./example_data/layout-parser-paper.pdf\")"
- ]
+ "file_path = \"./example_data/layout-parser-paper.pdf\"\n",
+ "loader = PDFPlumberLoader(file_path)"
+ ],
+ "outputs": [],
+ "execution_count": 3
},
{
"cell_type": "markdown",
@@ -90,78 +118,1028 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:40:11.901128Z",
+ "start_time": "2025-02-10T08:39:46.905899Z"
+ }
+ },
+ "source": [
+ "docs = loader.load()\n",
+ "docs[0]"
+ ],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Document(metadata={'author': '', 'creationdate': '2021-06-22T01:27:10+00:00', 'creator': 'LaTeX with hyperref', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'producer': 'pdfTeX-1.40.21', 'subject': '', 'title': '', 'trapped': 'False', 'source': './example_data/layout-parser-paper.pdf', 'file_path': './example_data/layout-parser-paper.pdf', 'total_pages': 16, 'page': 0}, page_content='LayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1 ((cid:0)), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recentadvancesindocumentimageanalysis(DIA)havebeen\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomescouldbeeasilydeployedinproductionandextendedforfurther\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportantinnovationsbyawideaudience.Thoughtherehavebeenon-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopmentindisciplineslikenaturallanguageprocessingandcomputer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademicresearchacross awiderangeof disciplinesinthesocialsciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitiveinterfacesforapplyingandcustomizingDLmodelsforlayoutde-\\ntection,characterrecognition,andmanyotherdocumentprocessingtasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\nKeywords: DocumentImageAnalysis·DeepLearning·LayoutAnalysis\\n· Character Recognition · Open Source library · Toolkit.\\n1 Introduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocumentimageanalysis(DIA)tasksincludingdocumentimageclassification[11,\\n1202 nuJ 12 ]VC.sc[ 2v84351.3012:viXra\\n')"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "execution_count": 4
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:40:20.094848Z",
+ "start_time": "2025-02-10T08:40:20.083124Z"
+ }
+ },
+ "source": [
+ "import pprint\n",
+ "\n",
+ "pprint.pp(docs[0].metadata)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'author': '',\n",
+ " 'creationdate': '2021-06-22T01:27:10+00:00',\n",
+ " 'creator': 'LaTeX with hyperref',\n",
+ " 'keywords': '',\n",
+ " 'moddate': '2021-06-22T01:27:10+00:00',\n",
+ " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n",
+ " '2020) kpathsea version 6.3.2',\n",
+ " 'producer': 'pdfTeX-1.40.21',\n",
+ " 'subject': '',\n",
+ " 'title': '',\n",
+ " 'trapped': 'False',\n",
+ " 'source': './example_data/layout-parser-paper.pdf',\n",
+ " 'file_path': './example_data/layout-parser-paper.pdf',\n",
+ " 'total_pages': 16,\n",
+ " 'page': 0}\n"
+ ]
+ }
+ ],
+ "execution_count": 5
+ },
+ {
+ "cell_type": "markdown",
"metadata": {},
+ "source": [
+ "## Lazy Load\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:40:45.605691Z",
+ "start_time": "2025-02-10T08:40:22.639608Z"
+ }
+ },
+ "source": [
+ "pages = []\n",
+ "for doc in loader.lazy_load():\n",
+ " pages.append(doc)\n",
+ " if len(pages) >= 10:\n",
+ " # do some paged operation, e.g.\n",
+ " # index.upsert(page)\n",
+ "\n",
+ " pages = []\n",
+ "len(pages)"
+ ],
"outputs": [
{
"data": {
"text/plain": [
- "Document(metadata={'source': './example_data/layout-parser-paper.pdf', 'file_path': './example_data/layout-parser-paper.pdf', 'page': 0, 'total_pages': 16, 'Author': '', 'CreationDate': 'D:20210622012710Z', 'Creator': 'LaTeX with hyperref', 'Keywords': '', 'ModDate': 'D:20210622012710Z', 'PTEX.Fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'Producer': 'pdfTeX-1.40.21', 'Subject': '', 'Title': '', 'Trapped': 'False'}, page_content='LayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1 ((cid:0)), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recentadvancesindocumentimageanalysis(DIA)havebeen\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomescouldbeeasilydeployedinproductionandextendedforfurther\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportantinnovationsbyawideaudience.Thoughtherehavebeenon-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopmentindisciplineslikenaturallanguageprocessingandcomputer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademicresearchacross awiderangeof disciplinesinthesocialsciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitiveinterfacesforapplyingandcustomizingDLmodelsforlayoutde-\\ntection,characterrecognition,andmanyotherdocumentprocessingtasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\nKeywords: DocumentImageAnalysis·DeepLearning·LayoutAnalysis\\n· Character Recognition · Open Source library · Toolkit.\\n1 Introduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocumentimageanalysis(DIA)tasksincludingdocumentimageclassification[11,\\n1202\\nnuJ\\n12\\n]VC.sc[\\n2v84351.3012:viXra\\n')"
+ "6"
]
},
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
+ "execution_count": 6
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:40:46.969036Z",
+ "start_time": "2025-02-10T08:40:46.964794Z"
+ }
+ },
"source": [
+ "print(pages[0].page_content[:100])\n",
+ "pprint.pp(pages[0].metadata)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "LayoutParser: A Unified Toolkit for DL-Based DIA 11\n",
+ "focuses on precision, efficiency, and robustness\n",
+ "{'author': '',\n",
+ " 'creationdate': '2021-06-22T01:27:10+00:00',\n",
+ " 'creator': 'LaTeX with hyperref',\n",
+ " 'keywords': '',\n",
+ " 'moddate': '2021-06-22T01:27:10+00:00',\n",
+ " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n",
+ " '2020) kpathsea version 6.3.2',\n",
+ " 'producer': 'pdfTeX-1.40.21',\n",
+ " 'subject': '',\n",
+ " 'title': '',\n",
+ " 'trapped': 'False',\n",
+ " 'source': './example_data/layout-parser-paper.pdf',\n",
+ " 'file_path': './example_data/layout-parser-paper.pdf',\n",
+ " 'total_pages': 16,\n",
+ " 'page': 10}\n"
+ ]
+ }
+ ],
+ "execution_count": 7
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The metadata attribute contains at least the following keys:\n",
+ "- source\n",
+ "- page (if in mode *page*)\n",
+ "- total_page\n",
+ "- creationdate\n",
+ "- creator\n",
+ "- producer\n",
+ "\n",
+ "Additional metadata are specific to each parser.\n",
+ "These pieces of information can be helpful (to categorize your PDFs for example)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Splitting mode & custom pages delimiter"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "When loading the PDF file you can split it in two different ways:\n",
+ "- By page\n",
+ "- As a single text flow\n",
+ "\n",
+ "By default PDFPlumberLoader will split the PDF by page."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Extract the PDF by page. Each page is extracted as a langchain Document object:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:41:14.319842Z",
+ "start_time": "2025-02-10T08:40:50.665569Z"
+ }
+ },
+ "source": [
+ "loader = PDFPlumberLoader(\n",
+ " \"./example_data/layout-parser-paper.pdf\",\n",
+ " mode=\"page\",\n",
+ ")\n",
"docs = loader.load()\n",
- "docs[0]"
+ "print(len(docs))\n",
+ "pprint.pp(docs[0].metadata)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "16\n",
+ "{'author': '',\n",
+ " 'creationdate': '2021-06-22T01:27:10+00:00',\n",
+ " 'creator': 'LaTeX with hyperref',\n",
+ " 'keywords': '',\n",
+ " 'moddate': '2021-06-22T01:27:10+00:00',\n",
+ " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n",
+ " '2020) kpathsea version 6.3.2',\n",
+ " 'producer': 'pdfTeX-1.40.21',\n",
+ " 'subject': '',\n",
+ " 'title': '',\n",
+ " 'trapped': 'False',\n",
+ " 'source': './example_data/layout-parser-paper.pdf',\n",
+ " 'file_path': './example_data/layout-parser-paper.pdf',\n",
+ " 'total_pages': 16,\n",
+ " 'page': 0}\n"
+ ]
+ }
+ ],
+ "execution_count": 8
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In this mode the pdf is split by pages and the resulting Documents metadata contains the page number. But in some cases we could want to process the pdf as a single text flow (so we don't cut some paragraphs in half). In this case you can use the *single* mode :"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Extract the whole PDF as a single langchain Document object:"
]
},
{
"cell_type": "code",
- "execution_count": 6,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:41:41.244786Z",
+ "start_time": "2025-02-10T08:41:17.564901Z"
+ }
+ },
+ "source": [
+ "loader = PDFPlumberLoader(\n",
+ " \"./example_data/layout-parser-paper.pdf\",\n",
+ " mode=\"single\",\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "print(len(docs))\n",
+ "pprint.pp(docs[0].metadata)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1\n",
+ "{'author': '',\n",
+ " 'creationdate': '2021-06-22T01:27:10+00:00',\n",
+ " 'creator': 'LaTeX with hyperref',\n",
+ " 'keywords': '',\n",
+ " 'moddate': '2021-06-22T01:27:10+00:00',\n",
+ " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n",
+ " '2020) kpathsea version 6.3.2',\n",
+ " 'producer': 'pdfTeX-1.40.21',\n",
+ " 'subject': '',\n",
+ " 'title': '',\n",
+ " 'trapped': 'False',\n",
+ " 'source': './example_data/layout-parser-paper.pdf',\n",
+ " 'file_path': './example_data/layout-parser-paper.pdf',\n",
+ " 'total_pages': 16}\n"
+ ]
+ }
+ ],
+ "execution_count": 9
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Logically, in this mode, the ‘page_number’ metadata disappears. Here's how to clearly identify where pages end in the text flow :"
+ ]
+ },
+ {
+ "cell_type": "markdown",
"metadata": {},
+ "source": "### Add a custom *pages_delimiter* to identify where are ends of pages in *single* mode:"
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:42:07.936745Z",
+ "start_time": "2025-02-10T08:41:44.463505Z"
+ }
+ },
+ "source": [
+ "loader = PDFPlumberLoader(\n",
+ " \"./example_data/layout-parser-paper.pdf\",\n",
+ " mode=\"single\",\n",
+ " pages_delimiter=\"\\n-------THIS IS A CUSTOM END OF PAGE-------\\n\",\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "print(docs[0].page_content[:5780])"
+ ],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "{'source': './example_data/layout-parser-paper.pdf', 'file_path': './example_data/layout-parser-paper.pdf', 'page': 0, 'total_pages': 16, 'Author': '', 'CreationDate': 'D:20210622012710Z', 'Creator': 'LaTeX with hyperref', 'Keywords': '', 'ModDate': 'D:20210622012710Z', 'PTEX.Fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'Producer': 'pdfTeX-1.40.21', 'Subject': '', 'Title': '', 'Trapped': 'False'}\n"
+ "LayoutParser: A Unified Toolkit for Deep\n",
+ "Learning Based Document Image Analysis\n",
+ "Zejiang Shen1 ((cid:0)), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\n",
+ "Lee4, Jacob Carlson3, and Weining Li5\n",
+ "1 Allen Institute for AI\n",
+ "shannons@allenai.org\n",
+ "2 Brown University\n",
+ "ruochen zhang@brown.edu\n",
+ "3 Harvard University\n",
+ "{melissadell,jacob carlson}@fas.harvard.edu\n",
+ "4 University of Washington\n",
+ "bcgl@cs.washington.edu\n",
+ "5 University of Waterloo\n",
+ "w422li@uwaterloo.ca\n",
+ "Abstract. Recentadvancesindocumentimageanalysis(DIA)havebeen\n",
+ "primarily driven by the application of neural networks. Ideally, research\n",
+ "outcomescouldbeeasilydeployedinproductionandextendedforfurther\n",
+ "investigation. However, various factors like loosely organized codebases\n",
+ "and sophisticated model configurations complicate the easy reuse of im-\n",
+ "portantinnovationsbyawideaudience.Thoughtherehavebeenon-going\n",
+ "efforts to improve reusability and simplify deep learning (DL) model\n",
+ "developmentindisciplineslikenaturallanguageprocessingandcomputer\n",
+ "vision, none of them are optimized for challenges in the domain of DIA.\n",
+ "This represents a major gap in the existing toolkit, as DIA is central to\n",
+ "academicresearchacross awiderangeof disciplinesinthesocialsciences\n",
+ "and humanities. This paper introduces LayoutParser, an open-source\n",
+ "library for streamlining the usage of DL in DIA research and applica-\n",
+ "tions. The core LayoutParser library comes with a set of simple and\n",
+ "intuitiveinterfacesforapplyingandcustomizingDLmodelsforlayoutde-\n",
+ "tection,characterrecognition,andmanyotherdocumentprocessingtasks.\n",
+ "To promote extensibility, LayoutParser also incorporates a community\n",
+ "platform for sharing both pre-trained models and full document digiti-\n",
+ "zation pipelines. We demonstrate that LayoutParser is helpful for both\n",
+ "lightweight and large-scale digitization pipelines in real-word use cases.\n",
+ "The library is publicly available at https://layout-parser.github.io.\n",
+ "Keywords: DocumentImageAnalysis·DeepLearning·LayoutAnalysis\n",
+ "· Character Recognition · Open Source library · Toolkit.\n",
+ "1 Introduction\n",
+ "Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of\n",
+ "documentimageanalysis(DIA)tasksincludingdocumentimageclassification[11,\n",
+ "1202 nuJ 12 ]VC.sc[ 2v84351.3012:viXra\n",
+ "-------THIS IS A CUSTOM END OF PAGE-------\n",
+ "2 Z. Shen et al.\n",
+ "37], layout detection [38, 22], table detection [26], and scene text detection [4].\n",
+ "A generalized learning-based framework dramatically reduces the need for the\n",
+ "manualspecificationofcomplicatedrules,whichisthestatusquowithtraditional\n",
+ "methods. DL has the potential to transform DIA pipelines and benefit a broad\n",
+ "spectrum of large-scale document digitization projects.\n",
+ "However, there are several practical difficulties for taking advantages of re-\n",
+ "cent advances in DL-based methods: 1) DL models are notoriously convoluted\n",
+ "for reuse and extension. Existing models are developed using distinct frame-\n",
+ "works like TensorFlow [1] or PyTorch [24], and the high-level parameters can\n",
+ "be obfuscated by implementation details [8]. It can be a time-consuming and\n",
+ "frustrating experience to debug, reproduce, and adapt existing models for DIA,\n",
+ "and many researchers who would benefit the most from using these methods lack\n",
+ "the technical background to implement them from scratch. 2) Document images\n",
+ "contain diverse and disparate patterns across domains, and customized training\n",
+ "is often required to achieve a desirable detection accuracy. Currently there is no\n",
+ "full-fledged infrastructure for easily curating the target document image datasets\n",
+ "and fine-tuning or re-training the models. 3) DIA usually requires a sequence of\n",
+ "modelsandotherprocessingtoobtainthefinaloutputs.Oftenresearchteamsuse\n",
+ "DL models and then perform further document analyses in separate processes,\n",
+ "and these pipelines are not documented in any central location (and often not\n",
+ "documented at all). This makes it difficult for research teams to learn about how\n",
+ "full pipelines are implemented and leads them to invest significant resources in\n",
+ "reinventing the DIA wheel.\n",
+ "LayoutParserprovidesaunifiedtoolkittosupportDL-baseddocumentimage\n",
+ "analysisandprocessing.Toaddresstheaforementionedchallenges,LayoutParser\n",
+ "is built with the following components:\n",
+ "1. Anoff-the-shelftoolkitforapplyingDLmodelsforlayoutdetection,character\n",
+ "recognition, and other DIA tasks (Section 3)\n",
+ "2. A rich repository of pre-trained neural network models (Model Zoo) that\n",
+ "underlies the off-the-shelf usage\n",
+ "3. Comprehensivetoolsforefficientdocumentimagedataannotationandmodel\n",
+ "tuning to support different levels of customization\n",
+ "4. A DL model hub and community platform for the easy sharing, distribu-\n",
+ "tion, and discussion of DIA models and pipelines, to promote reusability,\n",
+ "reproducibility, and extensibility (Section 4)\n",
+ "The library implements simple and intuitive Python APIs without sacrificing\n",
+ "generalizability and versatility, and can be easily installed via pip. Its convenient\n",
+ "functions for handling document image data can be seamlessly integrated with\n",
+ "existing DIA pipelines. With detailed documentations and carefully curated\n",
+ "tutorials, we hope this tool will benefit a variety of end-users, and will lead to\n",
+ "advances in applications in both industry and academic research.\n",
+ "LayoutParser is well aligned with recent efforts for improving DL model\n",
+ "reusability in other disciplines like natural language processing [8, 34] and com-\n",
+ "puter vision [35], but with a focus on unique challenges in DIA. We show\n",
+ "LayoutParsercanbeappliedinsophisticatedandlarge-scaledigitizationprojects\n",
+ "-------THIS IS A CUSTOM END OF PAGE-------\n",
+ "LayoutParser: A Unified Toolkit for DL-Based DIA 3\n",
+ "that require precision, efficiency, and robustness, as well as simple and light-\n",
+ "weight document processing tasks focusing on efficacy and flexibility (Section 5).\n",
+ "LayoutParser is being actively mainta\n"
]
}
],
+ "execution_count": 10
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
- "print(docs[0].metadata)"
+ "This could simply be \\n, or \\f to clearly indicate a page change, or \\ for seamless injection in a Markdown viewer without a visual effect."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Lazy Load"
+ "# Extract images from the PDF"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can extract images from your PDFs with a choice of three different solutions:\n",
+ "- rapidOCR (lightweight Optical Character Recognition tool)\n",
+ "- Tesseract (OCR tool with high precision)\n",
+ "- Multimodal language model\n",
+ "\n",
+ "You can tune these functions to choose the output format of the extracted images among *html*, *markdown* or *text*\n",
+ "\n",
+ "The result is inserted between the last and the second-to-last paragraphs of text of the page."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Extract images from the PDF with rapidOCR:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:42:14.077933Z",
+ "start_time": "2025-02-10T08:42:12.007265Z"
+ }
+ },
+ "source": [
+ "%pip install -qU rapidocr-onnxruntime"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
+ "execution_count": 11
+ },
{
"cell_type": "code",
- "execution_count": 7,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:42:38.934138Z",
+ "start_time": "2025-02-10T08:42:15.188110Z"
+ }
+ },
+ "source": [
+ "from langchain_community.document_loaders.parsers import RapidOCRBlobParser\n",
+ "\n",
+ "loader = PDFPlumberLoader(\n",
+ " \"./example_data/layout-parser-paper.pdf\",\n",
+ " mode=\"page\",\n",
+ " images_inner_format=\"markdown-img\",\n",
+ " images_parser=RapidOCRBlobParser(),\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "\n",
+ "print(docs[5].page_content)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "6 Z. Shen et al.\n",
+ "Fig.2: The relationship between the three types of layout data structures.\n",
+ "Coordinate supports three kinds of variation; TextBlock consists of the co-\n",
+ "ordinateinformationandextrafeatureslikeblocktext,types,andreadingorders;\n",
+ "a Layout object is a list of all possible layout elements, including other Layout\n",
+ "objects. They all support the same set of transformation and operation APIs for\n",
+ "maximum flexibility.\n",
+ "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n",
+ "on 5 different datasets. Description of the training dataset is provided alongside\n",
+ "with the trained models such that users can quickly identify the most suitable\n",
+ "models for their tasks. Additionally, when such a model is not readily available,\n",
+ "LayoutParser also supports training customized layout models and community\n",
+ "sharing of the models (detailed in Section 3.5).\n",
+ "3.2 Layout Data Structures\n",
+ "A critical feature of LayoutParser is the implementation of a series of data\n",
+ "structures and operations that can be used to efficiently process and manipulate\n",
+ "thelayoutelements.Indocumentimageanalysispipelines,variouspost-processing\n",
+ "on the layout analysis model outputs is usually required to obtain the final\n",
+ "outputs.Traditionally,thisrequiresexportingDLmodeloutputsandthenloading\n",
+ "the results into other pipelines. All model outputs from LayoutParser will be\n",
+ "stored in carefully engineered data types optimized for further processing, which\n",
+ "makes it possible to build an end-to-end document digitization pipeline within\n",
+ "LayoutParser. There are three key components in the data structure, namely\n",
+ "the Coordinate system, the TextBlock, and the Layout. They provide different\n",
+ "levels of abstraction for the layout data, and a set of APIs are supported for\n",
+ "transformations or operations on these classes.\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ],
+ "execution_count": 12
+ },
+ {
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "page = []\n",
- "for doc in loader.lazy_load():\n",
- " page.append(doc)\n",
- " if len(page) >= 10:\n",
- " # do some paged operation, e.g.\n",
- " # index.upsert(page)\n",
+ "Be careful, RapidOCR is designed to work with Chinese and English, not other languages."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Extract images from the PDF with Tesseract:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:44:01.430937Z",
+ "start_time": "2025-02-10T08:43:59.573391Z"
+ }
+ },
+ "source": [
+ "%pip install -qU pytesseract"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
+ "execution_count": 13
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:44:12.786743Z",
+ "start_time": "2025-02-10T08:44:02.309333Z"
+ }
+ },
+ "source": [
+ "from langchain_community.document_loaders.parsers import TesseractBlobParser\n",
"\n",
- " page = []"
+ "loader = PDFPlumberLoader(\n",
+ " \"./example_data/layout-parser-paper.pdf\",\n",
+ " mode=\"page\",\n",
+ " images_inner_format=\"html-img\",\n",
+ " images_parser=TesseractBlobParser(),\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "print(docs[5].page_content)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "6 Z. Shen et al.\n",
+ "Fig.2: The relationship between the three types of layout data structures.\n",
+ "Coordinate supports three kinds of variation; TextBlock consists of the co-\n",
+ "ordinateinformationandextrafeatureslikeblocktext,types,andreadingorders;\n",
+ "a Layout object is a list of all possible layout elements, including other Layout\n",
+ "objects. They all support the same set of transformation and operation APIs for\n",
+ "maximum flexibility.\n",
+ "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n",
+ "on 5 different datasets. Description of the training dataset is provided alongside\n",
+ "with the trained models such that users can quickly identify the most suitable\n",
+ "models for their tasks. Additionally, when such a model is not readily available,\n",
+ "LayoutParser also supports training customized layout models and community\n",
+ "sharing of the models (detailed in Section 3.5).\n",
+ "3.2 Layout Data Structures\n",
+ "A critical feature of LayoutParser is the implementation of a series of data\n",
+ "structures and operations that can be used to efficiently process and manipulate\n",
+ "thelayoutelements.Indocumentimageanalysispipelines,variouspost-processing\n",
+ "on the layout analysis model outputs is usually required to obtain the final\n",
+ "outputs.Traditionally,thisrequiresexportingDLmodeloutputsandthenloading\n",
+ "the results into other pipelines. All model outputs from LayoutParser will be\n",
+ "stored in carefully engineered data types optimized for further processing, which\n",
+ "makes it possible to build an end-to-end document digitization pipeline within\n",
+ "LayoutParser. There are three key components in the data structure, namely\n",
+ "the Coordinate system, the TextBlock, and the Layout. They provide different\n",
+ "levels of abstraction for the layout data, and a set of APIs are supported for\n",
+ "transformations or operations on these classes.\n",
+ "\n",
+ "![\"Coordinate\n",](\"#\")
\n",
+ "\n"
+ ]
+ }
+ ],
+ "execution_count": 14
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Extract images from the PDF with multimodal model:"
]
},
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:46:05.694249Z",
+ "start_time": "2025-02-10T08:46:03.558918Z"
+ }
+ },
+ "source": [
+ "%pip install -qU langchain_openai"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
+ "execution_count": 16
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:46:07.818185Z",
+ "start_time": "2025-02-10T08:46:07.794265Z"
+ }
+ },
+ "source": [
+ "import os\n",
+ "\n",
+ "from dotenv import load_dotenv\n",
+ "\n",
+ "load_dotenv()"
+ ],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "execution_count": 17
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:46:09.139886Z",
+ "start_time": "2025-02-10T08:46:09.137577Z"
+ }
+ },
+ "source": [
+ "from getpass import getpass\n",
+ "\n",
+ "if not os.environ.get(\"OPENAI_API_KEY\"):\n",
+ " os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")"
+ ],
+ "outputs": [],
+ "execution_count": 18
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:47:19.810461Z",
+ "start_time": "2025-02-10T08:46:10.995012Z"
+ }
+ },
+ "source": [
+ "from langchain_community.document_loaders.parsers import LLMImageBlobParser\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "loader = PDFPlumberLoader(\n",
+ " \"./example_data/layout-parser-paper.pdf\",\n",
+ " mode=\"page\",\n",
+ " images_inner_format=\"markdown-img\",\n",
+ " images_parser=LLMImageBlobParser(model=ChatOpenAI(model=\"gpt-4o\", max_tokens=1024)),\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "print(docs[5].page_content)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "6 Z. Shen et al.\n",
+ "Fig.2: The relationship between the three types of layout data structures.\n",
+ "Coordinate supports three kinds of variation; TextBlock consists of the co-\n",
+ "ordinateinformationandextrafeatureslikeblocktext,types,andreadingorders;\n",
+ "a Layout object is a list of all possible layout elements, including other Layout\n",
+ "objects. They all support the same set of transformation and operation APIs for\n",
+ "maximum flexibility.\n",
+ "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n",
+ "on 5 different datasets. Description of the training dataset is provided alongside\n",
+ "with the trained models such that users can quickly identify the most suitable\n",
+ "models for their tasks. Additionally, when such a model is not readily available,\n",
+ "LayoutParser also supports training customized layout models and community\n",
+ "sharing of the models (detailed in Section 3.5).\n",
+ "3.2 Layout Data Structures\n",
+ "A critical feature of LayoutParser is the implementation of a series of data\n",
+ "structures and operations that can be used to efficiently process and manipulate\n",
+ "thelayoutelements.Indocumentimageanalysispipelines,variouspost-processing\n",
+ "on the layout analysis model outputs is usually required to obtain the final\n",
+ "outputs.Traditionally,thisrequiresexportingDLmodeloutputsandthenloading\n",
+ "the results into other pipelines. All model outputs from LayoutParser will be\n",
+ "stored in carefully engineered data types optimized for further processing, which\n",
+ "makes it possible to build an end-to-end document digitization pipeline within\n",
+ "LayoutParser. There are three key components in the data structure, namely\n",
+ "the Coordinate system, the TextBlock, and the Layout. They provide different\n",
+ "levels of abstraction for the layout data, and a set of APIs are supported for\n",
+ "transformations or operations on these classes.\n",
+ "\n",
+ "![**Image Summary:**\n",
+ "Diagram illustrating coordinate, text block, and layout elements with transformation and operation APIs. Includes coordinate intervals, rectangles, quadrilaterals, and extra features like block text, block type, and reading order.\n",
+ "\n",
+ "**Extracted Text:**\n",
+ "```\n",
+ "Coordinate\n",
+ "\n",
+ "(x1, y1)\n",
+ "\n",
+ "Rectangle\n",
+ "\n",
+ "(x2, y2)\n",
+ "\n",
+ "(x1, y1)\n",
+ "\n",
+ "(x2, y2)\n",
+ "\n",
+ "(x4, y4)\n",
+ "\n",
+ "(x3, y3)\n",
+ "\n",
+ "Quadrilateral\n",
+ "\n",
+ "The same transformation and operation APIs\n",
+ "\n",
+ "textblock\n",
+ "\n",
+ "Coordinate\n",
+ "\n",
+ "+\n",
+ "\n",
+ "Extra features\n",
+ "\n",
+ "Block Text\n",
+ "\n",
+ "Block Type\n",
+ "\n",
+ "Reading Order\n",
+ "\n",
+ "…\n",
+ "\n",
+ "layout\n",
+ "\n",
+ "[ coordinate1, textblock1, ...\n",
+ "\n",
+ "…, textblock2, layout1 \\\\]\n",
+ "\n",
+ "A list of the layout elements\n",
+ "\n",
+ "x- interval\n",
+ "st a rt\n",
+ "\n",
+ "start\n",
+ "\n",
+ "y-interval\n",
+ "\n",
+ "en d\n",
+ "\n",
+ "end\n",
+ "```](#)\n",
+ "\n"
+ ]
+ }
+ ],
+ "execution_count": 19
+ },
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "## API reference\n",
+ "## Working with Files\n",
"\n",
- "For detailed documentation of all PDFPlumberLoader features and configurations head to the API reference: https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PDFPlumberLoader.html"
+ "Many document loaders involve parsing files. The difference between such loaders usually stems from how the file is parsed, rather than how the file is loaded. For example, you can use `open` to read the binary content of either a PDF or a markdown file, but you need different parsing logic to convert that binary data into text.\n",
+ "\n",
+ "As a result, it can be helpful to decouple the parsing logic from the loading logic, which makes it easier to re-use a given parser regardless of how the data was loaded.\n",
+ "You can use this strategy to analyze different files, with the same parsing parameters."
]
+ },
+ {
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2025-02-10T08:49:08.041155Z",
+ "start_time": "2025-02-10T08:48:40.584715Z"
+ }
+ },
+ "cell_type": "code",
+ "source": [
+ "from langchain_community.document_loaders import FileSystemBlobLoader\n",
+ "from langchain_community.document_loaders.generic import GenericLoader\n",
+ "from langchain_community.document_loaders.parsers import PDFPlumberParser\n",
+ "\n",
+ "loader = GenericLoader(\n",
+ " blob_loader=FileSystemBlobLoader(\n",
+ " path=\"./example_data/\",\n",
+ " glob=\"*.pdf\",\n",
+ " ),\n",
+ " blob_parser=PDFPlumberParser(),\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "print(docs[0].page_content)\n",
+ "pprint.pp(docs[0].metadata)"
+ ],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "LayoutParser: A Unified Toolkit for Deep\n",
+ "Learning Based Document Image Analysis\n",
+ "Zejiang Shen1 ((cid:0)), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\n",
+ "Lee4, Jacob Carlson3, and Weining Li5\n",
+ "1 Allen Institute for AI\n",
+ "shannons@allenai.org\n",
+ "2 Brown University\n",
+ "ruochen zhang@brown.edu\n",
+ "3 Harvard University\n",
+ "{melissadell,jacob carlson}@fas.harvard.edu\n",
+ "4 University of Washington\n",
+ "bcgl@cs.washington.edu\n",
+ "5 University of Waterloo\n",
+ "w422li@uwaterloo.ca\n",
+ "Abstract. Recentadvancesindocumentimageanalysis(DIA)havebeen\n",
+ "primarily driven by the application of neural networks. Ideally, research\n",
+ "outcomescouldbeeasilydeployedinproductionandextendedforfurther\n",
+ "investigation. However, various factors like loosely organized codebases\n",
+ "and sophisticated model configurations complicate the easy reuse of im-\n",
+ "portantinnovationsbyawideaudience.Thoughtherehavebeenon-going\n",
+ "efforts to improve reusability and simplify deep learning (DL) model\n",
+ "developmentindisciplineslikenaturallanguageprocessingandcomputer\n",
+ "vision, none of them are optimized for challenges in the domain of DIA.\n",
+ "This represents a major gap in the existing toolkit, as DIA is central to\n",
+ "academicresearchacross awiderangeof disciplinesinthesocialsciences\n",
+ "and humanities. This paper introduces LayoutParser, an open-source\n",
+ "library for streamlining the usage of DL in DIA research and applica-\n",
+ "tions. The core LayoutParser library comes with a set of simple and\n",
+ "intuitiveinterfacesforapplyingandcustomizingDLmodelsforlayoutde-\n",
+ "tection,characterrecognition,andmanyotherdocumentprocessingtasks.\n",
+ "To promote extensibility, LayoutParser also incorporates a community\n",
+ "platform for sharing both pre-trained models and full document digiti-\n",
+ "zation pipelines. We demonstrate that LayoutParser is helpful for both\n",
+ "lightweight and large-scale digitization pipelines in real-word use cases.\n",
+ "The library is publicly available at https://layout-parser.github.io.\n",
+ "Keywords: DocumentImageAnalysis·DeepLearning·LayoutAnalysis\n",
+ "· Character Recognition · Open Source library · Toolkit.\n",
+ "1 Introduction\n",
+ "Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of\n",
+ "documentimageanalysis(DIA)tasksincludingdocumentimageclassification[11,\n",
+ "1202 nuJ 12 ]VC.sc[ 2v84351.3012:viXra\n",
+ "\n",
+ "{'author': '',\n",
+ " 'creationdate': '2021-06-22T01:27:10+00:00',\n",
+ " 'creator': 'LaTeX with hyperref',\n",
+ " 'keywords': '',\n",
+ " 'moddate': '2021-06-22T01:27:10+00:00',\n",
+ " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n",
+ " '2020) kpathsea version 6.3.2',\n",
+ " 'producer': 'pdfTeX-1.40.21',\n",
+ " 'subject': '',\n",
+ " 'title': '',\n",
+ " 'trapped': 'False',\n",
+ " 'source': 'example_data/layout-parser-paper.pdf',\n",
+ " 'file_path': 'example_data/layout-parser-paper.pdf',\n",
+ " 'total_pages': 16,\n",
+ " 'page': 0}\n"
+ ]
+ }
+ ],
+ "execution_count": 20
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": "It is possible to work with files from cloud storage."
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "from langchain_community.document_loaders import CloudBlobLoader\n",
+ "from langchain_community.document_loaders.generic import GenericLoader\n",
+ "\n",
+ "loader = GenericLoader(\n",
+ " blob_loader=CloudBlobLoader(\n",
+ " url=\"s3://mybucket\", # Supports s3://, az://, gs://, file:// schemes.\n",
+ " glob=\"*.pdf\",\n",
+ " ),\n",
+ " blob_parser=PDFPlumberParser(),\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "print(docs[0].page_content)\n",
+ "pprint.pp(docs[0].metadata)"
+ ],
+ "outputs": [],
+ "execution_count": null
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -175,9 +1153,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.12.7"
}
},
"nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
}
diff --git a/libs/community/langchain_community/document_loaders/parsers/pdf.py b/libs/community/langchain_community/document_loaders/parsers/pdf.py
index 782edddad44afd..bfbfba99fbb81a 100644
--- a/libs/community/langchain_community/document_loaders/parsers/pdf.py
+++ b/libs/community/langchain_community/document_loaders/parsers/pdf.py
@@ -129,6 +129,7 @@ def _validate_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
The standard keys are:
- source
+ - page (if mode='page')
- total_page
- creationdate
- creator
@@ -1191,8 +1192,6 @@ class PyPDFium2Parser(BaseBlobParser):
# password=None,
mode="page",
pages_delimiter="\n\f",
- # extract_images = True,
- # images_to_text = convert_images_to_text_with_tesseract(),
)
Lazily parse the blob:
@@ -1370,98 +1369,558 @@ def _extract_images_from_page(self, page: pypdfium2._helpers.page.PdfPage) -> st
return _FORMAT_IMAGE_STR.format(image_text=_JOIN_IMAGES.join(str_images))
+# The legacy PDFPlumberParser use key with upper case.
+# This is not aligned with the new convention, which requires the key to be in
+# lower case.
+class _PDFPlumberParserMetadata(dict[object, Any]):
+ _warning_keys: set[str] = set()
+
+ def __init__(self, d: dict[str, Any]):
+ super().__init__({k.lower(): v for k, v in d.items()})
+ self._pdf_metadata_keys = set(d.keys())
+
+ def _lower(self, k: object) -> object:
+ if k in self._pdf_metadata_keys:
+ lk = str(k).lower()
+ if lk != k:
+ if k not in _PDFPlumberParserMetadata._warning_keys:
+ _PDFPlumberParserMetadata._warning_keys.add(str(k))
+ logger.warning(
+ 'The key "%s" with uppercase is deprecated. '
+ "Update your code and vectorstore.",
+ k,
+ )
+ return lk
+ else:
+ return k
+
+ def __contains__(self, k: object) -> bool:
+ return super().__contains__(self._lower(k))
+
+ def __delitem__(self, k: object) -> None:
+ super().__delitem__(self._lower(k))
+
+ def __getitem__(self, k: object) -> Any:
+ return super().__getitem__(self._lower(k))
+
+ def get(self, k: object, default: Any = None) -> Any:
+ return super().get(self._lower(k), default)
+
+ def __setitem__(self, k: object, v: Any) -> None:
+ super().__setitem__(self._lower(k), v)
+
+
class PDFPlumberParser(BaseBlobParser):
- """Parse `PDF` with `PDFPlumber`."""
+ """Parse a blob from a PDF using `pdfplumber` library.
+
+ This class provides methods to parse a blob from a PDF document, supporting various
+ configurations such as handling password-protected PDFs, extracting images, and
+ defining extraction mode.
+ It integrates the 'pdfplumber' library for PDF processing and offers synchronous
+ blob parsing.
+
+ Examples:
+ Setup:
+
+ .. code-block:: bash
+
+ pip install -U langchain-community pdfplumber
+
+ Load a blob from a PDF file:
+
+ .. code-block:: python
+
+ from langchain_core.documents.base import Blob
+
+ blob = Blob.from_path("./example_data/layout-parser-paper.pdf")
+
+ Instantiate the parser:
+
+ .. code-block:: python
+
+ from langchain_community.document_loaders.parsers import PDFPlumberParser
+
+ parser = PDFPlumberParser(
+ # password = None,
+ mode = "single",
+ pages_delimiter = "\n\f",
+ # extract_tables="markdown",
+ )
+
+ Lazily parse the blob:
+
+ .. code-block:: python
+
+ docs = []
+ docs_lazy = parser.lazy_parse(blob)
+
+ for doc in docs_lazy:
+ docs.append(doc)
+ print(docs[0].page_content[:100])
+ print(docs[0].metadata)
+ """
def __init__(
self,
text_kwargs: Optional[Mapping[str, Any]] = None,
dedupe: bool = False,
extract_images: bool = False,
+ *,
+ password: Optional[str] = None,
+ mode: Literal["single", "page"] = "page",
+ pages_delimiter: str = _DEFAULT_PAGES_DELIMITER,
+ images_parser: Optional[BaseImageBlobParser] = None,
+ images_inner_format: Literal["text", "markdown-img", "html-img"] = "text",
+ extract_tables: Optional[Literal["csv", "markdown", "html"]] = None,
+ extract_tables_settings: Optional[dict[str, Any]] = None,
) -> None:
"""Initialize the parser.
Args:
+ password: Optional password for opening encrypted PDFs.
+ mode: The extraction mode, either "single" for the entire document or "page"
+ for page-wise extraction.
+ pages_delimiter: A string delimiter to separate pages in single-mode
+ extraction.
+ extract_images: Whether to extract images from the PDF.
+ images_parser: Optional image blob parser.
+ images_inner_format: The format for the parsed output.
+ - "text" = return the content as is
+ - "markdown-img" = wrap the content into an image markdown link, w/ link
+ pointing to (`![body)(#)`]
+ - "html-img" = wrap the content as the `alt` text of an tag and link to
+ (`
`)
+ extract_tables: Whether to extract images from the PDF in a specific
+ format, such as "csv", "markdown" or "html".
text_kwargs: Keyword arguments to pass to ``pdfplumber.Page.extract_text()``
- dedupe: Avoiding the error of duplicate characters if `dedupe=True`.
+ dedupe: Avoiding the error of duplicate characters if `dedupe=True`
+ extract_tables_settings: Optional dictionary of settings for customizing
+ table extraction.
+
+ Returns:
+ This method does not directly return data. Use the `parse` or `lazy_parse`
+ methods to retrieve parsed documents with content and metadata.
+
+ Raises:
+ ValueError: If the `mode` is not "single" or "page".
+ ValueError: If the `extract_tables` is not "csv", "markdown" or "html".
+
+ """
+ super().__init__()
+ if mode not in ["single", "page"]:
+ raise ValueError("mode must be single or page")
+ if extract_tables and extract_tables not in ["csv", "markdown", "html"]:
+ raise ValueError("mode must be csv, markdown or html")
+ if not extract_images and not images_parser:
+ images_parser = RapidOCRBlobParser()
+ self.password = password
+ self.extract_images = extract_images
+ self.images_parser = images_parser
+ self.images_inner_format = images_inner_format
+ self.mode = mode
+ self.pages_delimiter = pages_delimiter
+ self.dedupe = dedupe
+ self.text_kwargs = text_kwargs or {}
+ self.extract_tables = extract_tables
+ self.extract_tables_settings = extract_tables_settings or {
+ "vertical_strategy": "lines",
+ "horizontal_strategy": "lines",
+ "snap_y_tolerance": 5,
+ "intersection_x_tolerance": 15,
+ }
+
+ def lazy_parse(self, blob: Blob) -> Iterator[Document]: # type: ignore[valid-type]
+ """
+ Lazily parse the blob.
+ Insert image, if possible, between two paragraphs.
+ In this way, a paragraph can be continued on the next page.
+
+ Args:
+ blob: The blob to parse.
+
+ Raises:
+ ImportError: If the `pypdf` package is not found.
+
+ Yield:
+ An iterator over the parsed documents.
"""
try:
- import PIL # noqa:F401
+ import pdfplumber
except ImportError:
raise ImportError(
- "pillow package not found, please install it with `pip install pillow`"
+ "pdfplumber package not found, please install it "
+ "with `pip install pdfplumber`"
)
- self.text_kwargs = text_kwargs or {}
- self.dedupe = dedupe
- self.extract_images = extract_images
-
- def lazy_parse(self, blob: Blob) -> Iterator[Document]: # type: ignore[valid-type]
- """Lazily parse the blob."""
- import pdfplumber
with blob.as_bytes_io() as file_path: # type: ignore[attr-defined]
- doc = pdfplumber.open(file_path) # open document
-
- yield from [
- Document(
- page_content=self._process_page_content(page)
- + "\n"
- + self._extract_images_from_page(page),
- metadata=dict(
- {
- "source": blob.source, # type: ignore[attr-defined]
- "file_path": blob.source, # type: ignore[attr-defined]
- "page": page.page_number - 1,
- "total_pages": len(doc.pages),
- },
- **{
- k: doc.metadata[k]
- for k in doc.metadata
- if type(doc.metadata[k]) in [str, int]
- },
+ doc = pdfplumber.open(file_path, password=self.password) # open document
+ from pdfplumber.utils import geometry # import WordExctractor, TextMap
+
+ contents = []
+ doc_metadata = _purge_metadata(
+ (
+ doc.metadata
+ | {
+ "source": blob.source,
+ "file_path": blob.source,
+ "total_pages": len(doc.pages),
+ }
+ )
+ )
+ for page in doc.pages:
+ tables_bbox: list[tuple[float, float, float, float]] = (
+ self._extract_tables_bbox_from_page(page)
+ )
+ tables_content = self._extract_tables_from_page(page)
+ images_bbox = [geometry.obj_to_bbox(image) for image in page.images]
+ image_from_page = self._extract_images_from_page(page)
+ page_text = []
+ extras = []
+ for content in self._split_page_content(
+ page,
+ tables_bbox,
+ tables_content,
+ images_bbox,
+ image_from_page,
+ ):
+ if isinstance(content, str): # Text
+ page_text.append(content)
+ elif isinstance(content, list): # Table
+ page_text.append(_JOIN_TABLES + self._convert_table(content))
+ else: # Image
+ image_bytes = io.BytesIO()
+ numpy.save(image_bytes, content)
+ blob = Blob.from_data(
+ image_bytes.getvalue(), mime_type="application/x-npy"
+ )
+ text_from_image = next(
+ self.images_parser.lazy_parse(blob)
+ ).page_content
+ extras.append(
+ _format_inner_image(
+ blob, text_from_image, self.images_inner_format
+ )
+ )
+
+ all_text = _merge_text_and_extras(extras, "".join(page_text).strip())
+
+ if self.mode == "page":
+ # For legacy compatibility, add the last '\n'_
+ if not all_text.endswith("\n"):
+ all_text += "\n"
+ yield Document(
+ page_content=all_text,
+ metadata=_validate_metadata(
+ _PDFPlumberParserMetadata(
+ doc_metadata
+ | {
+ "page": page.page_number - 1,
+ }
+ )
+ ),
+ )
+ else:
+ contents.append(all_text)
+ # "tables_as_html": [self._convert_table_to_html(table)
+ # for
+ # table in tables_content],
+ # "images": images_content,
+ # tables_as_html.extend([self._convert_table(table)
+ # for
+ # table in tables_content])
+ if self.mode == "single":
+ yield Document(
+ page_content=self.pages_delimiter.join(contents),
+ metadata=_validate_metadata(
+ _PDFPlumberParserMetadata(doc_metadata)
),
)
- for page in doc.pages
- ]
def _process_page_content(self, page: pdfplumber.page.Page) -> str:
- """Process the page content based on dedupe."""
+ """Process the page content based on dedupe.
+
+ Args:
+ page: The PDF page to process.
+
+ Returns:
+ The extracted text from the page.
+ """
if self.dedupe:
return page.dedupe_chars().extract_text(**self.text_kwargs)
return page.extract_text(**self.text_kwargs)
- def _extract_images_from_page(self, page: pdfplumber.page.Page) -> str:
- """Extract images from page and get the text with RapidOCR."""
+ def _split_page_content(
+ self,
+ page: pdfplumber.page.Page,
+ tables_bbox: list[tuple[float, float, float, float]],
+ tables_content: list[list[list[Any]]],
+ images_bbox: list[tuple[float, float, float, float]],
+ images_content: list[np.ndarray],
+ **kwargs: Any,
+ ) -> Iterator[Union[str, list[list[str]], np.ndarray]]:
+ """Split the page content into text, tables, and images.
+
+ Args:
+ page: The PDF page to process.
+ tables_bbox: Bounding boxes of tables on the page.
+ tables_content: Content of tables on the page.
+ images_bbox: Bounding boxes of images on the page.
+ images_content: Content of images on the page.
+ **kwargs: Additional keyword arguments.
+
+ Yields:
+ An iterator over the split content (text, tables, images).
+ """
+ from pdfplumber.utils import (
+ geometry,
+ text,
+ )
+
+ # Iterate over words. If a word is in a table,
+ # yield the accumulated text, and the table
+ # A the word is in a previously see table, ignore it
+ # Finish with the accumulated text
+ kwargs.update(
+ {
+ "keep_blank_chars": True,
+ # "use_text_flow": True,
+ "presorted": True,
+ "layout_bbox": kwargs.get("layout_bbox")
+ # or geometry.objects_to_bbox(page.chars),
+ or page.cropbox,
+ }
+ )
+ chars = page.dedupe_chars().objects["char"] if self.dedupe else page.chars
+
+ extractor = text.WordExtractor(
+ **{k: kwargs[k] for k in text.WORD_EXTRACTOR_KWARGS if k in kwargs}
+ )
+ wordmap = extractor.extract_wordmap(chars)
+ extract_wordmaps: list[Any] = []
+ used_arrays = [False] * len(tables_bbox)
+ for word, o in wordmap.tuples:
+ # print(f" Try with '{word['text']}' ...")
+ is_table = False
+ word_bbox = geometry.obj_to_bbox(word)
+ for i, table_bbox in enumerate(tables_bbox):
+ if geometry.get_bbox_overlap(word_bbox, table_bbox):
+ # Find a world in a table
+ # print(" Find in an array")
+ is_table = True
+ if not used_arrays[i]:
+ # First time I see a word in this array
+ # Yield the previous part
+ if extract_wordmaps:
+ new_wordmap = text.WordMap(tuples=extract_wordmaps)
+ new_textmap = new_wordmap.to_textmap(
+ **{
+ k: kwargs[k]
+ for k in text.TEXTMAP_KWARGS
+ if k in kwargs
+ }
+ )
+ # print(f"yield {new_textmap.to_string()}")
+ yield new_textmap.to_string()
+ extract_wordmaps.clear()
+ # and yield the table
+ used_arrays[i] = True
+ # print(f"yield table {i}")
+ yield tables_content[i]
+ break
+ if not is_table:
+ # print(f' Add {word["text"]}')
+ extract_wordmaps.append((word, o))
+ if extract_wordmaps:
+ # Text after the array ?
+ new_wordmap = text.WordMap(tuples=extract_wordmaps)
+ new_textmap = new_wordmap.to_textmap(
+ **{k: kwargs[k] for k in text.TEXTMAP_KWARGS if k in kwargs}
+ )
+ # print(f"yield {new_textmap.to_string()}")
+ yield new_textmap.to_string()
+ # Add images-
+ for content in images_content:
+ yield content
+
+ def _extract_images_from_page(self, page: pdfplumber.page.Page) -> list[np.ndarray]:
+ """Extract images from a PDF page.
+
+ Args:
+ page: The PDF page to extract images from.
+
+ Returns:
+ A list of extracted images as numpy arrays.
+ """
from PIL import Image
- if not self.extract_images:
- return ""
+ if not self.images_parser:
+ return []
images = []
for img in page.images:
- if img["stream"]["Filter"].name in _PDF_FILTER_WITHOUT_LOSS:
- if img["stream"]["BitsPerComponent"] == 1:
- images.append(
- np.array(
- Image.frombytes(
- "1",
- (img["stream"]["Width"], img["stream"]["Height"]),
- img["stream"].get_data(),
- ).convert("L")
- )
- )
- else:
+ if "Filter" in img["stream"]:
+ if img["stream"]["Filter"].name in _PDF_FILTER_WITHOUT_LOSS:
images.append(
np.frombuffer(img["stream"].get_data(), dtype=np.uint8).reshape(
img["stream"]["Height"], img["stream"]["Width"], -1
)
)
- elif img["stream"]["Filter"].name in _PDF_FILTER_WITH_LOSS:
- images.append(img["stream"].get_data())
- else:
- warnings.warn("Unknown PDF Filter!")
+ elif img["stream"]["Filter"].name in _PDF_FILTER_WITH_LOSS:
+ buf = np.frombuffer(img["stream"].get_data(), dtype=np.uint8)
+ images.append(np.array(Image.open(io.BytesIO(buf.tobytes()))))
+ else:
+ logger.warning("Unknown PDF Filter!")
+
+ return images
+
+ def _extract_tables_bbox_from_page(
+ self,
+ page: pdfplumber.page.Page,
+ ) -> list[tuple]:
+ """Extract bounding boxes of tables from a PDF page.
+
+ Args:
+ page: The PDF page to extract table bounding boxes from.
+
+ Returns:
+ A list of bounding boxes for tables on the page.
+ """
+ if not self.extract_tables:
+ return []
+ from pdfplumber.table import TableSettings
+
+ table_settings = self.extract_tables_settings
+ tset = TableSettings.resolve(table_settings)
+ return [table.bbox for table in page.find_tables(tset)]
+
+ def _extract_tables_from_page(
+ self,
+ page: pdfplumber.page.Page,
+ ) -> list[list[list[Any]]]:
+ """Extract tables from a PDF page.
+
+ Args:
+ page: The PDF page to extract tables from.
- return extract_from_images_with_rapidocr(images)
+ Returns:
+ A list of tables, where each table is a list of rows, and each row is a
+ list of cell values.
+ """
+ if not self.extract_tables:
+ return []
+ table_settings = self.extract_tables_settings
+ tables_list = page.extract_tables(table_settings)
+ return tables_list
+
+ def _convert_table(self, table: list[list[str]]) -> str:
+ """Convert a table to the specified format.
+
+ Args:
+ table: The table to convert.
+
+ Returns:
+ The table content as a string in the specified format.
+ """
+ format = self.extract_tables
+ if format is None:
+ return ""
+ if format == "markdown":
+ return self._convert_table_to_markdown(table)
+ elif format == "html":
+ return self._convert_table_to_html(table)
+ elif format == "csv":
+ return self._convert_table_to_csv(table)
+ else:
+ raise ValueError(f"Unknown table format: {format}")
+
+ def _convert_table_to_csv(self, table: list[list[str]]) -> str:
+ """Convert a table to CSV format.
+
+ Args:
+ table: The table to convert.
+
+ Returns:
+ The table content as a string in CSV format.
+ """
+ if not table:
+ return ""
+
+ output = ["\n\n"]
+
+ # skip first row in details if header is part of the table
+ # j = 0 if self.header.external else 1
+
+ # iterate over detail rows
+ for row in table:
+ line = ""
+ for i, cell in enumerate(row):
+ # output None cells with empty string
+ cell = "" if cell is None else cell.replace("\n", " ")
+ line += cell + ","
+ output.append(line)
+ return "\n".join(output) + "\n\n"
+
+ def _convert_table_to_html(self, table: list[list[str]]) -> str:
+ """
+ Convert table content as a string in HTML format.
+ If clean is true, markdown syntax is removed from cell content.
+
+ Args:
+ table: The table to convert.
+
+ Returns:
+ The table content as a string in HTML format.
+ """
+ if not len(table):
+ return ""
+ output = "\n"
+ clean = True
+
+ # iterate over detail rows
+ for row in table:
+ line = ""
+ for i, cell in enumerate(row):
+ # output None cells with empty string
+ cell = "" if cell is None else cell.replace("\n", " ")
+ if clean: # remove sensitive syntax
+ cell = html.escape(cell.replace("-", "-"))
+ line += "| " + cell + " | "
+ line += "
\n"
+ output += line
+ return output + "
\n"
+
+ def _convert_table_to_markdown(self, table: list[list[str]]) -> str:
+ """Convert table content as a string in Github-markdown format.
+
+ Args:
+ table: The table to convert.
+
+ Returns:
+ The table content as a string in Markdown format.
+ """
+ clean = False
+ if not table:
+ return ""
+ col_count = len(table[0])
+
+ output = "|" + "|".join("" for i in range(col_count)) + "|\n"
+ output += "|" + "|".join("---" for i in range(col_count)) + "|\n"
+
+ # skip first row in details if header is part of the table
+ # j = 0 if self.header.external else 1
+
+ # iterate over detail rows
+ for row in table:
+ line = "|"
+ for i, cell in enumerate(row):
+ # output None cells with empty string
+ cell = "" if cell is None else cell.replace("\n", " ")
+ if clean: # remove sensitive syntax
+ cell = html.escape(cell.replace("-", "-"))
+ line += cell + "|"
+ line += "\n"
+ output += line
+ return output + "\n"
class AmazonTextractPDFParser(BaseBlobParser):
diff --git a/libs/community/langchain_community/document_loaders/pdf.py b/libs/community/langchain_community/document_loaders/pdf.py
index b9e57b19ff1d6f..cddea3da8f4cd4 100644
--- a/libs/community/langchain_community/document_loaders/pdf.py
+++ b/libs/community/langchain_community/document_loaders/pdf.py
@@ -1013,7 +1013,59 @@ def load(self) -> list[Document]:
class PDFPlumberLoader(BasePDFLoader):
- """Load `PDF` files using `pdfplumber`."""
+ """Load and parse a PDF file using 'pdfplumber' library.
+
+ This class provides methods to load and parse PDF documents, supporting various
+ configurations such as handling password-protected files, extracting images, and
+ defining extraction mode. It integrates the `pdfplumber` library for PDF processing
+ and offers both synchronous and asynchronous document loading.
+
+ Examples:
+ Setup:
+
+ .. code-block:: bash
+
+ pip install -U langchain-community pdfplumber
+
+ Instantiate the loader:
+
+ .. code-block:: python
+
+ from langchain_community.document_loaders import PDFPlumberLoader
+
+ loader = PDFPlumberLoader(
+ file_path = "./example_data/layout-parser-paper.pdf",
+ # headers = None
+ # password = None,
+ mode = "single",
+ pages_delimiter = "\n\f",
+ images_inner_format = "text",
+ # extract_tables = None,
+ # extract_tables_settings = None,
+ # text_kwargs = {"use_text_flow": False, "keep_blank_chars": False},
+ # dedupe = False,
+ )
+
+ Lazy load documents:
+
+ .. code-block:: python
+
+ docs = []
+ docs_lazy = loader.lazy_load()
+
+ for doc in docs_lazy:
+ docs.append(doc)
+ print(docs[0].page_content[:100])
+ print(docs[0].metadata)
+
+ Load documents asynchronously:
+
+ .. code-block:: python
+
+ docs = await loader.aload()
+ print(docs[0].page_content[:100])
+ print(docs[0].metadata)
+ """
def __init__(
self,
@@ -1022,34 +1074,78 @@ def __init__(
dedupe: bool = False,
headers: Optional[dict] = None,
extract_images: bool = False,
+ *,
+ password: Optional[str] = None,
+ mode: Literal["single", "page"] = "page",
+ images_parser: Optional[BaseImageBlobParser] = None,
+ images_inner_format: Literal["text", "markdown-img", "html-img"] = "text",
+ pages_delimiter: str = _DEFAULT_PAGES_DELIMITER,
+ extract_tables: Optional[Literal["csv", "markdown", "html"]] = None,
+ extract_tables_settings: Optional[dict[str, Any]] = None,
) -> None:
- """Initialize with a file path."""
- try:
- import pdfplumber # noqa:F401
- except ImportError:
- raise ImportError(
- "pdfplumber package not found, please install it with "
- "`pip install pdfplumber`"
- )
+ """Initialize with a file path.
- super().__init__(file_path, headers=headers)
- self.text_kwargs = text_kwargs or {}
- self.dedupe = dedupe
- self.extract_images = extract_images
+ Args:
+ file_path: The path to the PDF file to be loaded.
+ headers: Optional headers to use for GET request to download a file from a
+ web path.
+ password: Optional password for opening encrypted PDFs.
+ mode: The extraction mode, either "single" for the entire document or "page"
+ for page-wise extraction.
+ pages_delimiter: A string delimiter to separate pages in single-mode
+ extraction.
+ extract_images: Whether to extract images from the PDF.
+ images_parser: Optional image blob parser.
+ images_inner_format: The format for the parsed output.
+ - "text" = return the content as is
+ - "markdown-img" = wrap the content into an image markdown link, w/ link
+ pointing to (`![body)(#)`]
+ - "html-img" = wrap the content as the `alt` text of an tag and link to
+ (``)
+ extract_tables: Whether to extract tables in a specific format, such as
+ "csv", "markdown", or "html".
+ extract_tables_settings: Optional dictionary of settings for customizing
+ table extraction.
+ text_kwargs: Keyword arguments to pass to ``pdfplumber.Page.extract_text()``
+ dedupe: Avoiding the error of duplicate characters if `dedupe=True`
- def load(self) -> list[Document]:
- """Load file."""
+ Returns:
+ This method does not directly return data. Use the `load`, `lazy_load`,
+ or `aload` methods
+ to retrieve parsed documents with content and metadata.
- parser = PDFPlumberParser(
- text_kwargs=self.text_kwargs,
- dedupe=self.dedupe,
- extract_images=self.extract_images,
+ Raises:
+ ImportError: If the `pdfplumber` package is not installed.
+ """
+ super().__init__(file_path, headers=headers)
+ self.parser = PDFPlumberParser(
+ password=password,
+ mode=mode,
+ pages_delimiter=pages_delimiter,
+ extract_images=extract_images,
+ images_parser=images_parser,
+ images_inner_format=images_inner_format,
+ extract_tables=extract_tables,
+ text_kwargs=text_kwargs,
+ extract_tables_settings=extract_tables_settings,
+ dedupe=dedupe,
)
+
+ def lazy_load(
+ self,
+ ) -> Iterator[Document]:
+ """
+ Lazy load given path as pages.
+ Insert image, if possible, between two paragraphs.
+ In this way, a paragraph can be continued on the next page.
+ """
if self.web_path:
- blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path) # type: ignore[attr-defined]
+ blob = Blob.from_data( # type: ignore[attr-defined]
+ open(self.file_path, "rb").read(), path=self.web_path
+ )
else:
blob = Blob.from_path(self.file_path) # type: ignore[attr-defined]
- return parser.parse(blob)
+ yield from self.parser.lazy_parse(blob)
class AmazonTextractPDFLoader(BasePDFLoader):
diff --git a/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py b/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py
index 1137dd79f2e783..37c2ac79911e2b 100644
--- a/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py
+++ b/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py
@@ -9,10 +9,7 @@
import langchain_community.document_loaders.parsers as pdf_parsers
from langchain_community.document_loaders.base import BaseBlobParser
from langchain_community.document_loaders.blob_loaders import Blob
-from langchain_community.document_loaders.parsers import (
- BaseImageBlobParser,
- PDFPlumberParser,
-)
+from langchain_community.document_loaders.parsers import BaseImageBlobParser
if TYPE_CHECKING:
from PIL.Image import Image
@@ -95,13 +92,6 @@ def _assert_with_duplicate_parser(parser: BaseBlobParser, dedupe: bool = False)
assert "11000000 SSeerriieess" == docs[0].page_content.split("\n")[0]
-def test_pdfplumber_parser() -> None:
- """Test PDFPlumber parser."""
- _assert_with_parser(PDFPlumberParser())
- _assert_with_duplicate_parser(PDFPlumberParser())
- _assert_with_duplicate_parser(PDFPlumberParser(dedupe=True), dedupe=True)
-
-
class EmptyImageBlobParser(BaseImageBlobParser):
def _analyze_image(self, img: "Image") -> str:
return "Hello world"
@@ -115,6 +105,7 @@ def _analyze_image(self, img: "Image") -> str:
"parser_factory,params",
[
("PDFMinerParser", {}),
+ ("PDFPlumberParser", {}),
("PyMuPDFParser", {}),
("PyPDFium2Parser", {}),
("PyPDFParser", {"extraction_mode": "plain"}),
@@ -245,6 +236,7 @@ def _std_assert_with_parser(parser: BaseBlobParser) -> None:
@pytest.mark.parametrize(
"parser_factory,params",
[
+ ("PDFPlumberParser", {}),
("PyMuPDFParser", {}),
],
)
diff --git a/libs/community/tests/integration_tests/document_loaders/test_pdf.py b/libs/community/tests/integration_tests/document_loaders/test_pdf.py
index 891308a0441e90..dc30b15e77fb73 100644
--- a/libs/community/tests/integration_tests/document_loaders/test_pdf.py
+++ b/libs/community/tests/integration_tests/document_loaders/test_pdf.py
@@ -167,6 +167,7 @@ def test_amazontextract_loader_failures() -> None:
"parser_factory,params",
[
("PDFMinerLoader", {}),
+ ("PDFPlumberLoader", {}),
("PyMuPDFLoader", {}),
("PyPDFium2Loader", {}),
("PyPDFLoader", {}),
diff --git a/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py b/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py
index e89a8d854dfad9..6dfaf9ccf40561 100644
--- a/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py
+++ b/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py
@@ -9,7 +9,9 @@
import langchain_community.document_loaders.parsers as pdf_parsers
from langchain_community.document_loaders.base import BaseBlobParser
from langchain_community.document_loaders.blob_loaders import Blob
-from langchain_community.document_loaders.parsers.pdf import _merge_text_and_extras
+from langchain_community.document_loaders.parsers.pdf import (
+ _merge_text_and_extras,
+)
_THIS_DIR = Path(__file__).parents[3]
@@ -75,6 +77,7 @@ def _assert_with_parser(parser: BaseBlobParser, *, splits_by_page: bool = True)
"parser_factory,require,params",
[
("PDFMinerParser", "pdfminer", {"splits_by_page": False}),
+ ("PDFPlumberParser", "pdfplumber", {}),
("PyMuPDFParser", "pymupdf", {}),
("PyPDFParser", "pypdf", {}),
("PyPDFium2Parser", "pypdfium2", {}),