feat: Setup function (#42)

* Added setup configuration * Added the setup function
lamalab-org · Jun 27, 2024 · d335206 · d335206
1 parent cac595d
commit d335206
Show file tree

Hide file tree

Showing 12 changed files with 46 additions and 142 deletions.
diff --git a/content/agents/agent.ipynb b/content/agents/agent.ipynb
@@ -14,21 +14,6 @@
     "This notebook aims to demonstrate how to construct LLM agents and explain their functioning. In this practical application, we will focus on extracting chemical reactions from an image.\n"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "tags": [
-     "remove-cell"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "import warnings\n",
-    "\n",
-    "warnings.filterwarnings(\"ignore\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -54,7 +39,9 @@
     "from langchain.agents.react.agent import create_react_agent\n",
     "from langchain_openai import ChatOpenAI\n",
     "\n",
-    "from litellm import completion"
+    "from litellm import completion\n",
+    "\n",
+    "import llmstructdata"
    ]
   },
   {
@@ -66,30 +53,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "tags": [
-     "remove-output"
-    ]
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "load_dotenv(\".env\", override=True)"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/content/beyond_text/beyond_images.ipynb b/content/beyond_text/beyond_images.ipynb
@@ -51,6 +51,8 @@
    },
    "outputs": [],
    "source": [
+    "import llmstructdata\n",
+    "\n",
     "from pdf2image import convert_from_path\n",
     "\n",
     "file_path = \"../obtaining_data/PDFs/10.26434_chemrxiv-2024-1l0sn.pdf\"\n",
@@ -355,7 +357,7 @@
     }
    ],
    "source": [
-    "from dotenv import load_dotenv\n",
+    "import os\n",
     "from litellm import completion\n",
     "\n",
     "\n",
@@ -398,8 +400,6 @@
     "\n",
     "\n",
     "# Load the OpenAI API key from environment variables\n",
-    "dotenv_path = \"../.env\"\n",
-    "load_dotenv(dotenv_path)\n",
     "api_key = os.getenv(\"OPENAI_API_KEY\")\n",
     "\n",
     "# Set the API key for LiteLLM\n",

diff --git a/content/constrained_decoding/index.ipynb b/content/constrained_decoding/index.ipynb
@@ -40,9 +40,8 @@
     "import instructor\n",
     "from IPython.display import SVG\n",
     "from openai import OpenAI\n",
-    "from dotenv import load_dotenv\n",
     "\n",
-    "_ = load_dotenv(\"../.env\", override=True)"
+    "import llmstructdata"
    ]
   },
   {

diff --git a/content/context_window/Dealing_with_context_window.ipynb b/content/context_window/Dealing_with_context_window.ipynb
@@ -260,6 +260,8 @@
    },
    "outputs": [],
    "source": [
+    "import llmstructdata\n",
+    "\n",
     "from sentence_transformers import SentenceTransformer\n",
     "\n",
     "model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",

diff --git a/content/document_parsing_and_cleaning/cleaning.ipynb b/content/document_parsing_and_cleaning/cleaning.ipynb
@@ -83,6 +83,8 @@
     }
    ],
    "source": [
+    "import llmstructdata\n",
+    "\n",
     "import re\n",
     "\n",
     "\n",

diff --git a/content/document_parsing_and_cleaning/parsing.ipynb b/content/document_parsing_and_cleaning/parsing.ipynb
@@ -26,7 +26,7 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "\n",
+    "import llmstructdata\n",
     "\n",
     "def convert_pdf_with_nougat(\n",
     "    pdf_path, output_dir, model=\"0.1.0-small\", batch_size=1, no_skipping=False\n",

diff --git a/content/finetune/choosing_paradigm.ipynb b/content/finetune/choosing_paradigm.ipynb
@@ -50,22 +50,6 @@
     "We will start by importing all the packages needed."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "544ff2bd",
-   "metadata": {
-    "tags": [
-     "remove-cell"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "import warnings\n",
-    "\n",
-    "warnings.filterwarnings(\"ignore\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -101,8 +85,9 @@
     "from litellm import completion\n",
     "from litellm.caching import Cache\n",
     "from statistics import mean\n",
-    "import numpy as np\n",
-    "import matplotlib.pyplot as plt"
+    "import numpy as np \n",
+    "import matplotlib.pyplot as plt \n",
+    "import llmstructdata"
    ]
   },
   {
@@ -117,32 +102,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "94595c41",
-   "metadata": {
-    "tags": [
-     "remove-output"
-    ]
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "litellm.cache = Cache()\n",
-    "load_dotenv(\".env\", override=True)"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "a7e82b75",
@@ -1926,20 +1885,6 @@
     "To study the results more graphically, we can plot all the results in several bar plots."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "53cf6669",
-   "metadata": {
-    "tags": [
-     "remove-cell"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "plt.style.use(\"../package/llmstructdata/book.mplstyle\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 26,

diff --git a/content/obtaining_data/crossref_search.ipynb b/content/obtaining_data/crossref_search.ipynb
@@ -37,6 +37,7 @@
    "outputs": [],
    "source": [
     "from crossref.restful import Works\n",
+    "import llmstructdata\n",
     "import json\n",
     "\n",
     "works = Works(timeout=60)\n",

diff --git a/content/obtaining_data/data_mining.ipynb b/content/obtaining_data/data_mining.ipynb
@@ -37,6 +37,7 @@
    "outputs": [],
    "source": [
     "from paperscraper.get_dumps import chemrxiv\n",
+    "import llmstructdata\n",
     "\n",
     "# Download of the ChemRxiv paper dump\n",
     "chemrxiv(save_path=\"chemrxiv_2020-11-10.jsonl\")"

diff --git a/content/perovskite/constrained_formulas.ipynb b/content/perovskite/constrained_formulas.ipynb
@@ -1,20 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "tags": [
-     "remove-cell"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "import warnings\n",
-    "\n",
-    "warnings.filterwarnings(\"ignore\")"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -26,15 +11,15 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "```{admonition} Motivation \n",
-    ":class:note\n",
+    "````{admonition} Motivation \n",
+    ":class: note\n",
     "\n",
     "In this notebook we want to collect a dataset on structured data that will allow us to train a composition-property prediction model later on. But we want to focus on a specific class of materials called **chalcogenide perovskites**.\n",
     "\n",
     "- We want to ensure that the output formulas retrieved can be parsed into a `pymatgen` `Composition` object. This is important to be able to reuse the extracted data with popular materials informatics packages, such as those leading in [MatBench](https://matbench.materialsproject.org/).\n",
     "- We want to ensure control over  the stoichiometry of the compounds retrieved. We also want to ensure that the formulas fulfill the criteria of charge neutrality. For this, we will employ a constrained decoding approach as exemplify in the [constrained decoding]() notebook.\n",
     "- We will validate if the formulas retrieved fulfill some stability criteria for being a perovskite compound. For this we will employ a similar approach as in the [validation example]() notebook.\n",
-    "```"
+    "````"
    ]
   },
   {
@@ -43,8 +28,8 @@
    "source": [
     "## What are Chalcogenide Perovskites?\n",
     "\n",
-    "```{admonition} Background\n",
-    ":class:note\n",
+    "````{admonition} Background\n",
+    ":class: note\n",
     "\n",
     "We aim to constrain our extraction pipeline to output formulas that adhere to specific chemical and structural criteria, focusing on **chalcogenide perovskite** compounds. \n",
     "\n",
@@ -55,7 +40,7 @@
     "- **Optoelectronic Properties**: They exhibit favorable band gaps and strong light absorption, making them ideal for photovoltaic applications.\n",
     "- **Stability**: Compared to their halide counterparts, chalcogenide perovskites often show enhanced chemical stability.\n",
     "- **Flexibility in Composition**: The ability to tune the composition allows for optimization of properties for specific applications like solar cells, photodetectors, and LEDs.\n",
-    "```"
+    "````"
    ]
   },
   {
@@ -84,7 +69,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "\n",
     "For this task, we want to retrieve the structure dataset from the review paper [Chalcogenide Perovskites: Tantalizing Prospects, Challenging Materials](https://onlinelibrary.wiley.com/doi/10.1002/adom.202101704).{cite}`Sopiha2021`"
    ]
   },
@@ -934,6 +918,8 @@
     }
    ],
    "source": [
+    "import llmstructdata\n",
+    "\n",
     "from pymatgen.core import Composition\n",
     "\n",
     "\n",
@@ -1074,10 +1060,7 @@
     "from pydantic import BaseModel, Field\n",
     "from typing import Optional, Literal, List\n",
     "from litellm import completion\n",
-    "import instructor\n",
-    "from dotenv import load_dotenv\n",
-    "\n",
-    "_ = load_dotenv(\"../../.env\", override=True)"
+    "import instructor"
    ]
   },
   {
@@ -3456,11 +3439,6 @@
     "\n",
     "```"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": []
   }
  ],
  "metadata": {

diff --git a/package/llmstructdata/__init__.py b/package/llmstructdata/__init__.py
@@ -1,11 +1,16 @@
+
 import matplotlib as mpl
 import matplotlib.pyplot as plt
+from dotenv import load_dotenv
 import warnings
-import pystow
+import litellm
+from litellm.caching import Cache
+
+litellm.cache = Cache()
 
 url = "https://raw.githubusercontent.com/lamalab-org/plotutils/main/kevin.mplstyle"
 path = pystow.ensure("matstructdata", "plotsettings", url=url)
 
-mpl.style.use(path)
+load_dotenv("../../.env", override=True)
 
 warnings.filterwarnings("ignore")
diff --git a/references.bib b/references.bib
@@ -66,11 +66,19 @@ @article{Ogasawara2015
 }
 
 @article{RxnScribe,
-    author = {Qian, Yujie and Guo, Jiang and Tu, Zhengkai and Coley, Connor W. and Barzilay, Regina},
-    title = {RxnScribe: A Sequence Generation Model for Reaction Diagram Parsing},
-    journal = {Journal of Chemical Information and Modeling},
-    doi = {10.1021/acs.jcim.3c00439}
-  }
+  title = {RxnScribe: A Sequence Generation Model for Reaction Diagram Parsing},
+  volume = {63},
+  ISSN = {1549-960X},
+  url = {http://dx.doi.org/10.1021/acs.jcim.3c00439},
+  DOI = {10.1021/acs.jcim.3c00439},
+  number = {13},
+  journal = {Journal of Chemical Information and Modeling},
+  publisher = {American Chemical Society (ACS)},
+  author = {Qian,  Yujie and Guo,  Jiang and Tu,  Zhengkai and Coley,  Connor W. and Barzilay,  Regina},
+  year = {2023},
+  month = jun,
+  pages = {4030–4041}
+}
 
 @misc{shi2024instruction,
       title={Instruction Tuning With Loss Over Instructions},
-Original file line number
+Diff line change
@@ Expand Up / @@ -83,6 +83,8 @@ @@
         }
        ],
        "source": [
+        "import llmstructdata\n",
+        "\n",
         "import re\n",
         "\n",
         "\n",
@@ Expand Down @@