milvus-langchain-wx-embeddings-crepioca.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pymilvus==2.4.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (2.4.0)\n",
      "Requirement already satisfied: setuptools>=67 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (75.1.0)\n",
      "Requirement already satisfied: grpcio<=1.60.0,>=1.49.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (1.60.0)\n",
      "Requirement already satisfied: protobuf>=3.20.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (5.28.1)\n",
      "Requirement already satisfied: environs<=9.5.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (9.5.0)\n",
      "Requirement already satisfied: ujson>=2.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (5.10.0)\n",
      "Requirement already satisfied: pandas>=1.2.4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (2.1.4)\n",
      "Requirement already satisfied: requests in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (2.32.2)\n",
      "Requirement already satisfied: minio>=7.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (7.2.8)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (17.0.0)\n",
      "Requirement already satisfied: azure-storage-blob in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (12.22.0)\n",
      "Requirement already satisfied: scipy in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pymilvus==2.4.0) (1.14.1)\n",
      "Requirement already satisfied: marshmallow>=3.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from environs<=9.5.0->pymilvus==2.4.0) (3.22.0)\n",
      "Requirement already satisfied: python-dotenv in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from environs<=9.5.0->pymilvus==2.4.0) (1.0.1)\n",
      "Requirement already satisfied: certifi in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from minio>=7.0.0->pymilvus==2.4.0) (2024.8.30)\n",
      "Requirement already satisfied: urllib3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from minio>=7.0.0->pymilvus==2.4.0) (2.2.3)\n",
      "Requirement already satisfied: argon2-cffi in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from minio>=7.0.0->pymilvus==2.4.0) (23.1.0)\n",
      "Requirement already satisfied: pycryptodome in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from minio>=7.0.0->pymilvus==2.4.0) (3.20.0)\n",
      "Requirement already satisfied: typing-extensions in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from minio>=7.0.0->pymilvus==2.4.0) (4.12.2)\n",
      "Requirement already satisfied: numpy<2,>=1.26.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas>=1.2.4->pymilvus==2.4.0) (1.26.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas>=1.2.4->pymilvus==2.4.0) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas>=1.2.4->pymilvus==2.4.0) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas>=1.2.4->pymilvus==2.4.0) (2024.1)\n",
      "Requirement already satisfied: azure-core>=1.28.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from azure-storage-blob->pymilvus==2.4.0) (1.31.0)\n",
      "Requirement already satisfied: cryptography>=2.1.4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from azure-storage-blob->pymilvus==2.4.0) (43.0.1)\n",
      "Requirement already satisfied: isodate>=0.6.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from azure-storage-blob->pymilvus==2.4.0) (0.6.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->pymilvus==2.4.0) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->pymilvus==2.4.0) (3.10)\n",
      "Requirement already satisfied: six>=1.11.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from azure-core>=1.28.0->azure-storage-blob->pymilvus==2.4.0) (1.16.0)\n",
      "Requirement already satisfied: cffi>=1.12 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from cryptography>=2.1.4->azure-storage-blob->pymilvus==2.4.0) (1.17.1)\n",
      "Requirement already satisfied: packaging>=17.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from marshmallow>=3.0.0->environs<=9.5.0->pymilvus==2.4.0) (24.1)\n",
      "Requirement already satisfied: argon2-cffi-bindings in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from argon2-cffi->minio>=7.0.0->pymilvus==2.4.0) (21.2.0)\n",
      "Requirement already satisfied: pycparser in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from cffi>=1.12->cryptography>=2.1.4->azure-storage-blob->pymilvus==2.4.0) (2.22)\n",
      "Requirement already satisfied: langchain in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (0.3.0)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (6.0.2)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (2.0.34)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (3.10.5)\n",
      "Requirement already satisfied: langchain-core<0.4.0,>=0.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (0.3.0)\n",
      "Requirement already satisfied: langchain-text-splitters<0.4.0,>=0.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (0.3.0)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (0.1.121)\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.26.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (1.26.4)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (2.9.1)\n",
      "Requirement already satisfied: requests<3,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (2.32.2)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain) (8.5.0)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.1.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.11.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain-core<0.4.0,>=0.3.0->langchain) (1.33)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain-core<0.4.0,>=0.3.0->langchain) (24.1)\n",
      "Requirement already satisfied: typing-extensions>=4.7 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain-core<0.4.0,>=0.3.0->langchain) (4.12.2)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (0.27.2)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.7)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.23.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.23.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain) (2024.8.30)\n",
      "Requirement already satisfied: anyio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (4.4.0)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (1.0.5)\n",
      "Requirement already satisfied: sniffio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (0.14.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.4.0,>=0.3.0->langchain) (3.0.0)\n",
      "Requirement already satisfied: ibm-generative-ai in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (3.0.0)\n",
      "Requirement already satisfied: aiolimiter<2.0.0,>=1.1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-generative-ai) (1.1.0)\n",
      "Requirement already satisfied: deprecated<2.0.0,>=1.2.14 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-generative-ai) (1.2.14)\n",
      "Requirement already satisfied: httpx<0.28.0,>=0.27.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-generative-ai) (0.27.2)\n",
      "Requirement already satisfied: httpx-sse<0.5.0,>=0.4.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-generative-ai) (0.4.0)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-generative-ai) (2.9.1)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from deprecated<2.0.0,>=1.2.14->ibm-generative-ai) (1.16.0)\n",
      "Requirement already satisfied: anyio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai) (4.4.0)\n",
      "Requirement already satisfied: certifi in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai) (2024.8.30)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai) (1.0.5)\n",
      "Requirement already satisfied: idna in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai) (3.10)\n",
      "Requirement already satisfied: sniffio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.27.0->ibm-generative-ai) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->ibm-generative-ai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.23.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->ibm-generative-ai) (2.23.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->ibm-generative-ai) (4.12.2)\n",
      "Requirement already satisfied: unstructured in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (0.15.12)\n",
      "Requirement already satisfied: chardet in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (5.2.0)\n",
      "Requirement already satisfied: filetype in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (1.2.0)\n",
      "Requirement already satisfied: python-magic in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (0.4.27)\n",
      "Requirement already satisfied: lxml in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (5.3.0)\n",
      "Requirement already satisfied: nltk in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (3.9.1)\n",
      "Requirement already satisfied: tabulate in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (0.9.0)\n",
      "Requirement already satisfied: requests in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (2.32.2)\n",
      "Requirement already satisfied: beautifulsoup4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (4.12.3)\n",
      "Requirement already satisfied: emoji in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (2.12.1)\n",
      "Requirement already satisfied: dataclasses-json in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (0.6.7)\n",
      "Requirement already satisfied: python-iso639 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (2024.4.27)\n",
      "Requirement already satisfied: langdetect in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (1.0.9)\n",
      "Requirement already satisfied: numpy<2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (1.26.4)\n",
      "Requirement already satisfied: rapidfuzz in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (3.9.7)\n",
      "Requirement already satisfied: backoff in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (2.2.1)\n",
      "Requirement already satisfied: typing-extensions in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (4.12.2)\n",
      "Requirement already satisfied: unstructured-client in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (0.25.8)\n",
      "Requirement already satisfied: wrapt in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (1.16.0)\n",
      "Requirement already satisfied: tqdm in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (4.66.5)\n",
      "Requirement already satisfied: psutil in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (6.0.0)\n",
      "Requirement already satisfied: python-oxmsg in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured) (0.0.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from beautifulsoup4->unstructured) (2.6)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from dataclasses-json->unstructured) (3.22.0)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from dataclasses-json->unstructured) (0.9.0)\n",
      "Requirement already satisfied: six in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langdetect->unstructured) (1.16.0)\n",
      "Requirement already satisfied: click in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from nltk->unstructured) (8.1.7)\n",
      "Requirement already satisfied: joblib in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from nltk->unstructured) (1.4.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from nltk->unstructured) (2024.9.11)\n",
      "Requirement already satisfied: olefile in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from python-oxmsg->unstructured) (0.47)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->unstructured) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->unstructured) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->unstructured) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->unstructured) (2024.8.30)\n",
      "Requirement already satisfied: deepdiff>=6.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (8.0.1)\n",
      "Requirement already satisfied: httpx>=0.27.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (0.27.2)\n",
      "Requirement already satisfied: jsonpath-python>=1.0.6 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (1.0.6)\n",
      "Requirement already satisfied: mypy-extensions>=1.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (1.0.0)\n",
      "Requirement already satisfied: nest-asyncio>=1.6.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (1.6.0)\n",
      "Requirement already satisfied: packaging>=23.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (24.1)\n",
      "Requirement already satisfied: pypdf>=4.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (4.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (2.9.0.post0)\n",
      "Requirement already satisfied: requests-toolbelt>=1.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from unstructured-client->unstructured) (1.0.0)\n",
      "Requirement already satisfied: orderly-set==5.2.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from deepdiff>=6.0->unstructured-client->unstructured) (5.2.2)\n",
      "Requirement already satisfied: anyio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx>=0.27.0->unstructured-client->unstructured) (4.4.0)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx>=0.27.0->unstructured-client->unstructured) (1.0.5)\n",
      "Requirement already satisfied: sniffio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx>=0.27.0->unstructured-client->unstructured) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.27.0->unstructured-client->unstructured) (0.14.0)\n",
      "Requirement already satisfied: huggingface-hub in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (0.24.7)\n",
      "Requirement already satisfied: filelock in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (3.16.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (2024.9.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (24.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (6.0.2)\n",
      "Requirement already satisfied: requests in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (2.32.2)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (4.66.5)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub) (4.12.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub) (2024.8.30)\n",
      "Requirement already satisfied: sentence-transformers in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (3.1.0)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.38.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (4.44.2)\n",
      "Requirement already satisfied: tqdm in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (4.66.5)\n",
      "Requirement already satisfied: torch>=1.11.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (2.4.1)\n",
      "Requirement already satisfied: numpy<2.0.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (1.26.4)\n",
      "Requirement already satisfied: scikit-learn in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (1.5.2)\n",
      "Requirement already satisfied: scipy in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (1.14.1)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (0.24.7)\n",
      "Requirement already satisfied: Pillow in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sentence-transformers) (10.4.0)\n",
      "Requirement already satisfied: filelock in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub>=0.19.3->sentence-transformers) (3.16.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub>=0.19.3->sentence-transformers) (2024.9.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub>=0.19.3->sentence-transformers) (24.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub>=0.19.3->sentence-transformers) (6.0.2)\n",
      "Requirement already satisfied: requests in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub>=0.19.3->sentence-transformers) (2.32.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from huggingface-hub>=0.19.3->sentence-transformers) (4.12.2)\n",
      "Requirement already satisfied: sympy in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (1.13.2)\n",
      "Requirement already satisfied: networkx in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (3.3)\n",
      "Requirement already satisfied: jinja2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (3.1.4)\n",
      "Requirement already satisfied: setuptools in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (75.1.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from transformers<5.0.0,>=4.38.0->sentence-transformers) (2024.9.11)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from transformers<5.0.0,>=4.38.0->sentence-transformers) (0.4.5)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from transformers<5.0.0,>=4.38.0->sentence-transformers) (0.19.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from scikit-learn->sentence-transformers) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from scikit-learn->sentence-transformers) (3.5.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from jinja2->torch>=1.11.0->sentence-transformers) (2.1.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub>=0.19.3->sentence-transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub>=0.19.3->sentence-transformers) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub>=0.19.3->sentence-transformers) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->huggingface-hub>=0.19.3->sentence-transformers) (2024.8.30)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from sympy->torch>=1.11.0->sentence-transformers) (1.3.0)\n",
      "Requirement already satisfied: ibm_watsonx_ai in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (1.1.9)\n",
      "Requirement already satisfied: requests in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (2.32.2)\n",
      "Requirement already satisfied: httpx in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (0.27.2)\n",
      "Requirement already satisfied: urllib3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (2.2.3)\n",
      "Requirement already satisfied: pandas<2.2.0,>=0.24.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (2.1.4)\n",
      "Requirement already satisfied: certifi in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (2024.8.30)\n",
      "Requirement already satisfied: lomond in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (0.3.3)\n",
      "Requirement already satisfied: tabulate in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (0.9.0)\n",
      "Requirement already satisfied: packaging in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (24.1)\n",
      "Requirement already satisfied: ibm-cos-sdk<2.14.0,>=2.12.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (2.13.6)\n",
      "Requirement already satisfied: importlib-metadata in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm_watsonx_ai) (8.5.0)\n",
      "Requirement already satisfied: ibm-cos-sdk-core==2.13.6 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-cos-sdk<2.14.0,>=2.12.0->ibm_watsonx_ai) (2.13.6)\n",
      "Requirement already satisfied: ibm-cos-sdk-s3transfer==2.13.6 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-cos-sdk<2.14.0,>=2.12.0->ibm_watsonx_ai) (2.13.6)\n",
      "Requirement already satisfied: jmespath<=1.0.1,>=0.10.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-cos-sdk<2.14.0,>=2.12.0->ibm_watsonx_ai) (1.0.1)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.9.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from ibm-cos-sdk-core==2.13.6->ibm-cos-sdk<2.14.0,>=2.12.0->ibm_watsonx_ai) (2.9.0.post0)\n",
      "Requirement already satisfied: numpy<2,>=1.26.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas<2.2.0,>=0.24.2->ibm_watsonx_ai) (1.26.4)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas<2.2.0,>=0.24.2->ibm_watsonx_ai) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pandas<2.2.0,>=0.24.2->ibm_watsonx_ai) (2024.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->ibm_watsonx_ai) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests->ibm_watsonx_ai) (3.10)\n",
      "Requirement already satisfied: anyio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx->ibm_watsonx_ai) (4.4.0)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx->ibm_watsonx_ai) (1.0.5)\n",
      "Requirement already satisfied: sniffio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx->ibm_watsonx_ai) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx->ibm_watsonx_ai) (0.14.0)\n",
      "Requirement already satisfied: zipp>=3.20 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from importlib-metadata->ibm_watsonx_ai) (3.20.2)\n",
      "Requirement already satisfied: six>=1.10.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from lomond->ibm_watsonx_ai) (1.16.0)\n",
      "Collecting langchain_community\n",
      "  Downloading langchain_community-0.3.0-py3-none-any.whl.metadata (2.8 kB)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (6.0.2)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (2.0.34)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (3.10.5)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (0.6.7)\n",
      "Requirement already satisfied: langchain<0.4.0,>=0.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (0.3.0)\n",
      "Requirement already satisfied: langchain-core<0.4.0,>=0.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (0.3.0)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.112 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (0.1.121)\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.26.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (1.26.4)\n",
      "Collecting pydantic-settings<3.0.0,>=2.4.0 (from langchain_community)\n",
      "  Downloading pydantic_settings-2.5.2-py3-none-any.whl.metadata (3.5 kB)\n",
      "Requirement already satisfied: requests<3,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (2.32.2)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain_community) (8.5.0)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (2.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (6.1.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.11.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (3.22.0)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (0.9.0)\n",
      "Requirement already satisfied: langchain-text-splitters<0.4.0,>=0.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain<0.4.0,>=0.3.0->langchain_community) (0.3.0)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain<0.4.0,>=0.3.0->langchain_community) (2.9.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain-core<0.4.0,>=0.3.0->langchain_community) (1.33)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain-core<0.4.0,>=0.3.0->langchain_community) (24.1)\n",
      "Requirement already satisfied: typing-extensions>=4.7 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langchain-core<0.4.0,>=0.3.0->langchain_community) (4.12.2)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.112->langchain_community) (0.27.2)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.112->langchain_community) (3.10.7)\n",
      "Requirement already satisfied: python-dotenv>=0.21.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic-settings<3.0.0,>=2.4.0->langchain_community) (1.0.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain_community) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain_community) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain_community) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from requests<3,>=2->langchain_community) (2024.8.30)\n",
      "Requirement already satisfied: anyio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.112->langchain_community) (4.4.0)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.112->langchain_community) (1.0.5)\n",
      "Requirement already satisfied: sniffio in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.112->langchain_community) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.112->langchain_community) (0.14.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.4.0,>=0.3.0->langchain_community) (3.0.0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain<0.4.0,>=0.3.0->langchain_community) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.23.3 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain<0.4.0,>=0.3.0->langchain_community) (2.23.3)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/flaviajaninebeo/Documents/GitHub/.venv/lib/python3.12/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain_community) (1.0.0)\n",
      "Downloading langchain_community-0.3.0-py3-none-any.whl (2.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m139.6 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:03\u001b[0m\n",
      "\u001b[?25hDownloading pydantic_settings-2.5.2-py3-none-any.whl (26 kB)\n",
      "Installing collected packages: pydantic-settings, langchain_community\n",
      "Successfully installed langchain_community-0.3.0 pydantic-settings-2.5.2\n"
     ]
    }
   ],
   "source": [
    "# ! conda create -y -n milvus python=3.10\n",
    "\n",
    "! pip install pymilvus==2.4.0\n",
    "! pip install langchain\n",
    "! pip install ibm-generative-ai\n",
    "! pip install unstructured\n",
    "! pip install huggingface-hub\n",
    "! pip install sentence-transformers\n",
    "! pip install ibm_watsonx_ai\n",
    "! pip install langchain_community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import requests\n",
    "import json\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from langchain_community.document_loaders import UnstructuredURLLoader\n",
    "\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "from langchain_community.vectorstores import Milvus\n",
    "\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.callbacks.base import BaseCallbackHandler\n",
    "\n",
    "# BAM\n",
    "from genai.extensions.langchain import LangChainInterface\n",
    "from genai.credentials import Credentials\n",
    "from genai import Client\n",
    "\n",
    "from genai.schema import TextGenerationParameters\n",
    "\n",
    "from ibm_watsonx_ai.foundation_models import Embeddings\n",
    "\n",
    "load_dotenv() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "token = \"seu IAM API key IBM Cloud\"\n",
    "project_id = \"aqui seu project_id do watsonx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gogo']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
    "text_splitter.split_text(\"gogo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<ibm_watsonx_ai.foundation_models.embeddings.embeddings.Embeddings at 0x10491f200>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_model = Embeddings(\n",
    "            model_id=\"ibm/slate-125m-english-rtrvr\",\n",
    "            credentials={\"api_key\": token, \"url\": \"https://us-south.ml.cloud.ibm.com\"},\n",
    "            project_id=project_id,\n",
    "            params={\n",
    "                \"truncate_input_tokens\": 512\n",
    "            }\n",
    "        )\n",
    "embedding_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# INITIALIZE A MILVUS COLLECTION WITH EMBEDDINGS FROM A SET OF DOCUMENTS\n",
    "\n",
    "urls = [\n",
    "    \"https://www.tuasaude.com/receitas-de-crepioca-para-emagrecer/\"\n",
    "]\n",
    "\n",
    "loader = UnstructuredURLLoader(urls=urls)\n",
    "data = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
    "docs = text_splitter.split_documents(data)\n",
    "\n",
    "vector_db = Milvus.from_documents(\n",
    "    docs,\n",
    "    embedding_model,\n",
    "    drop_old=True,  # Drop the old Milvus collection if it exists\n",
    "    connection_args={\"host\":\"7aff4bea-4196-42e0-a9b7-bed85680916f.cie9agrw03kb77s3pr1g.lakehouse.appdomain.cloud\", \n",
    "                     \"port\":\"30141\", \"secure\":True,\n",
    "                     \"user\":\"ibmlhapikey\", \"password\": token},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'source': 'https://www.tuasaude.com/receitas-de-crepioca-para-emagrecer/', 'pk': 452343555843950224}, page_content='Além disso, a crepioca é uma boa alternativa para pessoas com doença celíaca, já que não contém glúten, podendo substituir alimentos como pizzas, pães e bolos, por exemplo.\\n\\nInformação nutricional da crepioca\\n\\nA tabela a seguir traz a informação nutricional de 1 crepioca simples feita com 1 ovo e 2 colheres de sopa de tapioca:\\n\\nComponentes 1 crepioca simples Energia 146 calorias Proteínas 4,16 g Gorduras 3,48 g Carboidratos 24,55 g Fibras 0 g Vitamina A 45,2 mcg Luteína + zeaxantina 200,4 mcg\\n\\nPara se obter os benefícios da crepioca, é fundamental manter uma alimentação saudável e variada, além de fazer exercícios físicos regularmente.\\n\\nComo fazer a crepioca\\n\\nPara fazer a crepioca, basta misturar em uma tigela pequena 1 a 2 colheres de goma de tapioca, 1 ovo e sal. Em seguida, misturar todos os ingredientes com um garfo, até ficar bem homogêneo.'),\n",
       " Document(metadata={'source': 'https://www.tuasaude.com/receitas-de-crepioca-para-emagrecer/', 'pk': 452343555843950222}, page_content='Crepioca engorda? O que é, como fazer e calorias (com receitas)\\n\\nAtualizado em julho 2024\\n\\nRevisão clínica: Karla Leal Nutricionista\\n\\nEngorda?\\n\\nInformação nutricional\\n\\nComo fazer\\n\\nReceitas\\n\\nA crepioca é uma receita feita com ovo e tapioca, que pode conter frutas, queijo e aveia, por exemplo, podendo ser consumida em qualquer refeição do dia, como café da manhã, lanches, almoço e jantar.\\n\\nA crepioca ajuda no ganho ou manutenção da massa muscular, favorece o emagrecimento, mantém a saúde dos olhos, além de ser uma boa opção para pessoas com doença celíaca, por não conter glúten.\\n\\nLeia também: Glúten: o que é, alimentos que contêm e outras dúvidas tuasaude.com/gluten\\n\\nEsses benefícios são possíveis porque a crepioca contém proteínas, vitaminas e carotenoides, nutrientes que atuam na formação de músculos, têm ação antioxidante e promovem a sensação de saciedade.\\n\\nCrepioca engorda?'),\n",
       " Document(metadata={'source': 'https://www.tuasaude.com/receitas-de-crepioca-para-emagrecer/', 'pk': 452343555843950227}, page_content='Num pote, fazer uma pastinha com o queijo e o orégano, e reservar. Colocar o ovo, o sal e a goma de tapioca num pote e misturar tudo com um garfo.\\n\\nAquecer uma frigideira no fogo e transferir a massa, deixando assar até ambos os lados ficarem dourados. Transferir a crepioca para um prato e rechear com a pasta de queijo.\\n\\n5. Crepioca Doce\\n\\nIngredientes:\\n\\n1 ovo;\\n\\n2 colheres (de sopa) de goma de tapioca;\\n\\n1 colher (de chá) de cacau em pó sem açúcar;\\n\\n4 morangos lavados.\\n\\nModo de preparo:\\n\\nCortar os morangos em rodelas e reservar. Em uma tigela pequena, misturar o ovo e a goma de tapioca com o garfo até ficar uma massa homogênea. Adicionar o cacau em pó e misturar bem.\\n\\nLevar uma frigideira antiaderente ao fogo e adicionar a massa, assando dos dois lados. Transferir a crepioca para o prato, acrescentar os morangos e servir.\\n\\n6. Crepioca de aveia com legumes\\n\\nIngredientes:\\n\\n1 ovo;\\n\\n1 colher (de sopa) de goma de tapioca;\\n\\n1 colher (de sopa) de farelo de aveia;'),\n",
       " Document(metadata={'source': 'https://www.tuasaude.com/receitas-de-crepioca-para-emagrecer/', 'pk': 452343555843950226}, page_content='Modo de preparo:\\n\\nEm um recipiente fundo, bater bem o ovo com o garfo. Adicionar a goma de tapioca, o queijo e o sal, misturando bem. Levar para assar dos dois lados em frigideira preaquecida no fogo. Transferir para um prato e servir em seguida.\\n\\n3. Crepioca de frango\\n\\nIngredientes:\\n\\n1 ovo;\\n\\n2 colheres (de sopa) de goma de tapioca;\\n\\n2 colheres (de sopa) de frango cozido desfiado;\\n\\nSal, pimenta-do-reino e salsa a gosto.\\n\\nModo de preparo:\\n\\nEm um recipiente, bater bem o ovo com o garfo. Adicionar a goma, o frango e os temperos, e misturar tudo novamente. Aquecer uma frigideira antiaderente no fogo e despejar a mistura.\\n\\nAssar por cerca de 2 a 3 minutos em cada lado ou até a crepioca ficar dourada. Colocar num prato e servir em seguida.\\n\\n4. Crepioca fit\\n\\nIngredientes:\\n\\n1 ovo\\n\\n1 colher (de sopa) de goma de tapioca;\\n\\n1 colher (de sopa) de queijo cottage ou ricota;\\n\\nSal e orégano a gosto.\\n\\nModo de preparo:')]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TEST COLLECTION INITIALISATION WITH A SIMPLE SIMILARITY SEARCH\n",
    "\n",
    "query = \"Como fazer tapioca?\"\n",
    "docs = vector_db.similarity_search(query)\n",
    "\n",
    "docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SET LANGCHAIN INTERFACE FOR GEN AI\n",
    "\n",
    "creds = Credentials(token,\n",
    "                    api_endpoint=\"https://us-south.ml.cloud.ibm.com\") # credentials object to access the LLM service\n",
    "\n",
    "\n",
    "params = TextGenerationParameters(decoding_method=\"sample\",temperature=0.7, top_p=1, top_k=50, min_new_tokens=0, max_new_tokens=200\n",
    "                        )\n",
    "\n",
    "model = LangChainInterface(model_id= 'ibm/granite-20b-multilingual', parameters=params, client=Client(credentials=creds))\n",
    "\n",
    "prompt_template = (\n",
    "            \"\"\"[INST]\n",
    "            <|system|>\n",
    "            Aqui estão suas instruções:\n",
    "            1. Você receberá um contexto que deverá ser usado para responder às perguntas dos usuários. O contexto começará com \\\"BEGIN\\\" e terminará com \\\"END\\\".\n",
    "            2. Você deve gerar a próxima resposta utilizando informações disponíveis no contexto.\n",
    "            4. Não use nenhum outro conhecimento.\n",
    "            5. Especifique as frases no contexto que justificam a sua resposta.\n",
    "                BEGIN {context} END\n",
    "            [/INST]\n",
    "            {question}\n",
    "            <|assistant|>\n",
    "            \"\"\"\n",
    "        )\n",
    "\n",
    "customprompt = PromptTemplate(\n",
    "            template=prompt_template, input_variables=[\"context\", \"question\"]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LangChainInterface(client=<genai.client.Client object at 0x104f46f30>, model_id='ibm/granite-20b-multilingual', parameters=TextGenerationParameters(beam_width=None, decoding_method='sample', include_stop_sequence=None, length_penalty=None, max_new_tokens=200, min_new_tokens=0, random_seed=None, repetition_penalty=None, return_options=None, stop_sequences=None, temperature=0.7, time_limit=None, top_k=50, top_p=1.0, truncate_input_tokens=None, typical_p=None))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_docs(docs):\n",
    "    context = ''\n",
    "    for doc in docs:\n",
    "        context += doc.page_content\n",
    "    return context\n",
    "\n",
    "def get_wkc_token_on_cloud(apikey):\n",
    "    body = {\"grant_type\": \"urn:ibm:params:oauth:grant-type:apikey\",\n",
    "            \"apikey\": apikey}\n",
    "\n",
    "    token_request = requests.post('https://iam.cloud.ibm.com/identity/token',\n",
    "                                  data=body, headers={'content-type': 'application/x-www-form-urlencoded'})\n",
    "    try:\n",
    "        token_request.raise_for_status()\n",
    "    except requests.exceptions.HTTPError as e:\n",
    "        raise e  # return code not equals to 200\n",
    "\n",
    "    return json.loads(token_request.text)[\"access_token\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qa(question):\n",
    "    docs = vector_db.similarity_search(query)\n",
    "\n",
    "    print(str(len(docs))+\" nearest embeddings found;\")\n",
    "    \n",
    "    context = append_docs(docs=docs)\n",
    "    prompt = customprompt.format(context=context, question=question)\n",
    "\n",
    "    barear_token = get_wkc_token_on_cloud(token)\n",
    "\n",
    "    url = \"https://us-south.ml.cloud.ibm.com/ml/v1/text/generation?version=2023-05-29\"\n",
    "\n",
    "    body = {\n",
    "        \"input\": prompt,\n",
    "        \"parameters\": {\n",
    "            \"decoding_method\": \"greedy\",\n",
    "            \"max_new_tokens\": 900,\n",
    "        },\n",
    "        \"model_id\": \"ibm/granite-20b-multilingual\",\n",
    "        \"project_id\": project_id\n",
    "    }\n",
    "\n",
    "    headers = {\n",
    "        \"Accept\": \"application/json\",\n",
    "        \"Content-Type\": \"application/json\",\n",
    "        \"Authorization\": \"Bearer {}\".format(barear_token)\n",
    "    }\n",
    "\n",
    "    response = requests.post(\n",
    "        url,\n",
    "        headers=headers,\n",
    "        json=body\n",
    "    )\n",
    "\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\"Non-200 response: \" + str(response.text))\n",
    "    \n",
    "    data = response.json()\n",
    "    \n",
    "    return data['results'][0]['generated_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 nearest embeddings found;\n",
      " Aqui está uma receita simples de crepioca:\n",
      "             Ingredientes:\n",
      "             - 1 ovo;\n",
      "             - 2 colheres (de sopa) de goma de tapioca;\n",
      "             - Sal a gosto.\n",
      "\n",
      "1. Em um recipiente fundo, bater bem o ovo com o garfo.\n",
      "\n",
      "2. Adicionar a goma de tapioca e o sal, misturando bem.\n",
      "\n",
      "3. Despejar a mistura em uma frigideira antiaderente preaquecida no fogo.\n",
      "\n",
      "4. Deixar assar por cerca de 2 a 3 minutos em cada lado ou até a crepioca ficar dourada.\n",
      "\n",
      "5. Transferir para um prato e rechear com o queijo, frango, presunto, requeijão, frutas, entre outros ingredientes de sua preferência.\n",
      "\n",
      "A crepioca é uma ótima opção para o café da manhã, lanche da tarde, almoço ou jantar. É rápida e fácil de fazer, além de ser uma ótima fonte de proteínas, vitaminas e carboidratos. Você pode rechear com diversos ingredientes, de acordo com sua preferência e dieta.\n"
     ]
    }
   ],
   "source": [
    "print(qa(\"Como fazer crepioca?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 nearest embeddings found;\n",
      " A crepioca é uma opção de alimento que pode ser consumido sem medo, já que é considerada um alimento saudável. Ela é rica em proteínas, carboidratos e gorduras boas, que fornecem energia e ajudam no ganho de massa muscular.\n",
      "\n",
      "Além disso, a crepioca pode ser recheada com diversos ingredientes saudáveis, como frutas, queijo, aveia e até mesmo legumes, tornando-a uma opção nutritiva e saborosa.\n",
      "\n",
      "Portanto, a crepioca não é um alimento calórico, mas sim uma opção de alimento nutritivo e saboroso que pode fazer parte de uma dieta equilibrada e saudável.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(qa(\"Crepioca é calórico?\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "watsonx",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}