2_confluence_opensearch_http.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "812192f0-e1b8-4f71-8ee9-3f0d6637aa1a",
   "metadata": {},
   "source": [
    "# **Parse Confluence page into OCI OpenSearch Vectors**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b9365b9-82fe-4364-9187-63c986064527",
   "metadata": {},
   "source": [
    "Steps in OCI Functions:\n",
    "- Invoked by API Gateway / Confluence\n",
    "- Will retrieve the changed page_id\n",
    "- Will fetch the content from the page_id and chunk the content\n",
    "- Convert the chunks of texts into embeddings\n",
    "- Push embeddingsto OCI OpenSearch\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e99b2ad0-8796-4bdc-8f8b-77db6cb0ac56",
   "metadata": {},
   "source": [
    "## **Full func.py script**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4d808db1-f6e8-405c-95c9-952f1b32b59f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting func.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile func.py\n",
    "\n",
    "import io\n",
    "import json\n",
    "from fdk import response\n",
    "from atlassian import Confluence\n",
    "from bs4 import BeautifulSoup\n",
    "import tiktoken\n",
    "from langchain.vectorstores import OpenSearchVectorSearch\n",
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "import oci\n",
    "from tqdm import tqdm\n",
    "\n",
    "#load configuration\n",
    "from config import confluence_space_id, full_confluence_url, username_confluence, atlassian_api_token, host, username, password, index_name\n",
    "\n",
    "#Load all customer functions\n",
    "from helpers import create_embedding_model\n",
    "from helpers import ingest_documents_with_embeddings\n",
    "from helpers import chunk_text\n",
    "from helpers import parse_page\n",
    "from helpers import create_confluence_client\n",
    "from helpers import create_opensearch_client\n",
    "\n",
    "\n",
    "def handler(ctx, data: io.BytesIO=None):\n",
    "    \n",
    "    page_id = \"no_page_id_available\"\n",
    "    page_url = \"no_page_url_available\"\n",
    "\n",
    "    try:\n",
    "        \n",
    "        ## establish connections\n",
    "        confluence_client = create_confluence_client(full_confluence_url, username_confluence, atlassian_api_token)\n",
    "        oci_opensearch_client = create_opensearch_client(host, username, password)\n",
    "        \n",
    "        #get the body and values\n",
    "        body = json.loads(data.getvalue())\n",
    "        print(f\"Printing the body {body}\")\n",
    "        page_id = body.get(\"page_id\")\n",
    "        page_url = body.get(\"page_url\")\n",
    "        print(f\"Printing the page_id {page_id}\")\n",
    "        print(f\"Printing the page_url {page_url}\")\n",
    "        \n",
    "        # parse the page and create chunks\n",
    "        chunks = parse_page(confluence_client, page_id)\n",
    "        \n",
    "        #parse chunks of text into embeddings\n",
    "        embedding_model = create_embedding_model()\n",
    "        \n",
    "        # convert chunks into embeddings and push to OCI OpenSearch.\n",
    "        ingest_documents_with_embeddings(document_chunks=chunks, index_name=index_name, oci_opensearch_client=oci_opensearch_client, host=host, username=username, password=password, embedding_model=embedding_model, batch_size=5)\n",
    "        \n",
    "    except (Exception, ValueError) as ex:\n",
    "        print(str(ex), flush=True)\n",
    "        print(\"Not loaded any data\")\n",
    "\n",
    "    return response.Response(\n",
    "        ctx, response_data=json.dumps(\n",
    "            {\"message\": \"The changed page id is {0}\".format(page_id)}),\n",
    "        headers={\"Content-Type\": \"application/json\"}\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "35ac8ed1-c6ad-4ab1-a5c0-c9877828877a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile requirements.txt\n",
    "fdk>=0.1.86\n",
    "oci\n",
    "atlassian-python-api\n",
    "beautifulsoup4\n",
    "tiktoken\n",
    "opensearch-py\n",
    "transformers\n",
    "torch\n",
    "langchain\n",
    "langchain-huggingface\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "315ba169-cc53-4c73-9235-b32bc7d5f456",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30e282a2-1d8b-49d2-a3c2-7af67a0c0c3b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54718522-b89e-4bac-adab-185e88e60652",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch21_p39_gpu_v1]",
   "language": "python",
   "name": "conda-env-pytorch21_p39_gpu_v1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}