Added offline llm support

.gitignore

@@ -1 +1,3 @@
-venv
+venv
+chroma_db
+models

app.py

@@ -15,33 +15,63 @@ def save_uploaded_file(uploaded_file):
         return None
 
 st.set_page_config(page_title="Datadoc", page_icon=":robot:", layout="wide", initial_sidebar_state="expanded")
-st.title("Datadoc: Your AI DOC Assistant")
+
+col1, col2 = st.columns([6,1])
+with col1:
+    st.title("Datadoc: Your AI DOC Assistant")
+
+with col2:
+    vert_space = '<div style="padding: 20px;"></div>'
+    st.markdown(vert_space, unsafe_allow_html=True)
+    offline_mode = st.toggle("Offline Mode", key='offline_toggle')
 
 if 'generated' not in st.session_state:
     st.session_state['generated'] = []
 if 'past' not in st.session_state:
     st.session_state['past'] = []
 
 st.sidebar.title("Options")
-model_name = st.sidebar.radio("Choose a model:", ("Gemini Pro", "Gemini Pro Vision"))
-api_key = st.text_input("Enter your API key:", type="password")
+
+if(offline_mode):
+    model_name = st.sidebar.radio("Choose a model:", ("GPT4All offline",))
+else:
+    model_name = st.sidebar.radio("Choose a model:", ("Gemini Pro", "Gemini Pro Vision"))
+
+if not offline_mode:
+    api_key = st.text_input("Enter your API key:", type="password")
+
 clear_button = st.sidebar.button("Clear Conversation", key="clear")
 
-if model_name == "Gemini Pro":
+if not offline_mode and model_name == "Gemini Pro":
     model = "gemini-pro"
-else:
+elif not offline_mode and model_name == "Gemini Pro Vision":
     model = "gemini-pro-vision"
+else:
+    model = "GPT4All"
 
 if clear_button:
     st.session_state['generated'] = []
     st.session_state['past'] = []
 
 def generate_response(prompt, model, image_path=None, explain_to_kid=False):
-    st.session_state['past'].append(prompt)
-    response = get_llm_response(prompt, model, image_path, api_key, explain_to_kid)
-    st.session_state['generated'].append(response)
+    if len(st.session_state['past']) == len(st.session_state['generated']):
+        st.session_state['past'].append(prompt)
+    else:
+        st.session_state['past'][-1] = prompt
+
+    if not offline_mode:
+        response = get_llm_response(prompt, model, image_path, api_key, explain_to_kid)
+    else:
+        response = get_llm_response(prompt, model, image_path, explain_to_kid, offline=True)
+
+    if len(st.session_state['generated']) < len(st.session_state['past']):
+        st.session_state['generated'].append(response)
+    else:
+        st.session_state['generated'][-1] = response
+
     return response
 
+
 # container for chat history
 response_container = st.container()
 # container for text box
@@ -63,9 +93,9 @@ def generate_response(prompt, model, image_path=None, explain_to_kid=False):
         with col2:
             explain_kid = st.toggle("Child Mode", key='explain_toggle')
 
-    if submit_button and not api_key:
+    if not offline_mode and submit_button and not api_key:
         st.warning("Please enter your API key.")
-    elif submit_button and not uploaded_file and model_name == "Gemini Pro Vision":
+    elif not offline_mode and submit_button and not uploaded_file and model_name == "Gemini Pro Vision":
         st.warning("Please upload an image to use the Image Model.")
     elif uploaded_file:
         image_path = save_uploaded_file(uploaded_file)

get_result.py

@@ -10,7 +10,7 @@
 import os
 from load_model import load_image_llm,load_text_llm
 
-def get_result(vectordb,query,model,image_path=None,chat_history=[],explain_to_kid=False):
+def get_result(vectordb,query,model,image_path=None,chat_history=[],explain_to_kid=False,is_offline=False):
 
     if(model=="gemini-pro-vision"):
 
@@ -28,7 +28,7 @@ def get_result(vectordb,query,model,image_path=None,chat_history=[],explain_to_k
         # save_string_to_txt(content)
         return content,chat_history
     else:
-        text_llm = load_text_llm()
+        text_llm = load_text_llm(is_offline)
 
         retriever = vectordb.as_retriever()
 
@@ -91,12 +91,10 @@ def format_docs(docs):
             | text_llm
         )
 
-        # if(query.find("summary")!=-1 or query.find("Summary")!=-1 or query.find("summarize")!=-1 or query.find("Summarize")!=-1) :
-        #     summarize_chain = load_summarize_chain(text_llm,chain_type="stuff")
-        #     search = vectordb.similarity_search(" ")
-        #     summary = summarize_chain.invoke(input=search, question="Write a summary within 300 words.")
-        #     return summary['output_text']
-        # else:
         res = rag_chain.invoke({"question": query, "chat_history": chat_history})
         chat_history.extend([HumanMessage(content=query), res])
-        return res.content,chat_history
+
+        if not is_offline:
+            return res.content, chat_history
+        else:
+            return res, chat_history

load_model.py

@@ -1,9 +1,14 @@
 from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_community.llms import GPT4All
 
 def load_image_llm():
     llm = ChatGoogleGenerativeAI(model="gemini-pro-vision",convert_system_message_to_human=True)
     return llm
 
-def load_text_llm():
-    llm = ChatGoogleGenerativeAI(model="gemini-pro",convert_system_message_to_human=True)
+def load_text_llm(offline=False):
+    if not offline:
+        llm = ChatGoogleGenerativeAI(model="gemini-pro",convert_system_message_to_human=True)
+    else:
+        llm = GPT4All(model="models/mistral-7b-openorca.gguf2.Q4_0.gguf", n_threads=8)
+
     return llm

main.py

@@ -4,10 +4,12 @@
 
 chat_history = []
 
-def get_llm_response(query,model,image_path=None,api_key=None,explain_to_kid=False):
+def get_llm_response(query,model,image_path=None,api_key=None,explain_to_kid=False,offline=False):
     global chat_history
 
-    os.environ["GOOGLE_API_KEY"] = api_key
+    if not offline:
+        os.environ["GOOGLE_API_KEY"] = api_key
+
     vectordb = chroma_db_store()
-    result, chat_history = get_result(vectordb,query,model,image_path,chat_history,explain_to_kid)
+    result, chat_history = get_result(vectordb,query,model,image_path,chat_history,explain_to_kid,offline)
     return result

readme.md

@@ -1,5 +1,7 @@
 # Datadoc: Your AI DOC Assistant
 
+🌟 **Feature Update**: Offline LLM support. Now you can run whole system offline. [Click here](https://github.com/Adiii1436/datadoc?tab=readme-ov-file#features-). 
+
 Welcome to Datadoc, your personal AI document assistant. Datadoc is designed to help you extract information from documents without having to read or remember the entire content. It's like having a personal assistant who has read all your documents and can instantly recall any piece of information from them. 
 
 ![Screenshot from 2024-03-20 00-50-20](https://github.com/Adiii1436/datadoc/assets/73269919/26e596fd-722b-4f0e-92b5-4e68d611693b)
@@ -12,6 +14,10 @@ Welcome to Datadoc, your personal AI document assistant. Datadoc is designed to
     ![Screenshot from 2024-03-18 20-00-30](https://github.com/Adiii1436/datadoc/assets/73269919/77a890bd-2ade-4359-8693-44101902ff2a)
     ![Screenshot from 2024-03-18 20-09-18](https://github.com/Adiii1436/datadoc/assets/73269919/4d7367e9-6b6e-402e-859c-ed926f8d68e8)
 
+- **Offline Support**: Datadoc supports offline mode i.e now you can the LLM model locally on your system. And you also don't need GPU for this. If you prefer to run LLM locally you can use this feature.
+   - Download the Model: Download **mistral-7b-openorca.gguf2.Q4_0.gguf** model from the **Model Explorer** section in GPT4All [website](https://gpt4all.io/index.html).
+   - Place the model inside `models/mistral-7b-openorca.gguf2.Q4_0.gguf`.
+
 - **Child Mode**: It enables LLMs to elucidate topics as if they're explaining to a child. This feature proves invaluable for providing detailed and easily understandable explanations for each topic.
 
    - Without child mode: