Split Chats from Media DB (BREAKING), Mindmap viewing in gradio from PlantUML, token counts for conversations

App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py

@@ -24,7 +24,7 @@
     wait_random_exponential,
 )
 
-from App_Function_Libraries.Chat import chat_api_call
+from App_Function_Libraries.Chat.Chat_Functions import chat_api_call
 
 #
 #######################################################################################################################

App_Function_Libraries/Chat.py → App_Function_Libraries/Chat/Chat_Functions.py

@@ -1,11 +1,12 @@
-# Chat.py
+# Chat_Functions.py
 # Chat functions for interacting with the LLMs as chatbots
 import base64
 # Imports
 import json
 import logging
 import os
 import re
+import sqlite3
 import tempfile
 import time
 from datetime import datetime
@@ -14,7 +15,9 @@
 # External Imports
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_conversation_name, save_chat_history_to_database
+from App_Function_Libraries.DB.DB_Manager import get_conversation_name, save_chat_history_to_database, \
+    start_new_conversation, update_conversation_title, delete_messages_in_conversation, save_message
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_db_connection
 from App_Function_Libraries.LLM_API_Calls import chat_with_openai, chat_with_anthropic, chat_with_cohere, \
     chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface
 from App_Function_Libraries.LLM_API_Calls_Local import chat_with_aphrodite, chat_with_local_llm, chat_with_ollama, \
@@ -27,6 +30,16 @@
 #
 # Functions:
 
+def approximate_token_count(history):
+    total_text = ''
+    for user_msg, bot_msg in history:
+        if user_msg:
+            total_text += user_msg + ' '
+        if bot_msg:
+            total_text += bot_msg + ' '
+    total_tokens = len(total_text.split())
+    return total_tokens
+
 def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None):
     log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint})
     start_time = time.time()
@@ -173,56 +186,58 @@ def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content, med
     log_counter("save_chat_history_to_db_attempt")
     start_time = time.time()
     logging.info(f"Attempting to save chat history. Media content type: {type(media_content)}")
+
     try:
-        # Extract the media_id and media_name from the media_content
-        media_id = None
-        if isinstance(media_content, dict):
+        # First check if we can access the database
+        try:
+            with get_db_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT 1")
+        except sqlite3.DatabaseError as db_error:
+            logging.error(f"Database is corrupted or inaccessible: {str(db_error)}")
+            return conversation_id, "Database error: The database file appears to be corrupted. Please contact support."
+
+        # Now attempt the save
+        if not conversation_id:
+            # Only for new conversations, not updates
             media_id = None
-            logging.debug(f"Media content keys: {media_content.keys()}")
-            if 'content' in media_content:
+            if isinstance(media_content, dict) and 'content' in media_content:
                 try:
                     content = media_content['content']
-                    if isinstance(content, str):
-                        content_json = json.loads(content)
-                    elif isinstance(content, dict):
-                        content_json = content
-                    else:
-                        raise ValueError(f"Unexpected content type: {type(content)}")
-
-                    # Use the webpage_url as the media_id
+                    content_json = content if isinstance(content, dict) else json.loads(content)
                     media_id = content_json.get('webpage_url')
-                    # Use the title as the media_name
-                    media_name = content_json.get('title')
-
-                    logging.info(f"Extracted media_id: {media_id}, media_name: {media_name}")
-                except json.JSONDecodeError:
-                    logging.error("Failed to decode JSON from media_content['content']")
-                except Exception as e:
-                    logging.error(f"Error processing media_content: {str(e)}")
+                    media_name = media_name or content_json.get('title', 'Unnamed Media')
+                except (json.JSONDecodeError, AttributeError) as e:
+                    logging.error(f"Error processing media content: {str(e)}")
+                    media_id = "unknown_media"
+                    media_name = media_name or "Unnamed Media"
             else:
-                logging.warning("'content' key not found in media_content")
-        else:
-            logging.warning(f"media_content is not a dictionary. Type: {type(media_content)}")
-
-        if media_id is None:
-            # If we couldn't find a media_id, we'll use a placeholder
-            media_id = "unknown_media"
-            logging.warning(f"Unable to extract media_id from media_content. Using placeholder: {media_id}")
-
-        if media_name is None:
-            media_name = "Unnamed Media"
-            logging.warning(f"Unable to extract media_name from media_content. Using placeholder: {media_name}")
+                media_id = "unknown_media"
+                media_name = media_name or "Unnamed Media"
+
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            conversation_title = f"{media_name}_{timestamp}"
+            conversation_id = start_new_conversation(title=conversation_title, media_id=media_id)
+            logging.info(f"Created new conversation with ID: {conversation_id}")
+
+        # For both new and existing conversations
+        try:
+            delete_messages_in_conversation(conversation_id)
+            for user_msg, assistant_msg in chatbot:
+                if user_msg:
+                    save_message(conversation_id, "user", user_msg)
+                if assistant_msg:
+                    save_message(conversation_id, "assistant", assistant_msg)
+        except sqlite3.DatabaseError as db_error:
+            logging.error(f"Database error during message save: {str(db_error)}")
+            return conversation_id, "Database error: Unable to save messages. Please try again or contact support."
 
-        # Generate a unique conversation name using media_id and current timestamp
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        conversation_name = f"{media_name}_{timestamp}"
-
-        new_conversation_id = save_chat_history_to_database(chatbot, conversation_id, media_id, media_name,
-                                                            conversation_name)
         save_duration = time.time() - start_time
         log_histogram("save_chat_history_to_db_duration", save_duration)
         log_counter("save_chat_history_to_db_success")
-        return new_conversation_id, f"Chat history saved successfully as {conversation_name}!"
+
+        return conversation_id, "Chat history saved successfully!"
+
     except Exception as e:
         log_counter("save_chat_history_to_db_error", labels={"error": str(e)})
         error_message = f"Failed to save chat history: {str(e)}"

App_Function_Libraries/DB/Character_Chat_DB.py

@@ -560,6 +560,7 @@ def delete_character_chat(chat_id: int) -> bool:
     finally:
         conn.close()
 
+
 def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
     """
     Fetch chat IDs associated with any of the specified keywords.
@@ -589,16 +590,14 @@ def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
     finally:
         conn.close()
 
+
 def save_chat_history_to_character_db(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]]) -> Optional[int]:
     """Save chat history to the CharacterChats table.
 
     Returns the ID of the inserted chat or None if failed.
     """
     return add_character_chat(character_id, conversation_name, chat_history)
 
-def migrate_chat_to_media_db():
-    pass
-
 
 def search_db(query: str, fields: List[str], where_clause: str = "", page: int = 1, results_per_page: int = 5) -> List[Dict[str, Any]]:
     """