diff --git a/App_Function_Libraries/Chunk_Lib.py b/App_Function_Libraries/Chunk_Lib.py
index 384d38433..cd566cf25 100644
--- a/App_Function_Libraries/Chunk_Lib.py
+++ b/App_Function_Libraries/Chunk_Lib.py
@@ -7,6 +7,7 @@
####
# Import necessary libraries
import hashlib
+import json
import logging
import re
from typing import Any, Dict, List, Optional, Tuple
@@ -72,42 +73,53 @@ def load_document(file_path):
def improved_chunking_process(text: str, custom_chunk_options: Dict[str, Any] = None) -> List[Dict[str, Any]]:
logging.debug("Improved chunking process started...")
+
+ # Extract JSON metadata if present
+ json_content = {}
+ try:
+ json_end = text.index("}\n") + 1
+ json_content = json.loads(text[:json_end])
+ text = text[json_end:].strip()
+ logging.debug(f"Extracted JSON metadata: {json_content}")
+ except (ValueError, json.JSONDecodeError):
+ logging.debug("No JSON metadata found at the beginning of the text")
+
+ # Extract any additional header text
+ header_match = re.match(r"(This text was transcribed using.*?)\n\n", text, re.DOTALL)
+ header_text = ""
+ if header_match:
+ header_text = header_match.group(1)
+ text = text[len(header_text):].strip()
+ logging.debug(f"Extracted header text: {header_text}")
+
options = chunk_options.copy()
if custom_chunk_options:
options.update(custom_chunk_options)
chunk_method = options.get('method', 'words')
- base_size = options.get('base_size', 1000)
- min_size = options.get('min_size', 100)
max_size = options.get('max_size', 2000)
overlap = options.get('overlap', 0)
language = options.get('language', None)
- adaptive = options.get('adaptive', False)
- multi_level = options.get('multi_level', False)
if language is None:
language = detect_language(text)
- if adaptive:
- max_chunk_size = adaptive_chunk_size(text, base_size, min_size, max_size)
- else:
- max_chunk_size = base_size
-
- if multi_level:
- chunks = multi_level_chunking(text, chunk_method, max_chunk_size, overlap, language)
- else:
- chunks = chunk_text(text, chunk_method, max_chunk_size, overlap, language)
+ chunks = chunk_text(text, chunk_method, max_size, overlap, language)
chunks_with_metadata = []
+ total_chunks = len(chunks)
for i, chunk in enumerate(chunks):
- metadata = get_chunk_metadata(
- chunk,
- text,
- chunk_type=chunk_method,
- language=language
- )
- metadata['chunk_index'] = i
- metadata['total_chunks'] = len(chunks)
+ metadata = {
+ 'chunk_index': i,
+ 'total_chunks': total_chunks,
+ 'chunk_method': chunk_method,
+ 'max_size': max_size,
+ 'overlap': overlap,
+ 'language': language,
+ 'relative_position': i / total_chunks
+ }
+ metadata.update(json_content) # Add the extracted JSON content to metadata
+ metadata['header_text'] = header_text # Add the header text to metadata
chunks_with_metadata.append({
'text': chunk,
@@ -117,6 +129,7 @@ def improved_chunking_process(text: str, custom_chunk_options: Dict[str, Any] =
return chunks_with_metadata
+
def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]:
logging.debug("Multi-level chunking process started...")
# First level: chunk by paragraphs
diff --git a/App_Function_Libraries/Gradio_Related.py b/App_Function_Libraries/Gradio_Related.py
index 32a55efe1..7ad47559d 100644
--- a/App_Function_Libraries/Gradio_Related.py
+++ b/App_Function_Libraries/Gradio_Related.py
@@ -39,8 +39,9 @@
from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_view_tab, create_prompt_search_tab, \
create_search_summaries_tab, create_viewing_tab, create_search_tab
-from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_embeddings_tab, create_rag_tab, \
- create_view_embeddings_tab
+from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
+from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
+ create_purge_embeddings_tab
from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
create_delete_trash_tab, create_search_and_mark_trash_tab
from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
@@ -260,11 +261,9 @@ def launch_ui(share_public=None, server_mode=False):
create_search_tab()
create_search_summaries_tab()
- with gr.TabItem("RAG Search / Embeddings"):
+ with gr.TabItem("RAG Search"):
create_rag_tab()
create_rag_qa_chat_tab()
- create_embeddings_tab()
- create_view_embeddings_tab()
with gr.TabItem("Chat with an LLM"):
create_chat_interface()
@@ -295,6 +294,11 @@ def launch_ui(share_public=None, server_mode=False):
# FIXME
#create_compare_transcripts_tab()
+ with gr.TabItem("Embeddings Management"):
+ create_embeddings_tab()
+ create_view_embeddings_tab()
+ create_purge_embeddings_tab()
+
with gr.TabItem("Writing Tools"):
with gr.Tabs():
from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
diff --git a/App_Function_Libraries/Gradio_UI/Embeddings_tab.py b/App_Function_Libraries/Gradio_UI/Embeddings_tab.py
new file mode 100644
index 000000000..14ee790f0
--- /dev/null
+++ b/App_Function_Libraries/Gradio_UI/Embeddings_tab.py
@@ -0,0 +1,365 @@
+# Embeddings_tabc.py
+# Description: This file contains the code for the RAG Chat tab in the Gradio UI
+#
+# Imports
+import json
+import logging
+#
+# External Imports
+import gradio as gr
+
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process, determine_chunk_position
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database
+from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, \
+ store_in_chroma
+from App_Function_Libraries.RAG.Embeddings_Create import create_embedding
+#
+########################################################################################################################
+#
+# Functions:
+
+# FIXME - under construction
+def create_embeddings_tab():
+ with gr.TabItem("Create Embeddings"):
+ gr.Markdown("# Create Embeddings for All Content")
+
+ with gr.Row():
+ with gr.Column():
+ embedding_provider = gr.Radio(
+ choices=["huggingface", "local", "openai"],
+ label="Select Embedding Provider",
+ value="huggingface"
+ )
+ gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
+ gr.Markdown("OpenAI provider requires a valid API key. ")
+ gr.Markdown("OpenAI Embeddings models: `text-embedding-3-small`, `text-embedding-3-large`")
+ gr.Markdown("HuggingFace provider requires a valid model name, i.e. `dunzhang/stella_en_400M_v5`")
+ embedding_model = gr.Textbox(
+ label="Embedding Model",
+ value="Enter your embedding model name here", lines=3
+ )
+ embedding_api_url = gr.Textbox(
+ label="API URL (for local provider)",
+ value="http://localhost:8080/embedding",
+ visible=False
+ )
+
+ # Add chunking options
+ chunking_method = gr.Dropdown(
+ choices=["words", "sentences", "paragraphs", "tokens", "semantic"],
+ label="Chunking Method",
+ value="words"
+ )
+ max_chunk_size = gr.Slider(
+ minimum=1, maximum=8000, step=1, value=500,
+ label="Max Chunk Size"
+ )
+ chunk_overlap = gr.Slider(
+ minimum=0, maximum=4000, step=1, value=200,
+ label="Chunk Overlap"
+ )
+ adaptive_chunking = gr.Checkbox(
+ label="Use Adaptive Chunking",
+ value=False
+ )
+
+ create_button = gr.Button("Create Embeddings")
+
+ with gr.Column():
+ status_output = gr.Textbox(label="Status", lines=10)
+
+ def update_provider_options(provider):
+ return gr.update(visible=provider == "local")
+
+ embedding_provider.change(
+ fn=update_provider_options,
+ inputs=[embedding_provider],
+ outputs=[embedding_api_url]
+ )
+
+ def create_all_embeddings(provider, model, api_url, method, max_size, overlap, adaptive):
+ try:
+ all_content = get_all_content_from_database()
+ if not all_content:
+ return "No content found in the database."
+
+ chunk_options = {
+ 'method': method,
+ 'max_size': max_size,
+ 'overlap': overlap,
+ 'adaptive': adaptive
+ }
+
+ collection_name = "all_content_embeddings"
+ collection = chroma_client.get_or_create_collection(name=collection_name)
+
+ for item in all_content:
+ media_id = item['id']
+ text = item['content']
+
+ chunks = improved_chunking_process(text, chunk_options)
+ for i, chunk in enumerate(chunks):
+ chunk_text = chunk['text']
+ chunk_id = f"doc_{media_id}_chunk_{i}"
+
+ existing = collection.get(ids=[chunk_id])
+ if existing['ids']:
+ continue
+
+ embedding = create_embedding(chunk_text, provider, model, api_url)
+ metadata = {
+ "media_id": str(media_id),
+ "chunk_index": i,
+ "total_chunks": len(chunks),
+ "chunking_method": method,
+ "max_chunk_size": max_size,
+ "chunk_overlap": overlap,
+ "adaptive_chunking": adaptive,
+ "embedding_model": model,
+ "embedding_provider": provider,
+ **chunk['metadata']
+ }
+ store_in_chroma(collection_name, [chunk_text], [embedding], [chunk_id], [metadata])
+
+ return "Embeddings created and stored successfully for all content."
+ except Exception as e:
+ logging.error(f"Error during embedding creation: {str(e)}")
+ return f"Error: {str(e)}"
+
+ create_button.click(
+ fn=create_all_embeddings,
+ inputs=[embedding_provider, embedding_model, embedding_api_url,
+ chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking],
+ outputs=status_output
+ )
+
+
+def create_view_embeddings_tab():
+ with gr.TabItem("View/Update Embeddings"):
+ gr.Markdown("# View and Update Embeddings")
+ item_mapping = gr.State({})
+ with gr.Row():
+ with gr.Column():
+ item_dropdown = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+ refresh_button = gr.Button("Refresh Item List")
+ embedding_status = gr.Textbox(label="Embedding Status", interactive=False)
+ embedding_preview = gr.Textbox(label="Embedding Preview", interactive=False, lines=5)
+ embedding_metadata = gr.Textbox(label="Embedding Metadata", interactive=False, lines=10)
+
+ with gr.Column():
+ create_new_embedding_button = gr.Button("Create New Embedding")
+ embedding_provider = gr.Radio(
+ choices=["huggingface", "local", "openai"],
+ label="Select Embedding Provider",
+ value="huggingface"
+ )
+ gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
+ gr.Markdown("OpenAI provider requires a valid API key. ")
+ gr.Markdown("OpenAI Embeddings models: `text-embedding-3-small`, `text-embedding-3-large`")
+ gr.Markdown("HuggingFace provider requires a valid model name, i.e. `dunzhang/stella_en_400M_v5`")
+ embedding_model = gr.Textbox(
+ label="Embedding Model",
+ value="Enter your embedding model name here", lines=3
+ )
+ embedding_api_url = gr.Textbox(
+ label="API URL (for local provider)",
+ value="http://localhost:8080/embedding",
+ visible=False
+ )
+ chunking_method = gr.Dropdown(
+ choices=["words", "sentences", "paragraphs", "tokens", "semantic"],
+ label="Chunking Method",
+ value="words"
+ )
+ max_chunk_size = gr.Slider(
+ minimum=1, maximum=8000, step=1, value=500,
+ label="Max Chunk Size"
+ )
+ chunk_overlap = gr.Slider(
+ minimum=0, maximum=5000, step=1, value=200,
+ label="Chunk Overlap"
+ )
+ adaptive_chunking = gr.Checkbox(
+ label="Use Adaptive Chunking",
+ value=False
+ )
+
+ def get_items_with_embedding_status():
+ try:
+ items = get_all_content_from_database()
+ collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
+ choices = []
+ new_item_mapping = {}
+ for item in items:
+ try:
+ result = collection.get(ids=[f"doc_{item['id']}_chunk_0"])
+ embedding_exists = result is not None and result.get('ids') and len(result['ids']) > 0
+ status = "Embedding exists" if embedding_exists else "No embedding"
+ except Exception as e:
+ print(f"Error checking embedding for item {item['id']}: {str(e)}")
+ status = "Error checking"
+ choice = f"{item['title']} ({status})"
+ choices.append(choice)
+ new_item_mapping[choice] = item['id']
+ return gr.update(choices=choices), new_item_mapping
+ except Exception as e:
+ print(f"Error in get_items_with_embedding_status: {str(e)}")
+ return gr.update(choices=["Error: Unable to fetch items"]), {}
+
+ def update_provider_options(provider):
+ return gr.update(visible=provider == "local")
+
+ def check_embedding_status(selected_item, item_mapping):
+ if not selected_item:
+ return "Please select an item", "", ""
+
+ try:
+ item_id = item_mapping.get(selected_item)
+ if item_id is None:
+ return f"Invalid item selected: {selected_item}", "", ""
+
+ item_title = selected_item.rsplit(' (', 1)[0]
+ collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
+
+ result = collection.get(ids=[f"doc_{item_id}_chunk_0"], include=["embeddings", "metadatas"])
+ logging.info(f"ChromaDB result for item '{item_title}' (ID: {item_id}): {result}")
+
+ if not result['ids']:
+ return f"No embedding found for item '{item_title}' (ID: {item_id})", "", ""
+
+ if not result['embeddings'] or not result['embeddings'][0]:
+ return f"Embedding data missing for item '{item_title}' (ID: {item_id})", "", ""
+
+ embedding = result['embeddings'][0]
+ metadata = result['metadatas'][0] if result['metadatas'] else {}
+ embedding_preview = str(embedding[:50])
+ status = f"Embedding exists for item '{item_title}' (ID: {item_id})"
+ return status, f"First 50 elements of embedding:\n{embedding_preview}", json.dumps(metadata, indent=2)
+
+ except Exception as e:
+ logging.error(f"Error in check_embedding_status: {str(e)}")
+ return f"Error processing item: {selected_item}. Details: {str(e)}", "", ""
+
+ def create_new_embedding_for_item(selected_item, provider, model, api_url, method, max_size, overlap, adaptive, item_mapping):
+ if not selected_item:
+ return "Please select an item", "", ""
+
+ try:
+ item_id = item_mapping.get(selected_item)
+ if item_id is None:
+ return f"Invalid item selected: {selected_item}", "", ""
+
+ items = get_all_content_from_database()
+ item = next((item for item in items if item['id'] == item_id), None)
+ if not item:
+ return f"Item not found: {item_id}", "", ""
+
+ chunk_options = {
+ 'method': method,
+ 'max_size': max_size,
+ 'overlap': overlap,
+ 'adaptive': adaptive
+ }
+
+ chunks = improved_chunking_process(item['content'], chunk_options)
+ collection_name = "all_content_embeddings"
+ collection = chroma_client.get_or_create_collection(name=collection_name)
+
+ # Delete existing embeddings for this item
+ existing_ids = [f"doc_{item_id}_chunk_{i}" for i in range(len(chunks))]
+ collection.delete(ids=existing_ids)
+
+ for i, chunk in enumerate(chunks):
+ chunk_text = chunk['text']
+ chunk_metadata = chunk['metadata']
+ chunk_position = determine_chunk_position(chunk_metadata['relative_position'])
+
+ chunk_header = f"""
+ Original Document: {item['title']}
+ Chunk: {i + 1} of {len(chunks)}
+ Position: {chunk_position}
+ Header: {chunk_metadata.get('header_text', 'N/A')}
+
+ --- Chunk Content ---
+ """
+
+ full_chunk_text = chunk_header + chunk_text
+ chunk_id = f"doc_{item_id}_chunk_{i}"
+ embedding = create_embedding(full_chunk_text, provider, model, api_url)
+ metadata = {
+ "media_id": str(item_id),
+ "chunk_index": i,
+ "total_chunks": len(chunks),
+ "chunking_method": method,
+ "max_chunk_size": max_size,
+ "chunk_overlap": overlap,
+ "adaptive_chunking": adaptive,
+ "embedding_model": model,
+ "embedding_provider": provider,
+ **chunk_metadata
+ }
+ store_in_chroma(collection_name, [full_chunk_text], [embedding], [chunk_id], [metadata])
+
+ embedding_preview = str(embedding[:50])
+ status = f"New embeddings created and stored for item: {item['title']} (ID: {item_id})"
+ return status, f"First 50 elements of new embedding:\n{embedding_preview}", json.dumps(metadata, indent=2)
+ except Exception as e:
+ logging.error(f"Error in create_new_embedding_for_item: {str(e)}")
+ return f"Error creating embedding: {str(e)}", "", ""
+
+ refresh_button.click(
+ get_items_with_embedding_status,
+ outputs=[item_dropdown, item_mapping]
+ )
+ item_dropdown.change(
+ check_embedding_status,
+ inputs=[item_dropdown, item_mapping],
+ outputs=[embedding_status, embedding_preview, embedding_metadata]
+ )
+ create_new_embedding_button.click(
+ create_new_embedding_for_item,
+ inputs=[item_dropdown, embedding_provider, embedding_model, embedding_api_url,
+ chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking, item_mapping],
+ outputs=[embedding_status, embedding_preview, embedding_metadata]
+ )
+ embedding_provider.change(
+ update_provider_options,
+ inputs=[embedding_provider],
+ outputs=[embedding_api_url]
+ )
+
+ return item_dropdown, refresh_button, embedding_status, embedding_preview, embedding_metadata, create_new_embedding_button, embedding_provider, embedding_model, embedding_api_url, chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking
+
+
+def create_purge_embeddings_tab():
+ with gr.TabItem("Purge Embeddings"):
+ gr.Markdown("# Purge Embeddings")
+
+ with gr.Row():
+ with gr.Column():
+ purge_button = gr.Button("Purge All Embeddings")
+ with gr.Column():
+ status_output = gr.Textbox(label="Status", lines=10)
+
+ def purge_all_embeddings():
+ try:
+ collection_name = "all_content_embeddings"
+ chroma_client.delete_collection(collection_name)
+ chroma_client.create_collection(collection_name)
+ return "All embeddings have been purged successfully."
+ except Exception as e:
+ logging.error(f"Error during embedding purge: {str(e)}")
+ return f"Error: {str(e)}"
+
+ purge_button.click(
+ fn=purge_all_embeddings,
+ outputs=status_output
+ )
+
+
+
+#
+# End of file
+########################################################################################################################
diff --git a/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py b/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py
index 0d793b028..40f34a870 100644
--- a/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py
+++ b/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py
@@ -8,10 +8,7 @@
import gradio as gr
#
# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database
-from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, \
- check_embedding_status, store_in_chroma
-from App_Function_Libraries.RAG.Embeddings_Create import create_embedding
+
from App_Function_Libraries.RAG.RAG_Libary_2 import enhanced_rag_pipeline
#
########################################################################################################################
@@ -71,176 +68,6 @@ def perform_rag_search(query, keywords, api_choice):
search_button.click(perform_rag_search, inputs=[search_query, keywords_input, api_choice], outputs=[result_output, context_output])
-# FIXME - under construction
-def create_embeddings_tab():
- with gr.TabItem("Create Embeddings"):
- gr.Markdown("# Create Embeddings for All Content")
-
- with gr.Row():
- with gr.Column():
- embedding_provider = gr.Radio(
- choices=["openai", "local", "huggingface"],
- label="Select Embedding Provider",
- value="openai"
- )
- embedding_model = gr.Textbox(
- label="Embedding Model",
- value="text-embedding-3-small"
- )
- embedding_api_url = gr.Textbox(
- label="API URL (for local provider)",
- value="http://localhost:8080/embedding",
- visible=False
- )
- create_button = gr.Button("Create Embeddings")
-
- with gr.Column():
- status_output = gr.Textbox(label="Status", lines=10)
-
- def update_provider_options(provider):
- return gr.update(visible=provider == "local")
-
- embedding_provider.change(
- fn=update_provider_options,
- inputs=[embedding_provider],
- outputs=[embedding_api_url]
- )
-
- def create_all_embeddings(provider, model, api_url):
- try:
- all_content = get_all_content_from_database()
- if not all_content:
- return "No content found in the database."
-
- collection_name = "all_content_embeddings"
- collection = chroma_client.get_or_create_collection(name=collection_name)
-
- for item in all_content:
- media_id = item['id']
- text = item['content']
-
- existing = collection.get(ids=[f"doc_{media_id}"])
- if existing['ids']:
- continue
-
- embedding = create_embedding(text, provider, model, api_url)
- store_in_chroma(collection_name, [text], [embedding], [f"doc_{media_id}"], [{"media_id": media_id}])
-
- return "Embeddings created and stored successfully for all new content."
- except Exception as e:
- logging.error(f"Error during embedding creation: {str(e)}")
- return f"Error: {str(e)}"
-
- create_button.click(
- fn=create_all_embeddings,
- inputs=[embedding_provider, embedding_model, embedding_api_url],
- outputs=status_output
- )
-
-
-def create_view_embeddings_tab():
- with gr.TabItem("View/Update Embeddings"):
- gr.Markdown("# View and Update Embeddings")
- item_mapping = gr.State({})
- with gr.Row():
- with gr.Column():
- item_dropdown = gr.Dropdown(label="Select Item", choices=[], interactive=True)
- refresh_button = gr.Button("Refresh Item List")
- embedding_status = gr.Textbox(label="Embedding Status", interactive=False)
- embedding_preview = gr.Textbox(label="Embedding Preview", interactive=False, lines=5)
-
- with gr.Column():
- create_new_embedding_button = gr.Button("Create New Embedding")
- embedding_provider = gr.Radio(
- choices=["openai", "local", "huggingface"],
- label="Embedding Provider",
- value="openai"
- )
- embedding_model = gr.Textbox(
- label="Embedding Model",
- value="text-embedding-3-small",
- visible=True
- )
- embedding_api_url = gr.Textbox(
- label="API URL (for local provider)",
- value="http://localhost:8080/embedding",
- visible=False
- )
-
- def get_items_with_embedding_status():
- try:
- items = get_all_content_from_database()
- collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
- choices = []
- new_item_mapping = {}
- for item in items:
- try:
- result = collection.get(ids=[f"doc_{item['id']}"])
- embedding_exists = result is not None and result.get('ids') and len(result['ids']) > 0
- status = "Embedding exists" if embedding_exists else "No embedding"
- except Exception as e:
- print(f"Error checking embedding for item {item['id']}: {str(e)}")
- status = "Error checking"
- choice = f"{item['title']} ({status})"
- choices.append(choice)
- new_item_mapping[choice] = item['id']
- return gr.update(choices=choices), new_item_mapping
- except Exception as e:
- print(f"Error in get_items_with_embedding_status: {str(e)}")
- return gr.update(choices=["Error: Unable to fetch items"]), {}
-
- def update_provider_options(provider):
- return gr.update(visible=provider == "local")
-
- def create_new_embedding_for_item(selected_item, provider, model, api_url, item_mapping):
- if not selected_item:
- return "Please select an item", ""
-
- try:
- item_id = item_mapping.get(selected_item)
- if item_id is None:
- return f"Invalid item selected: {selected_item}", ""
-
- items = get_all_content_from_database()
- item = next((item for item in items if item['id'] == item_id), None)
- if not item:
- return f"Item not found: {item_id}", ""
-
- embedding = create_embedding(item['content'], provider, model, api_url)
-
- collection_name = "all_content_embeddings"
- metadata = {"media_id": item_id, "title": item['title']}
- store_in_chroma(collection_name, [item['content']], [embedding], [f"doc_{item_id}"],
- [{"media_id": item_id, "title": item['title']}])
-
- embedding_preview = str(embedding[:50])
- status = f"New embedding created and stored for item: {item['title']} (ID: {item_id})"
- return status, f"First 50 elements of new embedding:\n{embedding_preview}\n\nMetadata: {metadata}"
- except Exception as e:
- logging.error(f"Error in create_new_embedding_for_item: {str(e)}")
- return f"Error creating embedding: {str(e)}", ""
-
- refresh_button.click(
- get_items_with_embedding_status,
- outputs=[item_dropdown, item_mapping]
- )
- item_dropdown.change(
- check_embedding_status,
- inputs=[item_dropdown, item_mapping],
- outputs=[embedding_status, embedding_preview]
- )
- create_new_embedding_button.click(
- create_new_embedding_for_item,
- inputs=[item_dropdown, embedding_provider, embedding_model, embedding_api_url, item_mapping],
- outputs=[embedding_status, embedding_preview]
- )
- embedding_provider.change(
- update_provider_options,
- inputs=[embedding_provider],
- outputs=[embedding_api_url]
- )
-
- return item_dropdown, refresh_button, embedding_status, embedding_preview, create_new_embedding_button, embedding_provider, embedding_model, embedding_api_url
#
# End of file
diff --git a/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py b/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py
index a409ce671..1fe0a27f2 100644
--- a/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py
+++ b/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py
@@ -156,9 +156,9 @@ def update_prompts(preset_name):
with gr.Column():
chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'],
label="Chunking Method")
- max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50,
+ max_chunk_size = gr.Slider(minimum=100, maximum=8000, value=400, step=1,
label="Max Chunk Size")
- chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
+ chunk_overlap = gr.Slider(minimum=0, maximum=5000, value=100, step=1, label="Chunk Overlap")
use_adaptive_chunking = gr.Checkbox(
label="Use Adaptive Chunking (Adjust chunking based on text complexity)")
use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking")
diff --git a/App_Function_Libraries/RAG/ChromaDB_Library.py b/App_Function_Libraries/RAG/ChromaDB_Library.py
index cb2eb719e..7c2ca197c 100644
--- a/App_Function_Libraries/RAG/ChromaDB_Library.py
+++ b/App_Function_Libraries/RAG/ChromaDB_Library.py
@@ -14,7 +14,6 @@
from App_Function_Libraries.DB.SQLite_DB import process_chunks
from App_Function_Libraries.RAG.Embeddings_Create import create_embeddings_batch
# FIXME - related to Chunking
-from App_Function_Libraries.DB.DB_Manager import update_fts_for_media, db
from App_Function_Libraries.RAG.Embeddings_Create import create_embedding
from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
from App_Function_Libraries.Utils.Utils import get_database_path, ensure_directory_exists, \
diff --git a/App_Function_Libraries/RAG/Embeddings_Create.py b/App_Function_Libraries/RAG/Embeddings_Create.py
index 1a11f3fc6..93086a7cd 100644
--- a/App_Function_Libraries/RAG/Embeddings_Create.py
+++ b/App_Function_Libraries/RAG/Embeddings_Create.py
@@ -185,6 +185,9 @@ def create_openai_embedding(text: str, model: str) -> List[float]:
embedding = get_openai_embeddings(text, model)
return embedding
+
+
+
#Dead
# def create_local_embedding(text: str, model: str, api_url: str, api_key: str) -> List[float]:
# response = requests.post(
diff --git a/App_Function_Libraries/Tests/Integration/test_audio_transcription.py b/App_Function_Libraries/Tests/Integration/test_audio_transcription.py
new file mode 100644
index 000000000..d363b5ae3
--- /dev/null
+++ b/App_Function_Libraries/Tests/Integration/test_audio_transcription.py
@@ -0,0 +1,105 @@
+# test_audio_transcription.py
+# Integration test for audio transcription functionality
+# Usage: pytest tests/integration/test_audio_transcription.py
+import pytest
+import os
+import requests
+from pydub import AudioSegment
+from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav, speech_to_text
+from App_Function_Libraries.DB.DB_Manager import add_media_to_database
+from App_Function_Libraries.Utils.Utils import create_download_directory
+
+
+@pytest.fixture
+def test_audio_url():
+ # URL to a short, public domain audio file
+ return "https://upload.wikimedia.org/wikipedia/commons/1/1f/Dial_up_modem_noises.ogg"
+
+
+@pytest.fixture
+def test_download_path(tmp_path):
+ return create_download_directory(str(tmp_path))
+
+
+def download_audio(url, download_path):
+ response = requests.get(url)
+ if response.status_code == 200:
+ file_name = url.split("/")[-1]
+ file_path = os.path.join(download_path, file_name)
+ with open(file_path, "wb") as f:
+ f.write(response.content)
+ return file_path
+ else:
+ raise Exception(f"Failed to download audio file. Status code: {response.status_code}")
+
+@pytest.mark.slow
+def test_audio_transcription_integration(test_audio_url, test_download_path):
+ try:
+ # Step 1: Download test audio file
+ original_audio_path = download_audio(test_audio_url, test_download_path)
+ assert os.path.exists(original_audio_path)
+
+ # Step 2: Convert audio to WAV if it's not already in WAV format
+ if not original_audio_path.lower().endswith('.wav'):
+ audio = AudioSegment.from_file(original_audio_path)
+ wav_path = os.path.splitext(original_audio_path)[0] + '.wav'
+ audio.export(wav_path, format="wav")
+ else:
+ wav_path = original_audio_path
+
+ assert os.path.exists(wav_path)
+ assert wav_path.lower().endswith('.wav')
+
+ # Step 3: Perform speech-to-text
+ whisper_model = "tiny" # Using 'tiny' for faster testing, adjust as needed
+ segments = speech_to_text(wav_path, whisper_model=whisper_model, vad_filter=True)
+
+ assert segments is not None
+ assert len(segments) > 0
+
+ # Step 4: Add media to database
+ info_dict = {
+ "title": "Test Audio Transcription",
+ "uploader": "Integration Test",
+ "upload_date": "20220101"
+ }
+ summary = "Test audio transcription summary"
+ keywords = "test,integration,audio,transcription"
+ custom_prompt = "This is a test prompt for audio transcription"
+
+ result = add_media_to_database(
+ test_audio_url,
+ info_dict,
+ segments,
+ summary,
+ keywords,
+ custom_prompt,
+ whisper_model
+ )
+ assert "added/updated successfully" in result
+
+ # Additional assertions
+ assert any('Text' in segment for segment in segments), "No text found in transcription segments"
+
+ # Combine all transcribed text
+ transcription_text = ' '.join([segment.get('Text', '') for segment in segments])
+ assert len(transcription_text) > 0, "Transcription is empty"
+
+ # Check for expected content in the transcription
+ # Note: This audio file contains modem noises, so actual speech content might be limited
+ # Adjust these expected phrases based on what you expect in your test audio
+ expected_phrases = ["noise", "sound", "modem"]
+ for phrase in expected_phrases:
+ assert phrase.lower() in transcription_text.lower(), f"Expected phrase '{phrase}' not found in transcription"
+
+ # You could also check for the presence of timestamps or other expected metadata in the segments
+
+ except Exception as e:
+ pytest.fail(f"Integration test failed with error: {str(e)}")
+
+ finally:
+ # Clean up: remove downloaded files
+ if 'original_audio_path' in locals() and os.path.exists(original_audio_path):
+ os.remove(original_audio_path)
+ if 'wav_path' in locals() and os.path.exists(wav_path) and wav_path != original_audio_path:
+ os.remove(wav_path)
\ No newline at end of file
diff --git a/App_Function_Libraries/Tests/Integration/test_book_ingestion.py b/App_Function_Libraries/Tests/Integration/test_book_ingestion.py
new file mode 100644
index 000000000..6a2e90bd7
--- /dev/null
+++ b/App_Function_Libraries/Tests/Integration/test_book_ingestion.py
@@ -0,0 +1,107 @@
+# test_book_ingestion.py
+# Integration test for book ingestion functionality
+# Usage: pytest tests/integration/test_book_ingestion.py
+# This test assumes that your ingest_text_file function handles the parsing of the book's metadata (title, author, keywords) from the file content. If your function doesn't do this, you might need to adjust the test or the function to handle this.
+# The test doesn't check for things like chapter structure or more complex book formatting. Depending on your requirements, you might want to add more detailed checks.
+#
+import pytest
+import os
+import tempfile
+from App_Function_Libraries.Book_Ingestion_Lib import ingest_text_file
+from App_Function_Libraries.DB.DB_Manager import db, fetch_item_details
+
+
+@pytest.fixture
+def sample_book_content():
+ return """Title: Test Book
+Author: Integration Tester
+Keywords: test, integration, book
+
+Chapter 1: Introduction
+
+This is a test book for integration testing.
+It contains multiple lines and simulates a simple book structure.
+
+Chapter 2: Content
+
+The content of this book is not particularly meaningful.
+It's just for testing the book ingestion process.
+
+Chapter 3: Conclusion
+
+This concludes our test book. Thank you for reading!
+"""
+
+
+@pytest.fixture
+def sample_book_file(tmp_path, sample_book_content):
+ book_file = tmp_path / "test_book.txt"
+ with open(book_file, "w") as f:
+ f.write(sample_book_content)
+ return str(book_file)
+
+
+def test_book_ingestion_integration(sample_book_file):
+ try:
+ # Step 1: Ingest the book
+ result = ingest_text_file(
+ file_path=sample_book_file,
+ title="Test Book",
+ author="Integration Tester",
+ keywords="test,integration,book"
+ )
+ assert "ingested successfully" in result, f"Book ingestion failed: {result}"
+
+ # Step 2: Verify the book was added to the database
+ with db.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT id, title, author FROM Media WHERE title = ?", ("Test Book",))
+ book_record = cursor.fetchone()
+
+ assert book_record is not None, "Book not found in database"
+ book_id, book_title, book_author = book_record
+ assert book_title == "Test Book"
+ assert book_author == "Integration Tester"
+
+ # Step 3: Fetch and verify book details
+ content, prompt, summary = fetch_item_details(book_id)
+
+ assert "This is a test book for integration testing." in content
+ assert "Chapter 1: Introduction" in content
+ assert "Chapter 2: Content" in content
+ assert "Chapter 3: Conclusion" in content
+
+ # Verify that the content doesn't contain the metadata
+ assert "Title: Test Book" not in content
+ assert "Author: Integration Tester" not in content
+ assert "Keywords: test, integration, book" not in content
+
+ # Step 4: Verify keywords
+ cursor.execute("""
+ SELECT k.keyword
+ FROM Keywords k
+ JOIN MediaKeywords mk ON k.id = mk.keyword_id
+ WHERE mk.media_id = ?
+ """, (book_id,))
+ keywords = [row[0] for row in cursor.fetchall()]
+ assert set(keywords) == {"test", "integration", "book"}
+
+ # Additional checks can be added here, such as verifying the summary or prompt if applicable
+
+ except Exception as e:
+ pytest.fail(f"Integration test failed with error: {str(e)}")
+
+ finally:
+ # Clean up: remove the test book file
+ if os.path.exists(sample_book_file):
+ os.remove(sample_book_file)
+
+ # Optionally, remove the book from the database
+ with db.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("DELETE FROM Media WHERE title = ?", ("Test Book",))
+ conn.commit()
+
+
+if __name__ == "__main__":
+ pytest.main([__file__])
\ No newline at end of file
diff --git a/App_Function_Libraries/Tests/Integration/test_pdf_ingestion.py b/App_Function_Libraries/Tests/Integration/test_pdf_ingestion.py
new file mode 100644
index 000000000..2652f5f42
--- /dev/null
+++ b/App_Function_Libraries/Tests/Integration/test_pdf_ingestion.py
@@ -0,0 +1 @@
+cc
\ No newline at end of file
diff --git a/App_Function_Libraries/Tests/Integration/test_video_transcription.py b/App_Function_Libraries/Tests/Integration/test_video_transcription.py
new file mode 100644
index 000000000..53a946fc4
--- /dev/null
+++ b/App_Function_Libraries/Tests/Integration/test_video_transcription.py
@@ -0,0 +1,80 @@
+# test_video_transcription.py
+# Integration test for video transcription functionality
+# Usage: pytest tests/integration/test_video_transcription.py
+import pytest
+import os
+from App_Function_Libraries.Video_DL_Ingestion_Lib import download_video, extract_video_info
+from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav, speech_to_text
+from App_Function_Libraries.DB.DB_Manager import add_media_to_database
+from App_Function_Libraries.Utils.Utils import create_download_directory
+
+
+@pytest.fixture
+def test_video_url():
+ return "https://www.youtube.com/watch?v=dQw4w9WgXcQ" # Use a known, stable video URL for testing
+
+
+@pytest.fixture
+def test_download_path(tmp_path):
+ return create_download_directory(str(tmp_path))
+
+@pytest.mark.slow
+def test_video_transcription_integration(test_video_url, test_download_path):
+ try:
+ # Step 1: Extract video info
+ info_dict, title = extract_video_info(test_video_url)
+ assert info_dict is not None
+ assert title is not None
+
+ # Step 2: Download video
+ video_path = download_video(test_video_url, test_download_path, info_dict, True, "medium")
+ assert os.path.exists(video_path)
+
+ # Step 3: Convert video to WAV
+ audio_file_path = convert_to_wav(video_path, offset=0)
+ assert os.path.exists(audio_file_path)
+ assert audio_file_path.endswith('.wav')
+
+ # Step 4: Perform speech-to-text
+ segments = speech_to_text(audio_file_path, whisper_model="tiny", vad_filter=True)
+ assert segments is not None
+ assert len(segments) > 0
+
+ # Step 5: Add media to database
+ summary = "Test summary"
+ keywords = "test,integration,video"
+ custom_prompt = "This is a test prompt"
+ whisper_model = "tiny"
+
+ result = add_media_to_database(
+ test_video_url,
+ info_dict,
+ segments,
+ summary,
+ keywords,
+ custom_prompt,
+ whisper_model
+ )
+ assert "added/updated successfully" in result
+
+ # Additional assertions
+ assert any('Text' in segment for segment in segments), "No text found in transcription segments"
+
+ # You might want to check the content of the transcription
+ transcription_text = ' '.join([segment.get('Text', '') for segment in segments])
+ assert len(transcription_text) > 0, "Transcription is empty"
+
+ # Check if certain expected words are in the transcription (adjust based on the known content of your test video)
+ expected_words = ["never", "gonna", "give", "you", "up"] # Adjust these based on your test video content
+ for word in expected_words:
+ assert word.lower() in transcription_text.lower(), f"Expected word '{word}' not found in transcription"
+
+ except Exception as e:
+ pytest.fail(f"Integration test failed with error: {str(e)}")
+
+ finally:
+ # Clean up: remove downloaded files
+ if 'video_path' in locals() and os.path.exists(video_path):
+ os.remove(video_path)
+ if 'audio_file_path' in locals() and os.path.exists(audio_file_path):
+ os.remove(audio_file_path)
\ No newline at end of file
diff --git a/Docs/Database.md b/Docs/Database.md
new file mode 100644
index 000000000..8130052af
--- /dev/null
+++ b/Docs/Database.md
@@ -0,0 +1,39 @@
+# Database Notes
+
+## Table of Contents
+- []()
+- []()
+- []()
+- []()
+--------------------
+
+--------------------
+### SQLite
+- **101**
+ * https://www.sqlite.org/draft/about.html
+- **Concurrency**
+ * https://www.sqlite.org/lockingv3.html
+- **Write-Ahead Logging**
+ * https://www.sqlite.org/wal.html
+- **Links**
+ * https://antonz.org/sqlite-is-not-a-toy-database/
+ * https://antonz.org/json-virtual-columns/
+
+#### Writeup of SQLite Database in tldw
+- **F**
+ -
+--------------------
+
+--------------------
+### PostgreSQL
+- **101**
+ * https://www.postgresql.org/docs/current/index.html
+--------------------
+
+--------------------
+### OpenSearch
+- **101**
+ * https://opensearch.org/docs/latest/
+--------------------
+
+
diff --git a/Server_API/API_Plan.md b/Server_API/API_Plan.md
new file mode 100644
index 000000000..2a4498497
--- /dev/null
+++ b/Server_API/API_Plan.md
@@ -0,0 +1,94 @@
+# API Plan
+
+## Overview
+- First stab at a mapping of the API endpoints and their functionality
+
+- **Media Management**
+```
+ GET /api/media - List all media items
+ GET /api/media/{id} - Get details of a specific media item
+ POST /api/media - Add a new media item
+ PUT /api/media/{id} - Update an existing media item
+ DELETE /api/media/{id} - Delete a media item
+```
+
+- **Transcription and Summarization**
+```
+ POST /api/transcribe - Transcribe audio/video
+ POST /api/summarize - Summarize text content
+```
+
+- **Keyword Management**
+```
+ GET /api/keywords - List all keywords
+ POST /api/keywords - Add a new keyword
+ DELETE /api/keywords/{keyword} - Delete a keyword
+```
+
+- **Search**
+```
+ GET /api/search - Search across all content
+```
+
+- **PDF Operations**
+```
+ POST /api/pdf/parse - Parse a PDF file
+ POST /api/pdf/ingest - Ingest a PDF into the database
+```
+
+- **Book Operations**
+```
+ POST /api/book/ingest - Ingest a book into the database
+```
+
+- **Chat Management**
+```
+ GET /api/chat - List all chat conversations
+ GET /api/chat/{id} - Get details of a specific chat conversation
+ POST /api/chat - Create a new chat conversation
+ POST /api/chat/{id}/message - Add a message to a chat conversation
+ PUT /api/chat/{id}/message/{message_id} - Update a chat message
+ DELETE /api/chat/{id}/message/{message_id} - Delete a chat message
+```
+
+- **Document Versioning**
+```
+ GET /api/document/{id}/versions - List all versions of a document
+ GET /api/document/{id}/versions/{version_number} - Get a specific version of a document
+ POST /api/document/{id}/versions - Create a new version of a document
+```
+
+- **RAG (Retrieval-Augmented Generation)**
+```
+ POST /api/rag/search - Perform a RAG search
+```
+
+- **Embedding Management**
+```
+ POST /api/embeddings - Create embeddings for content
+ GET /api/embeddings/{id} - Get embeddings for a specific item
+```
+
+- **Prompt Management**
+```
+ GET /api/prompts - List all prompts
+ GET /api/prompts/{id} - Get details of a specific prompt
+ POST /api/prompts - Create a new prompt
+ PUT /api/prompts/{id} - Update an existing prompt
+ DELETE /api/prompts/{id} - Delete a prompt
+```
+
+- **Import Management**
+```
+ POST /api/import - Import content from an external source
+```
+
+- **Trash Management**
+```
+ GET /api/trash - List items in trash
+ POST /api/trash/{id} - Move an item to trash
+ DELETE /api/trash/{id} - Permanently delete an item from trash
+ POST /api/trash/{id}/restore - Restore an item from trash
+```
+
+
diff --git a/requirements.txt b/requirements.txt
index 6b28e5120..068c01442 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,8 +27,9 @@ fire==0.6.0
flatbuffers==24.3.25
fonttools==4.51.0
fsspec==2024.3.1
-gradio==4.29.0
-gradio_client==0.16.1
+#gradio==4.29.0
+gradio==4.44.0
+gradio_client==1.3.0
h11==0.14.0
httpcore==1.0.5
httptools==0.6.1
@@ -99,7 +100,8 @@ timm==0.9.16
tokenizers==0.15.2
tomlkit==0.12.0
toolz==0.12.1
-torchvision==0.17.2
+#torchvision==0.17.2
+torchvision==0.19.1
tqdm==4.66.3
#Inference Engine
transformers==4.39.3